[{"data":1,"prerenderedAt":43857},["ShallowReactive",2],{"blog-posts":3,"content-\u002Fposts\u002Fllm-memory-compute-tradeoff":143},[4,17,25,32,41,49,55,61,66,71,78,85,93,99,106,114,120,126,132,138],{"_path":5,"title":6,"description":7,"date":8,"categories":9,"tags":10,"draft":16},"\u002Fposts\u002Fllm-memory-compute-tradeoff","现代 LLM 的 Memory-Compute Trade-off：从 MHA 到 MLA、Sparse 与 Linear Attention","ds4.1f 上线了。它的架构 novel 在哪，便宜和快又是从哪省出来的。","2026-09-11 12:00:00","技术杂谈",[11,12,13,14,15],"LLMs","Attention","MLA","DeepSeek","长上下文",false,{"_path":18,"title":19,"description":20,"date":21,"categories":22,"tags":23,"draft":16},"\u002Fposts\u002Fwhy-we-still-need-to-write-a-blog","为什么还需要手写博客？","AI这么强，还要古法写博客吗？","2026-09-01 13:17:53","生活杂谈",[24],"博客",{"_path":26,"title":27,"description":28,"date":29,"categories":30,"tags":31,"draft":16},"\u002Fposts\u002Finteresting-music-rhythm","有趣的音乐节奏","","2026-07-16 12:47:24","音乐",null,{"_path":33,"title":34,"description":35,"date":36,"categories":37,"tags":38,"draft":16},"\u002Fposts\u002Ftowards-monosemanticity","LLM 可解释性（一）：SAE与单语义","读 Anthropic《Towards Monosemanticity》。从难以解释的神经元出发, 讲清superposition为什么让单个神经元混成一团, SAE怎么把多语义的激活分解成单语义特征。","2026-07-12 22:00:00","技术",[11,39,40],"可解释性","SAE",{"_path":42,"title":43,"description":28,"date":44,"categories":37,"tags":45,"draft":16},"\u002Fposts\u002Fclaude-code-settings","Claude Code 配置分享","2026-06-23 14:22:12",[11,46,47,48],"AI Agent","Agent","Claude Code",{"_path":50,"title":51,"description":52,"date":53,"categories":9,"tags":54,"draft":16},"\u002Fposts\u002Fmy-thoughts-on-the-ai-industry","简单聊聊对 AI Agent 行业的理解","写这篇文章的初衷，是我发觉逐渐忘了前两个月的 AI\u002FLLMs 是什么样的能力和情况了。意识中似乎某些在突然加速，但理性和直觉告诉我这不对，作为行内人，我知道这是一个渐进的、累积的过程。我想自己回忆一下，Agent的发展。","2026-06-20 13:14:25",[11,46,47],{"_path":56,"title":57,"description":28,"date":58,"categories":37,"tags":59,"draft":16},"\u002Fposts\u002Frotational-positional-encoding","RoPE","2026-06-17 18:31:40",[11,60,57],"Transformer",{"_path":62,"title":63,"description":28,"date":64,"categories":37,"tags":65,"draft":16},"\u002Fposts\u002Fintroduction-to-large-language-model","大语言模型入门基础","2026-06-16 22:36:11",[60,11],{"_path":67,"title":68,"description":28,"date":69,"categories":9,"tags":70,"draft":16},"\u002Fposts\u002Fthought-about-coding-agent","关于 Coding Agent 使用的遐思","2026-06-03 17:54:20",[47],{"_path":72,"title":73,"description":74,"date":75,"categories":37,"tags":76},"\u002Fposts\u002Fabout-ssh-usage","关于SSH的使用——客户端篇","介绍SSH客户端的用法。","2025-06-17 02:44:57",[77],"工具",{"_path":79,"title":80,"description":81,"date":82,"categories":9,"tags":83},"\u002Fposts\u002Fopensourcesoftwarestarttutorial","如何开始一个开源项目？","浅谈我对开源项目的认知。","2025-04-17 18:00:00",[84],"开源",{"_path":86,"title":87,"description":88,"date":89,"categories":90,"tags":91},"\u002Fposts\u002Fmotivation","渴求动力","有意识的生活并主动地行动会使得生活更美好。","2025-03-05 00:44:57","生活-情感",[92],"随笔",{"_path":94,"title":95,"description":96,"date":97,"categories":90,"tags":98},"\u002Fposts\u002F2024summary","2024年终总结","啊，痛彻的时光；啊，悲怆的人！","2025-01-01 01:00:00",[92],{"_path":100,"title":101,"description":102,"date":103,"categories":37,"tags":104},"\u002Fposts\u002Fvps-crash-startup","快速搭建个人VPS","如何快速搭建个人VPS。","2024-12-31 19:00:00",[105],"VPS",{"_path":107,"title":108,"description":109,"date":110,"categories":37,"tags":111},"\u002Fposts\u002Fdata-label-software","AI数据（自动）标注软件概述","对市面上目前的AI数据（自动）标注软件进行调研。","2024-12-20 14:00:00",[112,113],"数据标注","深度学习",{"_path":115,"title":116,"description":117,"date":118,"categories":90,"tags":119},"\u002Fposts\u002Fself-destruction","生活的崩坏与自我毁灭","是什么在牵引着我？我的行为，我的思想？我的感受？我所处的地方？它们在何处？","2024-12-19 12:00:00",[92],{"_path":121,"title":122,"description":123,"date":124,"categories":37,"tags":125},"\u002Fposts\u002Fcs194-196_llmagents","LLM Agents 课程笔记","UCB CS294\u002F194-196 Large Language Model Agents 的课程笔记。","2024-11-30 20:00:00",[47],{"_path":127,"title":128,"description":129,"date":130,"categories":90,"tags":131},"\u002Fposts\u002Fwhatsyourworth","价值、位置","你们的生存究竟有何价值？如果毫无价值，你们究竟为何存在？","2024-11-29 02:00:00",[92],{"_path":133,"title":134,"description":135,"date":136,"categories":90,"tags":137},"\u002Fposts\u002Fblogrelifever2","重启博客v2","忙里偷闲学习前端，还是决定结合想法手搓一个自己的博客。","2024-11-25 00:00:00",[24],{"_path":139,"title":140,"description":141,"date":142,"categories":90,"tags":31},"\u002Fposts\u002Fblogrelife","重启博客","在我的表达欲与现实所需(问题解决记录)驱动下，我决定重写博客。","2024-10-21 20:44:57",{"_path":5,"_dir":144,"_draft":16,"_partial":16,"_locale":28,"title":6,"description":7,"date":8,"categories":9,"tags":145,"draft":16,"body":146,"_type":43851,"_id":43852,"_source":43853,"_file":43854,"_stem":43855,"_extension":43856},"posts",[11,12,13,14,15],{"type":147,"children":148,"toc":43833},"root",[149,163,168,175,1000,1562,3196,4676,5351,5639,5821,6022,6027,6032,6038,6063,6075,6087,6092,6523,7262,7634,7640,7833,7838,7843,7937,8060,8315,8445,9168,9490,9495,9657,9673,9800,9808,9820,9825,9830,9984,10939,11153,12378,13090,13095,14071,14263,15438,16905,17243,17248,17253,18249,18531,19937,20489,20889,21464,21472,22172,23090,23095,23380,23385,23627,23632,24911,24962,25468,25571,26086,26133,26760,26816,26826,26831,27017,27022,27027,27077,27599,27690,28120,28124,28709,28714,28719,28742,29206,29216,29227,29232,29237,29507,29772,29777,29887,29897,29902,30330,30429,30435,30442,30871,30879,30884,30975,30980,31013,31018,31023,31238,31340,31455,31659,31665,31891,31896,32043,32048,32054,32066,32586,32591,33030,33367,33456,33813,34004,34016,34022,34033,34039,34044,34099,34104,34165,34191,34196,34201,34278,34442,34448,34829,34837,34981,35127,35363,35368,35535,35715,35729,35734,35831,35933,36401,36406,36420,36464,36601,36722,36771,36811,36816,36821,36826,37379,37393,37405,37417,37422,37499,37504,37509,37741,37746,37911,38080,38367,38372,38382,38387,38505,38767,38772,38833,38844,38849,38854,38859,39062,39072,39077,39318,39323,39328,39336,39341,39346,39733,39738,40260,40590,40847,40974,40979,41243,41251,41579,41680,41765,41775,41780,41785,41888,41893,41903,42153,42163,42326,42331,42540,42545,42830,42840,42845,42850,42862,43007,43012,43022,43027,43033,43038,43052,43066,43071,43076,43088,43187,43207,43258,43263,43271,43276,43281,43391,43396,43561,43566,43609,43619,43624,43629,43634,43639,43644,43775,43787,43792,43802,43812,43822,43827],{"type":150,"tag":151,"props":152,"children":153},"element","p",{},[154,157],{"type":155,"value":156},"text","GPT6出来之后，体感上很多人都很焦虑迷茫，真的到了AGI time，接下来该做什么呢？虽然说Astra用下来，在一些case上还是会overfit，但真的已经能胜任大多数人完成的任务了（在数字世界里）。这两天ds4.1f正式上线了，如此novel的架构让我感到新奇，模型架构与硬件的co-design达到一定水平的杰作，让我们来看一下吧。",{"type":150,"tag":158,"props":159,"children":160},"strong",{},[161],{"type":155,"value":162},"这篇由 AI 整理。",{"type":150,"tag":151,"props":164,"children":165},{},[166],{"type":155,"value":167},"从 MHA 出发，一路到 Linear Attention，看每一代分别在砍哪一项成本。",{"type":150,"tag":169,"props":170,"children":172},"h2",{"id":171},"mha-与-kv-cache",[173],{"type":155,"value":174},"MHA 与 KV Cache",{"type":150,"tag":151,"props":176,"children":177},{},[178,180,528,530,722,724,768,770,838,840,998],{"type":155,"value":179},"MHA（Multi-Head Attention）是这一切的起点，2017 年《Attention Is All You Need》里提出来的。数学上它就是多个独立的单头注意力拼起来。假设输入向量序列是 ",{"type":150,"tag":181,"props":182,"children":185},"span",{"className":183},[184],"katex",[186,277],{"type":150,"tag":181,"props":187,"children":190},{"className":188},[189],"katex-mathml",[191],{"type":150,"tag":192,"props":193,"children":195},"math",{"xmlns":194},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[196],{"type":150,"tag":197,"props":198,"children":199},"semantics",{},[200,270],{"type":150,"tag":201,"props":202,"children":203},"mrow",{},[204,220,227,239,243,248,254,258],{"type":150,"tag":205,"props":206,"children":207},"msub",{},[208,214],{"type":150,"tag":209,"props":210,"children":211},"mi",{},[212],{"type":155,"value":213},"x",{"type":150,"tag":215,"props":216,"children":217},"mn",{},[218],{"type":155,"value":219},"1",{"type":150,"tag":221,"props":222,"children":224},"mo",{"separator":223},"true",[225],{"type":155,"value":226},",",{"type":150,"tag":205,"props":228,"children":229},{},[230,234],{"type":150,"tag":209,"props":231,"children":232},{},[233],{"type":155,"value":213},{"type":150,"tag":215,"props":235,"children":236},{},[237],{"type":155,"value":238},"2",{"type":150,"tag":221,"props":240,"children":241},{"separator":223},[242],{"type":155,"value":226},{"type":150,"tag":221,"props":244,"children":245},{},[246],{"type":155,"value":247},"⋯",{"type":150,"tag":249,"props":250,"children":251},"mtext",{},[252],{"type":155,"value":253}," ",{"type":150,"tag":221,"props":255,"children":256},{"separator":223},[257],{"type":155,"value":226},{"type":150,"tag":205,"props":259,"children":260},{},[261,265],{"type":150,"tag":209,"props":262,"children":263},{},[264],{"type":155,"value":213},{"type":150,"tag":209,"props":266,"children":267},{},[268],{"type":155,"value":269},"l",{"type":150,"tag":271,"props":272,"children":274},"annotation",{"encoding":273},"application\u002Fx-tex",[275],{"type":155,"value":276},"x_1, x_2, \\cdots, x_l",{"type":150,"tag":181,"props":278,"children":281},{"className":279,"ariaHidden":223},[280],"katex-html",[282],{"type":150,"tag":181,"props":283,"children":286},{"className":284},[285],"base",[287,293,368,374,380,437,442,446,452,456,460,465,469],{"type":150,"tag":181,"props":288,"children":292},{"className":289,"style":291},[290],"strut","height:0.625em;vertical-align:-0.1944em;",[],{"type":150,"tag":181,"props":294,"children":297},{"className":295},[296],"mord",[298,304],{"type":150,"tag":181,"props":299,"children":302},{"className":300},[296,301],"mathnormal",[303],{"type":155,"value":213},{"type":150,"tag":181,"props":305,"children":308},{"className":306},[307],"msupsub",[309],{"type":150,"tag":181,"props":310,"children":314},{"className":311},[312,313],"vlist-t","vlist-t2",[315,356],{"type":150,"tag":181,"props":316,"children":319},{"className":317},[318],"vlist-r",[320,349],{"type":150,"tag":181,"props":321,"children":325},{"className":322,"style":324},[323],"vlist","height:0.3011em;",[326],{"type":150,"tag":181,"props":327,"children":329},{"style":328},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[330,336],{"type":150,"tag":181,"props":331,"children":335},{"className":332,"style":334},[333],"pstrut","height:2.7em;",[],{"type":150,"tag":181,"props":337,"children":343},{"className":338},[339,340,341,342],"sizing","reset-size6","size3","mtight",[344],{"type":150,"tag":181,"props":345,"children":347},{"className":346},[296,342],[348],{"type":155,"value":219},{"type":150,"tag":181,"props":350,"children":353},{"className":351},[352],"vlist-s",[354],{"type":155,"value":355},"​",{"type":150,"tag":181,"props":357,"children":359},{"className":358},[318],[360],{"type":150,"tag":181,"props":361,"children":364},{"className":362,"style":363},[323],"height:0.15em;",[365],{"type":150,"tag":181,"props":366,"children":367},{},[],{"type":150,"tag":181,"props":369,"children":372},{"className":370},[371],"mpunct",[373],{"type":155,"value":226},{"type":150,"tag":181,"props":375,"children":379},{"className":376,"style":378},[377],"mspace","margin-right:0.1667em;",[],{"type":150,"tag":181,"props":381,"children":383},{"className":382},[296],[384,389],{"type":150,"tag":181,"props":385,"children":387},{"className":386},[296,301],[388],{"type":155,"value":213},{"type":150,"tag":181,"props":390,"children":392},{"className":391},[307],[393],{"type":150,"tag":181,"props":394,"children":396},{"className":395},[312,313],[397,426],{"type":150,"tag":181,"props":398,"children":400},{"className":399},[318],[401,421],{"type":150,"tag":181,"props":402,"children":404},{"className":403,"style":324},[323],[405],{"type":150,"tag":181,"props":406,"children":407},{"style":328},[408,412],{"type":150,"tag":181,"props":409,"children":411},{"className":410,"style":334},[333],[],{"type":150,"tag":181,"props":413,"children":415},{"className":414},[339,340,341,342],[416],{"type":150,"tag":181,"props":417,"children":419},{"className":418},[296,342],[420],{"type":155,"value":238},{"type":150,"tag":181,"props":422,"children":424},{"className":423},[352],[425],{"type":155,"value":355},{"type":150,"tag":181,"props":427,"children":429},{"className":428},[318],[430],{"type":150,"tag":181,"props":431,"children":433},{"className":432,"style":363},[323],[434],{"type":150,"tag":181,"props":435,"children":436},{},[],{"type":150,"tag":181,"props":438,"children":440},{"className":439},[371],[441],{"type":155,"value":226},{"type":150,"tag":181,"props":443,"children":445},{"className":444,"style":378},[377],[],{"type":150,"tag":181,"props":447,"children":450},{"className":448},[449],"minner",[451],{"type":155,"value":247},{"type":150,"tag":181,"props":453,"children":455},{"className":454,"style":378},[377],[],{"type":150,"tag":181,"props":457,"children":459},{"className":458,"style":378},[377],[],{"type":150,"tag":181,"props":461,"children":463},{"className":462},[371],[464],{"type":155,"value":226},{"type":150,"tag":181,"props":466,"children":468},{"className":467,"style":378},[377],[],{"type":150,"tag":181,"props":470,"children":472},{"className":471},[296],[473,478],{"type":150,"tag":181,"props":474,"children":476},{"className":475},[296,301],[477],{"type":155,"value":213},{"type":150,"tag":181,"props":479,"children":481},{"className":480},[307],[482],{"type":150,"tag":181,"props":483,"children":485},{"className":484},[312,313],[486,517],{"type":150,"tag":181,"props":487,"children":489},{"className":488},[318],[490,512],{"type":150,"tag":181,"props":491,"children":494},{"className":492,"style":493},[323],"height:0.3361em;",[495],{"type":150,"tag":181,"props":496,"children":497},{"style":328},[498,502],{"type":150,"tag":181,"props":499,"children":501},{"className":500,"style":334},[333],[],{"type":150,"tag":181,"props":503,"children":505},{"className":504},[339,340,341,342],[506],{"type":150,"tag":181,"props":507,"children":510},{"className":508,"style":509},[296,301,342],"margin-right:0.0197em;",[511],{"type":155,"value":269},{"type":150,"tag":181,"props":513,"children":515},{"className":514},[352],[516],{"type":155,"value":355},{"type":150,"tag":181,"props":518,"children":520},{"className":519},[318],[521],{"type":150,"tag":181,"props":522,"children":524},{"className":523,"style":363},[323],[525],{"type":150,"tag":181,"props":526,"children":527},{},[],{"type":155,"value":529},"，其中 ",{"type":150,"tag":181,"props":531,"children":533},{"className":532},[184],[534,584],{"type":150,"tag":181,"props":535,"children":537},{"className":536},[189],[538],{"type":150,"tag":192,"props":539,"children":540},{"xmlns":194},[541],{"type":150,"tag":197,"props":542,"children":543},{},[544,579],{"type":150,"tag":201,"props":545,"children":546},{},[547,559,564],{"type":150,"tag":205,"props":548,"children":549},{},[550,554],{"type":150,"tag":209,"props":551,"children":552},{},[553],{"type":155,"value":213},{"type":150,"tag":209,"props":555,"children":556},{},[557],{"type":155,"value":558},"i",{"type":150,"tag":221,"props":560,"children":561},{},[562],{"type":155,"value":563},"∈",{"type":150,"tag":565,"props":566,"children":567},"msup",{},[568,574],{"type":150,"tag":209,"props":569,"children":571},{"mathvariant":570},"double-struck",[572],{"type":155,"value":573},"R",{"type":150,"tag":209,"props":575,"children":576},{},[577],{"type":155,"value":578},"d",{"type":150,"tag":271,"props":580,"children":581},{"encoding":273},[582],{"type":155,"value":583},"x_i \\in \\mathbb{R}^d",{"type":150,"tag":181,"props":585,"children":587},{"className":586,"ariaHidden":223},[280],[588,670],{"type":150,"tag":181,"props":589,"children":591},{"className":590},[285],[592,597,655,660,666],{"type":150,"tag":181,"props":593,"children":596},{"className":594,"style":595},[290],"height:0.6891em;vertical-align:-0.15em;",[],{"type":150,"tag":181,"props":598,"children":600},{"className":599},[296],[601,606],{"type":150,"tag":181,"props":602,"children":604},{"className":603},[296,301],[605],{"type":155,"value":213},{"type":150,"tag":181,"props":607,"children":609},{"className":608},[307],[610],{"type":150,"tag":181,"props":611,"children":613},{"className":612},[312,313],[614,644],{"type":150,"tag":181,"props":615,"children":617},{"className":616},[318],[618,639],{"type":150,"tag":181,"props":619,"children":622},{"className":620,"style":621},[323],"height:0.3117em;",[623],{"type":150,"tag":181,"props":624,"children":625},{"style":328},[626,630],{"type":150,"tag":181,"props":627,"children":629},{"className":628,"style":334},[333],[],{"type":150,"tag":181,"props":631,"children":633},{"className":632},[339,340,341,342],[634],{"type":150,"tag":181,"props":635,"children":637},{"className":636},[296,301,342],[638],{"type":155,"value":558},{"type":150,"tag":181,"props":640,"children":642},{"className":641},[352],[643],{"type":155,"value":355},{"type":150,"tag":181,"props":645,"children":647},{"className":646},[318],[648],{"type":150,"tag":181,"props":649,"children":651},{"className":650,"style":363},[323],[652],{"type":150,"tag":181,"props":653,"children":654},{},[],{"type":150,"tag":181,"props":656,"children":659},{"className":657,"style":658},[377],"margin-right:0.2778em;",[],{"type":150,"tag":181,"props":661,"children":664},{"className":662},[663],"mrel",[665],{"type":155,"value":563},{"type":150,"tag":181,"props":667,"children":669},{"className":668,"style":658},[377],[],{"type":150,"tag":181,"props":671,"children":673},{"className":672},[285],[674,679],{"type":150,"tag":181,"props":675,"children":678},{"className":676,"style":677},[290],"height:0.8491em;",[],{"type":150,"tag":181,"props":680,"children":682},{"className":681},[296],[683,689],{"type":150,"tag":181,"props":684,"children":687},{"className":685},[296,686],"mathbb",[688],{"type":155,"value":573},{"type":150,"tag":181,"props":690,"children":692},{"className":691},[307],[693],{"type":150,"tag":181,"props":694,"children":696},{"className":695},[312],[697],{"type":150,"tag":181,"props":698,"children":700},{"className":699},[318],[701],{"type":150,"tag":181,"props":702,"children":704},{"className":703,"style":677},[323],[705],{"type":150,"tag":181,"props":706,"children":708},{"style":707},"top:-3.063em;margin-right:0.05em;",[709,713],{"type":150,"tag":181,"props":710,"children":712},{"className":711,"style":334},[333],[],{"type":150,"tag":181,"props":714,"children":716},{"className":715},[339,340,341,342],[717],{"type":150,"tag":181,"props":718,"children":720},{"className":719},[296,301,342],[721],{"type":155,"value":578},{"type":155,"value":723},"。",{"type":150,"tag":181,"props":725,"children":727},{"className":726},[184],[728,750],{"type":150,"tag":181,"props":729,"children":731},{"className":730},[189],[732],{"type":150,"tag":192,"props":733,"children":734},{"xmlns":194},[735],{"type":150,"tag":197,"props":736,"children":737},{},[738,746],{"type":150,"tag":201,"props":739,"children":740},{},[741],{"type":150,"tag":209,"props":742,"children":743},{},[744],{"type":155,"value":745},"h",{"type":150,"tag":271,"props":747,"children":748},{"encoding":273},[749],{"type":155,"value":745},{"type":150,"tag":181,"props":751,"children":753},{"className":752,"ariaHidden":223},[280],[754],{"type":150,"tag":181,"props":755,"children":757},{"className":756},[285],[758,763],{"type":150,"tag":181,"props":759,"children":762},{"className":760,"style":761},[290],"height:0.6944em;",[],{"type":150,"tag":181,"props":764,"children":766},{"className":765},[296,301],[767],{"type":155,"value":745},{"type":155,"value":769}," 是 head 数，",{"type":150,"tag":181,"props":771,"children":773},{"className":772},[184],[774,808],{"type":150,"tag":181,"props":775,"children":777},{"className":776},[189],[778],{"type":150,"tag":192,"props":779,"children":780},{"xmlns":194},[781],{"type":150,"tag":197,"props":782,"children":783},{},[784,803],{"type":150,"tag":201,"props":785,"children":786},{},[787,793,798],{"type":150,"tag":221,"props":788,"children":790},{"stretchy":789},"false",[791],{"type":155,"value":792},"(",{"type":150,"tag":209,"props":794,"children":795},{},[796],{"type":155,"value":797},"s",{"type":150,"tag":221,"props":799,"children":800},{"stretchy":789},[801],{"type":155,"value":802},")",{"type":150,"tag":271,"props":804,"children":805},{"encoding":273},[806],{"type":155,"value":807},"(s)",{"type":150,"tag":181,"props":809,"children":811},{"className":810,"ariaHidden":223},[280],[812],{"type":150,"tag":181,"props":813,"children":815},{"className":814},[285],[816,821,827,832],{"type":150,"tag":181,"props":817,"children":820},{"className":818,"style":819},[290],"height:1em;vertical-align:-0.25em;",[],{"type":150,"tag":181,"props":822,"children":825},{"className":823},[824],"mopen",[826],{"type":155,"value":792},{"type":150,"tag":181,"props":828,"children":830},{"className":829},[296,301],[831],{"type":155,"value":797},{"type":150,"tag":181,"props":833,"children":836},{"className":834},[835],"mclose",[837],{"type":155,"value":802},{"type":155,"value":839}," 是 head 的下标，",{"type":150,"tag":181,"props":841,"children":843},{"className":842},[184],[844,903],{"type":150,"tag":181,"props":845,"children":847},{"className":846},[189],[848],{"type":150,"tag":192,"props":849,"children":850},{"xmlns":194},[851],{"type":150,"tag":197,"props":852,"children":853},{},[854,898],{"type":150,"tag":201,"props":855,"children":856},{},[857,861,866,870,874,878,882,886,890,894],{"type":150,"tag":209,"props":858,"children":859},{},[860],{"type":155,"value":797},{"type":150,"tag":221,"props":862,"children":863},{},[864],{"type":155,"value":865},"=",{"type":150,"tag":215,"props":867,"children":868},{},[869],{"type":155,"value":219},{"type":150,"tag":221,"props":871,"children":872},{"separator":223},[873],{"type":155,"value":226},{"type":150,"tag":215,"props":875,"children":876},{},[877],{"type":155,"value":238},{"type":150,"tag":221,"props":879,"children":880},{"separator":223},[881],{"type":155,"value":226},{"type":150,"tag":221,"props":883,"children":884},{},[885],{"type":155,"value":247},{"type":150,"tag":249,"props":887,"children":888},{},[889],{"type":155,"value":253},{"type":150,"tag":221,"props":891,"children":892},{"separator":223},[893],{"type":155,"value":226},{"type":150,"tag":209,"props":895,"children":896},{},[897],{"type":155,"value":745},{"type":150,"tag":271,"props":899,"children":900},{"encoding":273},[901],{"type":155,"value":902},"s = 1, 2, \\cdots, h",{"type":150,"tag":181,"props":904,"children":906},{"className":905,"ariaHidden":223},[280],[907,934],{"type":150,"tag":181,"props":908,"children":910},{"className":909},[285],[911,916,921,925,930],{"type":150,"tag":181,"props":912,"children":915},{"className":913,"style":914},[290],"height:0.4306em;",[],{"type":150,"tag":181,"props":917,"children":919},{"className":918},[296,301],[920],{"type":155,"value":797},{"type":150,"tag":181,"props":922,"children":924},{"className":923,"style":658},[377],[],{"type":150,"tag":181,"props":926,"children":928},{"className":927},[663],[929],{"type":155,"value":865},{"type":150,"tag":181,"props":931,"children":933},{"className":932,"style":658},[377],[],{"type":150,"tag":181,"props":935,"children":937},{"className":936},[285],[938,943,948,953,957,962,967,971,976,980,984,989,993],{"type":150,"tag":181,"props":939,"children":942},{"className":940,"style":941},[290],"height:0.8889em;vertical-align:-0.1944em;",[],{"type":150,"tag":181,"props":944,"children":946},{"className":945},[296],[947],{"type":155,"value":219},{"type":150,"tag":181,"props":949,"children":951},{"className":950},[371],[952],{"type":155,"value":226},{"type":150,"tag":181,"props":954,"children":956},{"className":955,"style":378},[377],[],{"type":150,"tag":181,"props":958,"children":960},{"className":959},[296],[961],{"type":155,"value":238},{"type":150,"tag":181,"props":963,"children":965},{"className":964},[371],[966],{"type":155,"value":226},{"type":150,"tag":181,"props":968,"children":970},{"className":969,"style":378},[377],[],{"type":150,"tag":181,"props":972,"children":974},{"className":973},[449],[975],{"type":155,"value":247},{"type":150,"tag":181,"props":977,"children":979},{"className":978,"style":378},[377],[],{"type":150,"tag":181,"props":981,"children":983},{"className":982,"style":378},[377],[],{"type":150,"tag":181,"props":985,"children":987},{"className":986},[371],[988],{"type":155,"value":226},{"type":150,"tag":181,"props":990,"children":992},{"className":991,"style":378},[377],[],{"type":150,"tag":181,"props":994,"children":996},{"className":995},[296,301],[997],{"type":155,"value":745},{"type":155,"value":999},"。那么：",{"type":150,"tag":181,"props":1001,"children":1004},{"className":1002},[1003],"katex-display",[1005],{"type":150,"tag":181,"props":1006,"children":1008},{"className":1007},[184],[1009,1154],{"type":150,"tag":181,"props":1010,"children":1012},{"className":1011},[189],[1013],{"type":150,"tag":192,"props":1014,"children":1016},{"xmlns":194,"display":1015},"block",[1017],{"type":150,"tag":197,"props":1018,"children":1019},{},[1020,1149],{"type":150,"tag":201,"props":1021,"children":1022},{},[1023,1036,1040,1045,1072,1076,1102,1106,1110,1114,1118,1144],{"type":150,"tag":205,"props":1024,"children":1025},{},[1026,1031],{"type":150,"tag":209,"props":1027,"children":1028},{},[1029],{"type":155,"value":1030},"o",{"type":150,"tag":209,"props":1032,"children":1033},{},[1034],{"type":155,"value":1035},"t",{"type":150,"tag":221,"props":1037,"children":1038},{},[1039],{"type":155,"value":865},{"type":150,"tag":221,"props":1041,"children":1042},{"stretchy":789},[1043],{"type":155,"value":1044},"[",{"type":150,"tag":1046,"props":1047,"children":1048},"msubsup",{},[1049,1053,1057],{"type":150,"tag":209,"props":1050,"children":1051},{},[1052],{"type":155,"value":1030},{"type":150,"tag":209,"props":1054,"children":1055},{},[1056],{"type":155,"value":1035},{"type":150,"tag":201,"props":1058,"children":1059},{},[1060,1064,1068],{"type":150,"tag":221,"props":1061,"children":1062},{"stretchy":789},[1063],{"type":155,"value":792},{"type":150,"tag":215,"props":1065,"children":1066},{},[1067],{"type":155,"value":219},{"type":150,"tag":221,"props":1069,"children":1070},{"stretchy":789},[1071],{"type":155,"value":802},{"type":150,"tag":221,"props":1073,"children":1074},{"separator":223},[1075],{"type":155,"value":226},{"type":150,"tag":1046,"props":1077,"children":1078},{},[1079,1083,1087],{"type":150,"tag":209,"props":1080,"children":1081},{},[1082],{"type":155,"value":1030},{"type":150,"tag":209,"props":1084,"children":1085},{},[1086],{"type":155,"value":1035},{"type":150,"tag":201,"props":1088,"children":1089},{},[1090,1094,1098],{"type":150,"tag":221,"props":1091,"children":1092},{"stretchy":789},[1093],{"type":155,"value":792},{"type":150,"tag":215,"props":1095,"children":1096},{},[1097],{"type":155,"value":238},{"type":150,"tag":221,"props":1099,"children":1100},{"stretchy":789},[1101],{"type":155,"value":802},{"type":150,"tag":221,"props":1103,"children":1104},{"separator":223},[1105],{"type":155,"value":226},{"type":150,"tag":221,"props":1107,"children":1108},{},[1109],{"type":155,"value":247},{"type":150,"tag":249,"props":1111,"children":1112},{},[1113],{"type":155,"value":253},{"type":150,"tag":221,"props":1115,"children":1116},{"separator":223},[1117],{"type":155,"value":226},{"type":150,"tag":1046,"props":1119,"children":1120},{},[1121,1125,1129],{"type":150,"tag":209,"props":1122,"children":1123},{},[1124],{"type":155,"value":1030},{"type":150,"tag":209,"props":1126,"children":1127},{},[1128],{"type":155,"value":1035},{"type":150,"tag":201,"props":1130,"children":1131},{},[1132,1136,1140],{"type":150,"tag":221,"props":1133,"children":1134},{"stretchy":789},[1135],{"type":155,"value":792},{"type":150,"tag":209,"props":1137,"children":1138},{},[1139],{"type":155,"value":745},{"type":150,"tag":221,"props":1141,"children":1142},{"stretchy":789},[1143],{"type":155,"value":802},{"type":150,"tag":221,"props":1145,"children":1146},{"stretchy":789},[1147],{"type":155,"value":1148},"]",{"type":150,"tag":271,"props":1150,"children":1151},{"encoding":273},[1152],{"type":155,"value":1153},"o_t = [o_t^{(1)}, o_t^{(2)}, \\cdots, o_t^{(h)}]",{"type":150,"tag":181,"props":1155,"children":1157},{"className":1156,"ariaHidden":223},[280],[1158,1238],{"type":150,"tag":181,"props":1159,"children":1161},{"className":1160},[285],[1162,1167,1225,1229,1234],{"type":150,"tag":181,"props":1163,"children":1166},{"className":1164,"style":1165},[290],"height:0.5806em;vertical-align:-0.15em;",[],{"type":150,"tag":181,"props":1168,"children":1170},{"className":1169},[296],[1171,1176],{"type":150,"tag":181,"props":1172,"children":1174},{"className":1173},[296,301],[1175],{"type":155,"value":1030},{"type":150,"tag":181,"props":1177,"children":1179},{"className":1178},[307],[1180],{"type":150,"tag":181,"props":1181,"children":1183},{"className":1182},[312,313],[1184,1214],{"type":150,"tag":181,"props":1185,"children":1187},{"className":1186},[318],[1188,1209],{"type":150,"tag":181,"props":1189,"children":1192},{"className":1190,"style":1191},[323],"height:0.2806em;",[1193],{"type":150,"tag":181,"props":1194,"children":1195},{"style":328},[1196,1200],{"type":150,"tag":181,"props":1197,"children":1199},{"className":1198,"style":334},[333],[],{"type":150,"tag":181,"props":1201,"children":1203},{"className":1202},[339,340,341,342],[1204],{"type":150,"tag":181,"props":1205,"children":1207},{"className":1206},[296,301,342],[1208],{"type":155,"value":1035},{"type":150,"tag":181,"props":1210,"children":1212},{"className":1211},[352],[1213],{"type":155,"value":355},{"type":150,"tag":181,"props":1215,"children":1217},{"className":1216},[318],[1218],{"type":150,"tag":181,"props":1219,"children":1221},{"className":1220,"style":363},[323],[1222],{"type":150,"tag":181,"props":1223,"children":1224},{},[],{"type":150,"tag":181,"props":1226,"children":1228},{"className":1227,"style":658},[377],[],{"type":150,"tag":181,"props":1230,"children":1232},{"className":1231},[663],[1233],{"type":155,"value":865},{"type":150,"tag":181,"props":1235,"children":1237},{"className":1236,"style":658},[377],[],{"type":150,"tag":181,"props":1239,"children":1241},{"className":1240},[285],[1242,1247,1252,1343,1348,1352,1439,1444,1448,1453,1457,1461,1466,1470,1557],{"type":150,"tag":181,"props":1243,"children":1246},{"className":1244,"style":1245},[290],"height:1.2948em;vertical-align:-0.25em;",[],{"type":150,"tag":181,"props":1248,"children":1250},{"className":1249},[824],[1251],{"type":155,"value":1044},{"type":150,"tag":181,"props":1253,"children":1255},{"className":1254},[296],[1256,1261],{"type":150,"tag":181,"props":1257,"children":1259},{"className":1258},[296,301],[1260],{"type":155,"value":1030},{"type":150,"tag":181,"props":1262,"children":1264},{"className":1263},[307],[1265],{"type":150,"tag":181,"props":1266,"children":1268},{"className":1267},[312,313],[1269,1331],{"type":150,"tag":181,"props":1270,"children":1272},{"className":1271},[318],[1273,1326],{"type":150,"tag":181,"props":1274,"children":1277},{"className":1275,"style":1276},[323],"height:1.0448em;",[1278,1295],{"type":150,"tag":181,"props":1279,"children":1281},{"style":1280},"top:-2.4542em;margin-left:0em;margin-right:0.05em;",[1282,1286],{"type":150,"tag":181,"props":1283,"children":1285},{"className":1284,"style":334},[333],[],{"type":150,"tag":181,"props":1287,"children":1289},{"className":1288},[339,340,341,342],[1290],{"type":150,"tag":181,"props":1291,"children":1293},{"className":1292},[296,301,342],[1294],{"type":155,"value":1035},{"type":150,"tag":181,"props":1296,"children":1298},{"style":1297},"top:-3.2198em;margin-right:0.05em;",[1299,1303],{"type":150,"tag":181,"props":1300,"children":1302},{"className":1301,"style":334},[333],[],{"type":150,"tag":181,"props":1304,"children":1306},{"className":1305},[339,340,341,342],[1307],{"type":150,"tag":181,"props":1308,"children":1310},{"className":1309},[296,342],[1311,1316,1321],{"type":150,"tag":181,"props":1312,"children":1314},{"className":1313},[824,342],[1315],{"type":155,"value":792},{"type":150,"tag":181,"props":1317,"children":1319},{"className":1318},[296,342],[1320],{"type":155,"value":219},{"type":150,"tag":181,"props":1322,"children":1324},{"className":1323},[835,342],[1325],{"type":155,"value":802},{"type":150,"tag":181,"props":1327,"children":1329},{"className":1328},[352],[1330],{"type":155,"value":355},{"type":150,"tag":181,"props":1332,"children":1334},{"className":1333},[318],[1335],{"type":150,"tag":181,"props":1336,"children":1339},{"className":1337,"style":1338},[323],"height:0.2458em;",[1340],{"type":150,"tag":181,"props":1341,"children":1342},{},[],{"type":150,"tag":181,"props":1344,"children":1346},{"className":1345},[371],[1347],{"type":155,"value":226},{"type":150,"tag":181,"props":1349,"children":1351},{"className":1350,"style":378},[377],[],{"type":150,"tag":181,"props":1353,"children":1355},{"className":1354},[296],[1356,1361],{"type":150,"tag":181,"props":1357,"children":1359},{"className":1358},[296,301],[1360],{"type":155,"value":1030},{"type":150,"tag":181,"props":1362,"children":1364},{"className":1363},[307],[1365],{"type":150,"tag":181,"props":1366,"children":1368},{"className":1367},[312,313],[1369,1428],{"type":150,"tag":181,"props":1370,"children":1372},{"className":1371},[318],[1373,1423],{"type":150,"tag":181,"props":1374,"children":1376},{"className":1375,"style":1276},[323],[1377,1393],{"type":150,"tag":181,"props":1378,"children":1379},{"style":1280},[1380,1384],{"type":150,"tag":181,"props":1381,"children":1383},{"className":1382,"style":334},[333],[],{"type":150,"tag":181,"props":1385,"children":1387},{"className":1386},[339,340,341,342],[1388],{"type":150,"tag":181,"props":1389,"children":1391},{"className":1390},[296,301,342],[1392],{"type":155,"value":1035},{"type":150,"tag":181,"props":1394,"children":1395},{"style":1297},[1396,1400],{"type":150,"tag":181,"props":1397,"children":1399},{"className":1398,"style":334},[333],[],{"type":150,"tag":181,"props":1401,"children":1403},{"className":1402},[339,340,341,342],[1404],{"type":150,"tag":181,"props":1405,"children":1407},{"className":1406},[296,342],[1408,1413,1418],{"type":150,"tag":181,"props":1409,"children":1411},{"className":1410},[824,342],[1412],{"type":155,"value":792},{"type":150,"tag":181,"props":1414,"children":1416},{"className":1415},[296,342],[1417],{"type":155,"value":238},{"type":150,"tag":181,"props":1419,"children":1421},{"className":1420},[835,342],[1422],{"type":155,"value":802},{"type":150,"tag":181,"props":1424,"children":1426},{"className":1425},[352],[1427],{"type":155,"value":355},{"type":150,"tag":181,"props":1429,"children":1431},{"className":1430},[318],[1432],{"type":150,"tag":181,"props":1433,"children":1435},{"className":1434,"style":1338},[323],[1436],{"type":150,"tag":181,"props":1437,"children":1438},{},[],{"type":150,"tag":181,"props":1440,"children":1442},{"className":1441},[371],[1443],{"type":155,"value":226},{"type":150,"tag":181,"props":1445,"children":1447},{"className":1446,"style":378},[377],[],{"type":150,"tag":181,"props":1449,"children":1451},{"className":1450},[449],[1452],{"type":155,"value":247},{"type":150,"tag":181,"props":1454,"children":1456},{"className":1455,"style":378},[377],[],{"type":150,"tag":181,"props":1458,"children":1460},{"className":1459,"style":378},[377],[],{"type":150,"tag":181,"props":1462,"children":1464},{"className":1463},[371],[1465],{"type":155,"value":226},{"type":150,"tag":181,"props":1467,"children":1469},{"className":1468,"style":378},[377],[],{"type":150,"tag":181,"props":1471,"children":1473},{"className":1472},[296],[1474,1479],{"type":150,"tag":181,"props":1475,"children":1477},{"className":1476},[296,301],[1478],{"type":155,"value":1030},{"type":150,"tag":181,"props":1480,"children":1482},{"className":1481},[307],[1483],{"type":150,"tag":181,"props":1484,"children":1486},{"className":1485},[312,313],[1487,1546],{"type":150,"tag":181,"props":1488,"children":1490},{"className":1489},[318],[1491,1541],{"type":150,"tag":181,"props":1492,"children":1494},{"className":1493,"style":1276},[323],[1495,1511],{"type":150,"tag":181,"props":1496,"children":1497},{"style":1280},[1498,1502],{"type":150,"tag":181,"props":1499,"children":1501},{"className":1500,"style":334},[333],[],{"type":150,"tag":181,"props":1503,"children":1505},{"className":1504},[339,340,341,342],[1506],{"type":150,"tag":181,"props":1507,"children":1509},{"className":1508},[296,301,342],[1510],{"type":155,"value":1035},{"type":150,"tag":181,"props":1512,"children":1513},{"style":1297},[1514,1518],{"type":150,"tag":181,"props":1515,"children":1517},{"className":1516,"style":334},[333],[],{"type":150,"tag":181,"props":1519,"children":1521},{"className":1520},[339,340,341,342],[1522],{"type":150,"tag":181,"props":1523,"children":1525},{"className":1524},[296,342],[1526,1531,1536],{"type":150,"tag":181,"props":1527,"children":1529},{"className":1528},[824,342],[1530],{"type":155,"value":792},{"type":150,"tag":181,"props":1532,"children":1534},{"className":1533},[296,301,342],[1535],{"type":155,"value":745},{"type":150,"tag":181,"props":1537,"children":1539},{"className":1538},[835,342],[1540],{"type":155,"value":802},{"type":150,"tag":181,"props":1542,"children":1544},{"className":1543},[352],[1545],{"type":155,"value":355},{"type":150,"tag":181,"props":1547,"children":1549},{"className":1548},[318],[1550],{"type":150,"tag":181,"props":1551,"children":1553},{"className":1552,"style":1338},[323],[1554],{"type":150,"tag":181,"props":1555,"children":1556},{},[],{"type":150,"tag":181,"props":1558,"children":1560},{"className":1559},[835],[1561],{"type":155,"value":1148},{"type":150,"tag":181,"props":1563,"children":1565},{"className":1564},[1003],[1566],{"type":150,"tag":181,"props":1567,"children":1569},{"className":1568},[184],[1570,2007],{"type":150,"tag":181,"props":1571,"children":1573},{"className":1572},[189],[1574],{"type":150,"tag":192,"props":1575,"children":1576},{"xmlns":194,"display":1015},[1577],{"type":150,"tag":197,"props":1578,"children":1579},{},[1580,2002],{"type":150,"tag":201,"props":1581,"children":1582},{},[1583,1609,1613,1656,1660,1687,1691,1726,1730,1764,1768,1773],{"type":150,"tag":1046,"props":1584,"children":1585},{},[1586,1590,1594],{"type":150,"tag":209,"props":1587,"children":1588},{},[1589],{"type":155,"value":1030},{"type":150,"tag":209,"props":1591,"children":1592},{},[1593],{"type":155,"value":1035},{"type":150,"tag":201,"props":1595,"children":1596},{},[1597,1601,1605],{"type":150,"tag":221,"props":1598,"children":1599},{"stretchy":789},[1600],{"type":155,"value":792},{"type":150,"tag":209,"props":1602,"children":1603},{},[1604],{"type":155,"value":797},{"type":150,"tag":221,"props":1606,"children":1607},{"stretchy":789},[1608],{"type":155,"value":802},{"type":150,"tag":221,"props":1610,"children":1611},{},[1612],{"type":155,"value":865},{"type":150,"tag":201,"props":1614,"children":1615},{},[1616,1622,1626,1630,1635,1640,1644,1648,1652],{"type":150,"tag":209,"props":1617,"children":1619},{"mathvariant":1618},"normal",[1620],{"type":155,"value":1621},"A",{"type":150,"tag":209,"props":1623,"children":1624},{"mathvariant":1618},[1625],{"type":155,"value":1035},{"type":150,"tag":209,"props":1627,"children":1628},{"mathvariant":1618},[1629],{"type":155,"value":1035},{"type":150,"tag":209,"props":1631,"children":1632},{"mathvariant":1618},[1633],{"type":155,"value":1634},"e",{"type":150,"tag":209,"props":1636,"children":1637},{"mathvariant":1618},[1638],{"type":155,"value":1639},"n",{"type":150,"tag":209,"props":1641,"children":1642},{"mathvariant":1618},[1643],{"type":155,"value":1035},{"type":150,"tag":209,"props":1645,"children":1646},{"mathvariant":1618},[1647],{"type":155,"value":558},{"type":150,"tag":209,"props":1649,"children":1650},{"mathvariant":1618},[1651],{"type":155,"value":1030},{"type":150,"tag":209,"props":1653,"children":1654},{"mathvariant":1618},[1655],{"type":155,"value":1639},{"type":150,"tag":221,"props":1657,"children":1658},{"stretchy":789},[1659],{"type":155,"value":792},{"type":150,"tag":1046,"props":1661,"children":1662},{},[1663,1668,1672],{"type":150,"tag":209,"props":1664,"children":1665},{},[1666],{"type":155,"value":1667},"q",{"type":150,"tag":209,"props":1669,"children":1670},{},[1671],{"type":155,"value":1035},{"type":150,"tag":201,"props":1673,"children":1674},{},[1675,1679,1683],{"type":150,"tag":221,"props":1676,"children":1677},{"stretchy":789},[1678],{"type":155,"value":792},{"type":150,"tag":209,"props":1680,"children":1681},{},[1682],{"type":155,"value":797},{"type":150,"tag":221,"props":1684,"children":1685},{"stretchy":789},[1686],{"type":155,"value":802},{"type":150,"tag":221,"props":1688,"children":1689},{"separator":223},[1690],{"type":155,"value":226},{"type":150,"tag":1046,"props":1692,"children":1693},{},[1694,1699,1711],{"type":150,"tag":209,"props":1695,"children":1696},{},[1697],{"type":155,"value":1698},"k",{"type":150,"tag":201,"props":1700,"children":1701},{},[1702,1707],{"type":150,"tag":221,"props":1703,"children":1704},{},[1705],{"type":155,"value":1706},"≤",{"type":150,"tag":209,"props":1708,"children":1709},{},[1710],{"type":155,"value":1035},{"type":150,"tag":201,"props":1712,"children":1713},{},[1714,1718,1722],{"type":150,"tag":221,"props":1715,"children":1716},{"stretchy":789},[1717],{"type":155,"value":792},{"type":150,"tag":209,"props":1719,"children":1720},{},[1721],{"type":155,"value":797},{"type":150,"tag":221,"props":1723,"children":1724},{"stretchy":789},[1725],{"type":155,"value":802},{"type":150,"tag":221,"props":1727,"children":1728},{"separator":223},[1729],{"type":155,"value":226},{"type":150,"tag":1046,"props":1731,"children":1732},{},[1733,1738,1749],{"type":150,"tag":209,"props":1734,"children":1735},{},[1736],{"type":155,"value":1737},"v",{"type":150,"tag":201,"props":1739,"children":1740},{},[1741,1745],{"type":150,"tag":221,"props":1742,"children":1743},{},[1744],{"type":155,"value":1706},{"type":150,"tag":209,"props":1746,"children":1747},{},[1748],{"type":155,"value":1035},{"type":150,"tag":201,"props":1750,"children":1751},{},[1752,1756,1760],{"type":150,"tag":221,"props":1753,"children":1754},{"stretchy":789},[1755],{"type":155,"value":792},{"type":150,"tag":209,"props":1757,"children":1758},{},[1759],{"type":155,"value":797},{"type":150,"tag":221,"props":1761,"children":1762},{"stretchy":789},[1763],{"type":155,"value":802},{"type":150,"tag":221,"props":1765,"children":1766},{"stretchy":789},[1767],{"type":155,"value":802},{"type":150,"tag":221,"props":1769,"children":1770},{},[1771],{"type":155,"value":1772},"≜",{"type":150,"tag":1774,"props":1775,"children":1776},"mfrac",{},[1777,1905],{"type":150,"tag":201,"props":1778,"children":1779},{},[1780,1804,1809,1814,1818,1844,1875,1879],{"type":150,"tag":1781,"props":1782,"children":1783},"munder",{},[1784,1789],{"type":150,"tag":221,"props":1785,"children":1786},{},[1787],{"type":155,"value":1788},"∑",{"type":150,"tag":201,"props":1790,"children":1791},{},[1792,1796,1800],{"type":150,"tag":209,"props":1793,"children":1794},{},[1795],{"type":155,"value":558},{"type":150,"tag":221,"props":1797,"children":1798},{},[1799],{"type":155,"value":1706},{"type":150,"tag":209,"props":1801,"children":1802},{},[1803],{"type":155,"value":1035},{"type":150,"tag":209,"props":1805,"children":1806},{},[1807],{"type":155,"value":1808},"exp",{"type":150,"tag":221,"props":1810,"children":1811},{},[1812],{"type":155,"value":1813},"⁡",{"type":150,"tag":221,"props":1815,"children":1816},{"stretchy":789},[1817],{"type":155,"value":792},{"type":150,"tag":1046,"props":1819,"children":1820},{},[1821,1825,1829],{"type":150,"tag":209,"props":1822,"children":1823},{},[1824],{"type":155,"value":1667},{"type":150,"tag":209,"props":1826,"children":1827},{},[1828],{"type":155,"value":1035},{"type":150,"tag":201,"props":1830,"children":1831},{},[1832,1836,1840],{"type":150,"tag":221,"props":1833,"children":1834},{"stretchy":789},[1835],{"type":155,"value":792},{"type":150,"tag":209,"props":1837,"children":1838},{},[1839],{"type":155,"value":797},{"type":150,"tag":221,"props":1841,"children":1842},{"stretchy":789},[1843],{"type":155,"value":802},{"type":150,"tag":1046,"props":1845,"children":1846},{},[1847,1851,1855],{"type":150,"tag":209,"props":1848,"children":1849},{},[1850],{"type":155,"value":1698},{"type":150,"tag":209,"props":1852,"children":1853},{},[1854],{"type":155,"value":558},{"type":150,"tag":201,"props":1856,"children":1857},{},[1858,1862,1866,1870],{"type":150,"tag":221,"props":1859,"children":1860},{"stretchy":789},[1861],{"type":155,"value":792},{"type":150,"tag":209,"props":1863,"children":1864},{},[1865],{"type":155,"value":797},{"type":150,"tag":221,"props":1867,"children":1868},{"stretchy":789},[1869],{"type":155,"value":802},{"type":150,"tag":209,"props":1871,"children":1872},{"mathvariant":1618},[1873],{"type":155,"value":1874},"⊤",{"type":150,"tag":221,"props":1876,"children":1877},{"stretchy":789},[1878],{"type":155,"value":802},{"type":150,"tag":1046,"props":1880,"children":1881},{},[1882,1886,1890],{"type":150,"tag":209,"props":1883,"children":1884},{},[1885],{"type":155,"value":1737},{"type":150,"tag":209,"props":1887,"children":1888},{},[1889],{"type":155,"value":558},{"type":150,"tag":201,"props":1891,"children":1892},{},[1893,1897,1901],{"type":150,"tag":221,"props":1894,"children":1895},{"stretchy":789},[1896],{"type":155,"value":792},{"type":150,"tag":209,"props":1898,"children":1899},{},[1900],{"type":155,"value":797},{"type":150,"tag":221,"props":1902,"children":1903},{"stretchy":789},[1904],{"type":155,"value":802},{"type":150,"tag":201,"props":1906,"children":1907},{},[1908,1930,1934,1938,1942,1968,1998],{"type":150,"tag":1781,"props":1909,"children":1910},{},[1911,1915],{"type":150,"tag":221,"props":1912,"children":1913},{},[1914],{"type":155,"value":1788},{"type":150,"tag":201,"props":1916,"children":1917},{},[1918,1922,1926],{"type":150,"tag":209,"props":1919,"children":1920},{},[1921],{"type":155,"value":558},{"type":150,"tag":221,"props":1923,"children":1924},{},[1925],{"type":155,"value":1706},{"type":150,"tag":209,"props":1927,"children":1928},{},[1929],{"type":155,"value":1035},{"type":150,"tag":209,"props":1931,"children":1932},{},[1933],{"type":155,"value":1808},{"type":150,"tag":221,"props":1935,"children":1936},{},[1937],{"type":155,"value":1813},{"type":150,"tag":221,"props":1939,"children":1940},{"stretchy":789},[1941],{"type":155,"value":792},{"type":150,"tag":1046,"props":1943,"children":1944},{},[1945,1949,1953],{"type":150,"tag":209,"props":1946,"children":1947},{},[1948],{"type":155,"value":1667},{"type":150,"tag":209,"props":1950,"children":1951},{},[1952],{"type":155,"value":1035},{"type":150,"tag":201,"props":1954,"children":1955},{},[1956,1960,1964],{"type":150,"tag":221,"props":1957,"children":1958},{"stretchy":789},[1959],{"type":155,"value":792},{"type":150,"tag":209,"props":1961,"children":1962},{},[1963],{"type":155,"value":797},{"type":150,"tag":221,"props":1965,"children":1966},{"stretchy":789},[1967],{"type":155,"value":802},{"type":150,"tag":1046,"props":1969,"children":1970},{},[1971,1975,1979],{"type":150,"tag":209,"props":1972,"children":1973},{},[1974],{"type":155,"value":1698},{"type":150,"tag":209,"props":1976,"children":1977},{},[1978],{"type":155,"value":558},{"type":150,"tag":201,"props":1980,"children":1981},{},[1982,1986,1990,1994],{"type":150,"tag":221,"props":1983,"children":1984},{"stretchy":789},[1985],{"type":155,"value":792},{"type":150,"tag":209,"props":1987,"children":1988},{},[1989],{"type":155,"value":797},{"type":150,"tag":221,"props":1991,"children":1992},{"stretchy":789},[1993],{"type":155,"value":802},{"type":150,"tag":209,"props":1995,"children":1996},{"mathvariant":1618},[1997],{"type":155,"value":1874},{"type":150,"tag":221,"props":1999,"children":2000},{"stretchy":789},[2001],{"type":155,"value":802},{"type":150,"tag":271,"props":2003,"children":2004},{"encoding":273},[2005],{"type":155,"value":2006},"o_t^{(s)} = \\mathrm{Attention}(q_t^{(s)}, k^{(s)}_{\\le t}, v^{(s)}_{\\le t}) \\triangleq \\frac{\\sum_{i \\le t} \\exp(q_t^{(s)} k_i^{(s)\\top}) v_i^{(s)}}{\\sum_{i \\le t} \\exp(q_t^{(s)} k_i^{(s)\\top})}",{"type":150,"tag":181,"props":2008,"children":2010},{"className":2009,"ariaHidden":223},[280],[2011,2120,2466],{"type":150,"tag":181,"props":2012,"children":2014},{"className":2013},[285],[2015,2020,2107,2111,2116],{"type":150,"tag":181,"props":2016,"children":2019},{"className":2017,"style":2018},[290],"height:1.2906em;vertical-align:-0.2458em;",[],{"type":150,"tag":181,"props":2021,"children":2023},{"className":2022},[296],[2024,2029],{"type":150,"tag":181,"props":2025,"children":2027},{"className":2026},[296,301],[2028],{"type":155,"value":1030},{"type":150,"tag":181,"props":2030,"children":2032},{"className":2031},[307],[2033],{"type":150,"tag":181,"props":2034,"children":2036},{"className":2035},[312,313],[2037,2096],{"type":150,"tag":181,"props":2038,"children":2040},{"className":2039},[318],[2041,2091],{"type":150,"tag":181,"props":2042,"children":2044},{"className":2043,"style":1276},[323],[2045,2061],{"type":150,"tag":181,"props":2046,"children":2047},{"style":1280},[2048,2052],{"type":150,"tag":181,"props":2049,"children":2051},{"className":2050,"style":334},[333],[],{"type":150,"tag":181,"props":2053,"children":2055},{"className":2054},[339,340,341,342],[2056],{"type":150,"tag":181,"props":2057,"children":2059},{"className":2058},[296,301,342],[2060],{"type":155,"value":1035},{"type":150,"tag":181,"props":2062,"children":2063},{"style":1297},[2064,2068],{"type":150,"tag":181,"props":2065,"children":2067},{"className":2066,"style":334},[333],[],{"type":150,"tag":181,"props":2069,"children":2071},{"className":2070},[339,340,341,342],[2072],{"type":150,"tag":181,"props":2073,"children":2075},{"className":2074},[296,342],[2076,2081,2086],{"type":150,"tag":181,"props":2077,"children":2079},{"className":2078},[824,342],[2080],{"type":155,"value":792},{"type":150,"tag":181,"props":2082,"children":2084},{"className":2083},[296,301,342],[2085],{"type":155,"value":797},{"type":150,"tag":181,"props":2087,"children":2089},{"className":2088},[835,342],[2090],{"type":155,"value":802},{"type":150,"tag":181,"props":2092,"children":2094},{"className":2093},[352],[2095],{"type":155,"value":355},{"type":150,"tag":181,"props":2097,"children":2099},{"className":2098},[318],[2100],{"type":150,"tag":181,"props":2101,"children":2103},{"className":2102,"style":1338},[323],[2104],{"type":150,"tag":181,"props":2105,"children":2106},{},[],{"type":150,"tag":181,"props":2108,"children":2110},{"className":2109,"style":658},[377],[],{"type":150,"tag":181,"props":2112,"children":2114},{"className":2113},[663],[2115],{"type":155,"value":865},{"type":150,"tag":181,"props":2117,"children":2119},{"className":2118,"style":658},[377],[],{"type":150,"tag":181,"props":2121,"children":2123},{"className":2122},[285],[2124,2129,2139,2144,2233,2238,2242,2341,2346,2350,2447,2452,2456,2462],{"type":150,"tag":181,"props":2125,"children":2128},{"className":2126,"style":2127},[290],"height:1.4004em;vertical-align:-0.3556em;",[],{"type":150,"tag":181,"props":2130,"children":2132},{"className":2131},[296],[2133],{"type":150,"tag":181,"props":2134,"children":2137},{"className":2135},[296,2136],"mathrm",[2138],{"type":155,"value":12},{"type":150,"tag":181,"props":2140,"children":2142},{"className":2141},[824],[2143],{"type":155,"value":792},{"type":150,"tag":181,"props":2145,"children":2147},{"className":2146},[296],[2148,2154],{"type":150,"tag":181,"props":2149,"children":2152},{"className":2150,"style":2151},[296,301],"margin-right:0.0359em;",[2153],{"type":155,"value":1667},{"type":150,"tag":181,"props":2155,"children":2157},{"className":2156},[307],[2158],{"type":150,"tag":181,"props":2159,"children":2161},{"className":2160},[312,313],[2162,2222],{"type":150,"tag":181,"props":2163,"children":2165},{"className":2164},[318],[2166,2217],{"type":150,"tag":181,"props":2167,"children":2169},{"className":2168,"style":1276},[323],[2170,2187],{"type":150,"tag":181,"props":2171,"children":2173},{"style":2172},"top:-2.4542em;margin-left:-0.0359em;margin-right:0.05em;",[2174,2178],{"type":150,"tag":181,"props":2175,"children":2177},{"className":2176,"style":334},[333],[],{"type":150,"tag":181,"props":2179,"children":2181},{"className":2180},[339,340,341,342],[2182],{"type":150,"tag":181,"props":2183,"children":2185},{"className":2184},[296,301,342],[2186],{"type":155,"value":1035},{"type":150,"tag":181,"props":2188,"children":2189},{"style":1297},[2190,2194],{"type":150,"tag":181,"props":2191,"children":2193},{"className":2192,"style":334},[333],[],{"type":150,"tag":181,"props":2195,"children":2197},{"className":2196},[339,340,341,342],[2198],{"type":150,"tag":181,"props":2199,"children":2201},{"className":2200},[296,342],[2202,2207,2212],{"type":150,"tag":181,"props":2203,"children":2205},{"className":2204},[824,342],[2206],{"type":155,"value":792},{"type":150,"tag":181,"props":2208,"children":2210},{"className":2209},[296,301,342],[2211],{"type":155,"value":797},{"type":150,"tag":181,"props":2213,"children":2215},{"className":2214},[835,342],[2216],{"type":155,"value":802},{"type":150,"tag":181,"props":2218,"children":2220},{"className":2219},[352],[2221],{"type":155,"value":355},{"type":150,"tag":181,"props":2223,"children":2225},{"className":2224},[318],[2226],{"type":150,"tag":181,"props":2227,"children":2229},{"className":2228,"style":1338},[323],[2230],{"type":150,"tag":181,"props":2231,"children":2232},{},[],{"type":150,"tag":181,"props":2234,"children":2236},{"className":2235},[371],[2237],{"type":155,"value":226},{"type":150,"tag":181,"props":2239,"children":2241},{"className":2240,"style":378},[377],[],{"type":150,"tag":181,"props":2243,"children":2245},{"className":2244},[296],[2246,2252],{"type":150,"tag":181,"props":2247,"children":2250},{"className":2248,"style":2249},[296,301],"margin-right:0.0315em;",[2251],{"type":155,"value":1698},{"type":150,"tag":181,"props":2253,"children":2255},{"className":2254},[307],[2256],{"type":150,"tag":181,"props":2257,"children":2259},{"className":2258},[312,313],[2260,2329],{"type":150,"tag":181,"props":2261,"children":2263},{"className":2262},[318],[2264,2324],{"type":150,"tag":181,"props":2265,"children":2267},{"className":2266,"style":1276},[323],[2268,2294],{"type":150,"tag":181,"props":2269,"children":2271},{"style":2270},"top:-2.4396em;margin-left:-0.0315em;margin-right:0.05em;",[2272,2276],{"type":150,"tag":181,"props":2273,"children":2275},{"className":2274,"style":334},[333],[],{"type":150,"tag":181,"props":2277,"children":2279},{"className":2278},[339,340,341,342],[2280],{"type":150,"tag":181,"props":2281,"children":2283},{"className":2282},[296,342],[2284,2289],{"type":150,"tag":181,"props":2285,"children":2287},{"className":2286},[663,342],[2288],{"type":155,"value":1706},{"type":150,"tag":181,"props":2290,"children":2292},{"className":2291},[296,301,342],[2293],{"type":155,"value":1035},{"type":150,"tag":181,"props":2295,"children":2296},{"style":1297},[2297,2301],{"type":150,"tag":181,"props":2298,"children":2300},{"className":2299,"style":334},[333],[],{"type":150,"tag":181,"props":2302,"children":2304},{"className":2303},[339,340,341,342],[2305],{"type":150,"tag":181,"props":2306,"children":2308},{"className":2307},[296,342],[2309,2314,2319],{"type":150,"tag":181,"props":2310,"children":2312},{"className":2311},[824,342],[2313],{"type":155,"value":792},{"type":150,"tag":181,"props":2315,"children":2317},{"className":2316},[296,301,342],[2318],{"type":155,"value":797},{"type":150,"tag":181,"props":2320,"children":2322},{"className":2321},[835,342],[2323],{"type":155,"value":802},{"type":150,"tag":181,"props":2325,"children":2327},{"className":2326},[352],[2328],{"type":155,"value":355},{"type":150,"tag":181,"props":2330,"children":2332},{"className":2331},[318],[2333],{"type":150,"tag":181,"props":2334,"children":2337},{"className":2335,"style":2336},[323],"height:0.3556em;",[2338],{"type":150,"tag":181,"props":2339,"children":2340},{},[],{"type":150,"tag":181,"props":2342,"children":2344},{"className":2343},[371],[2345],{"type":155,"value":226},{"type":150,"tag":181,"props":2347,"children":2349},{"className":2348,"style":378},[377],[],{"type":150,"tag":181,"props":2351,"children":2353},{"className":2352},[296],[2354,2359],{"type":150,"tag":181,"props":2355,"children":2357},{"className":2356,"style":2151},[296,301],[2358],{"type":155,"value":1737},{"type":150,"tag":181,"props":2360,"children":2362},{"className":2361},[307],[2363],{"type":150,"tag":181,"props":2364,"children":2366},{"className":2365},[312,313],[2367,2436],{"type":150,"tag":181,"props":2368,"children":2370},{"className":2369},[318],[2371,2431],{"type":150,"tag":181,"props":2372,"children":2374},{"className":2373,"style":1276},[323],[2375,2401],{"type":150,"tag":181,"props":2376,"children":2378},{"style":2377},"top:-2.4396em;margin-left:-0.0359em;margin-right:0.05em;",[2379,2383],{"type":150,"tag":181,"props":2380,"children":2382},{"className":2381,"style":334},[333],[],{"type":150,"tag":181,"props":2384,"children":2386},{"className":2385},[339,340,341,342],[2387],{"type":150,"tag":181,"props":2388,"children":2390},{"className":2389},[296,342],[2391,2396],{"type":150,"tag":181,"props":2392,"children":2394},{"className":2393},[663,342],[2395],{"type":155,"value":1706},{"type":150,"tag":181,"props":2397,"children":2399},{"className":2398},[296,301,342],[2400],{"type":155,"value":1035},{"type":150,"tag":181,"props":2402,"children":2403},{"style":1297},[2404,2408],{"type":150,"tag":181,"props":2405,"children":2407},{"className":2406,"style":334},[333],[],{"type":150,"tag":181,"props":2409,"children":2411},{"className":2410},[339,340,341,342],[2412],{"type":150,"tag":181,"props":2413,"children":2415},{"className":2414},[296,342],[2416,2421,2426],{"type":150,"tag":181,"props":2417,"children":2419},{"className":2418},[824,342],[2420],{"type":155,"value":792},{"type":150,"tag":181,"props":2422,"children":2424},{"className":2423},[296,301,342],[2425],{"type":155,"value":797},{"type":150,"tag":181,"props":2427,"children":2429},{"className":2428},[835,342],[2430],{"type":155,"value":802},{"type":150,"tag":181,"props":2432,"children":2434},{"className":2433},[352],[2435],{"type":155,"value":355},{"type":150,"tag":181,"props":2437,"children":2439},{"className":2438},[318],[2440],{"type":150,"tag":181,"props":2441,"children":2443},{"className":2442,"style":2336},[323],[2444],{"type":150,"tag":181,"props":2445,"children":2446},{},[],{"type":150,"tag":181,"props":2448,"children":2450},{"className":2449},[835],[2451],{"type":155,"value":802},{"type":150,"tag":181,"props":2453,"children":2455},{"className":2454,"style":658},[377],[],{"type":150,"tag":181,"props":2457,"children":2460},{"className":2458},[663,2459],"amsrm",[2461],{"type":155,"value":1772},{"type":150,"tag":181,"props":2463,"children":2465},{"className":2464,"style":658},[377],[],{"type":150,"tag":181,"props":2467,"children":2469},{"className":2468},[285],[2470,2475],{"type":150,"tag":181,"props":2471,"children":2474},{"className":2472,"style":2473},[290],"height:3.1594em;vertical-align:-1.3297em;",[],{"type":150,"tag":181,"props":2476,"children":2478},{"className":2477},[296],[2479,2484,3192],{"type":150,"tag":181,"props":2480,"children":2483},{"className":2481},[824,2482],"nulldelimiter",[],{"type":150,"tag":181,"props":2485,"children":2487},{"className":2486},[1774],[2488],{"type":150,"tag":181,"props":2489,"children":2491},{"className":2490},[312,313],[2492,3180],{"type":150,"tag":181,"props":2493,"children":2495},{"className":2494},[318],[2496,3175],{"type":150,"tag":181,"props":2497,"children":2500},{"className":2498,"style":2499},[323],"height:1.8297em;",[2501,2792,2806],{"type":150,"tag":181,"props":2502,"children":2504},{"style":2503},"top:-2.11em;",[2505,2510],{"type":150,"tag":181,"props":2506,"children":2509},{"className":2507,"style":2508},[333],"height:3.0448em;",[],{"type":150,"tag":181,"props":2511,"children":2513},{"className":2512},[296],[2514,2592,2596,2601,2606,2693,2787],{"type":150,"tag":181,"props":2515,"children":2518},{"className":2516},[2517],"mop",[2519,2527],{"type":150,"tag":181,"props":2520,"children":2525},{"className":2521,"style":2524},[2517,2522,2523],"op-symbol","small-op","position:relative;top:0em;",[2526],{"type":155,"value":1788},{"type":150,"tag":181,"props":2528,"children":2530},{"className":2529},[307],[2531],{"type":150,"tag":181,"props":2532,"children":2534},{"className":2533},[312,313],[2535,2580],{"type":150,"tag":181,"props":2536,"children":2538},{"className":2537},[318],[2539,2575],{"type":150,"tag":181,"props":2540,"children":2543},{"className":2541,"style":2542},[323],"height:0.162em;",[2544],{"type":150,"tag":181,"props":2545,"children":2547},{"style":2546},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[2548,2552],{"type":150,"tag":181,"props":2549,"children":2551},{"className":2550,"style":334},[333],[],{"type":150,"tag":181,"props":2553,"children":2555},{"className":2554},[339,340,341,342],[2556],{"type":150,"tag":181,"props":2557,"children":2559},{"className":2558},[296,342],[2560,2565,2570],{"type":150,"tag":181,"props":2561,"children":2563},{"className":2562},[296,301,342],[2564],{"type":155,"value":558},{"type":150,"tag":181,"props":2566,"children":2568},{"className":2567},[663,342],[2569],{"type":155,"value":1706},{"type":150,"tag":181,"props":2571,"children":2573},{"className":2572},[296,301,342],[2574],{"type":155,"value":1035},{"type":150,"tag":181,"props":2576,"children":2578},{"className":2577},[352],[2579],{"type":155,"value":355},{"type":150,"tag":181,"props":2581,"children":2583},{"className":2582},[318],[2584],{"type":150,"tag":181,"props":2585,"children":2588},{"className":2586,"style":2587},[323],"height:0.3949em;",[2589],{"type":150,"tag":181,"props":2590,"children":2591},{},[],{"type":150,"tag":181,"props":2593,"children":2595},{"className":2594,"style":378},[377],[],{"type":150,"tag":181,"props":2597,"children":2599},{"className":2598},[2517],[2600],{"type":155,"value":1808},{"type":150,"tag":181,"props":2602,"children":2604},{"className":2603},[824],[2605],{"type":155,"value":792},{"type":150,"tag":181,"props":2607,"children":2609},{"className":2608},[296],[2610,2615],{"type":150,"tag":181,"props":2611,"children":2613},{"className":2612,"style":2151},[296,301],[2614],{"type":155,"value":1667},{"type":150,"tag":181,"props":2616,"children":2618},{"className":2617},[307],[2619],{"type":150,"tag":181,"props":2620,"children":2622},{"className":2621},[312,313],[2623,2682],{"type":150,"tag":181,"props":2624,"children":2626},{"className":2625},[318],[2627,2677],{"type":150,"tag":181,"props":2628,"children":2630},{"className":2629,"style":1276},[323],[2631,2647],{"type":150,"tag":181,"props":2632,"children":2633},{"style":2172},[2634,2638],{"type":150,"tag":181,"props":2635,"children":2637},{"className":2636,"style":334},[333],[],{"type":150,"tag":181,"props":2639,"children":2641},{"className":2640},[339,340,341,342],[2642],{"type":150,"tag":181,"props":2643,"children":2645},{"className":2644},[296,301,342],[2646],{"type":155,"value":1035},{"type":150,"tag":181,"props":2648,"children":2649},{"style":1297},[2650,2654],{"type":150,"tag":181,"props":2651,"children":2653},{"className":2652,"style":334},[333],[],{"type":150,"tag":181,"props":2655,"children":2657},{"className":2656},[339,340,341,342],[2658],{"type":150,"tag":181,"props":2659,"children":2661},{"className":2660},[296,342],[2662,2667,2672],{"type":150,"tag":181,"props":2663,"children":2665},{"className":2664},[824,342],[2666],{"type":155,"value":792},{"type":150,"tag":181,"props":2668,"children":2670},{"className":2669},[296,301,342],[2671],{"type":155,"value":797},{"type":150,"tag":181,"props":2673,"children":2675},{"className":2674},[835,342],[2676],{"type":155,"value":802},{"type":150,"tag":181,"props":2678,"children":2680},{"className":2679},[352],[2681],{"type":155,"value":355},{"type":150,"tag":181,"props":2683,"children":2685},{"className":2684},[318],[2686],{"type":150,"tag":181,"props":2687,"children":2689},{"className":2688,"style":1338},[323],[2690],{"type":150,"tag":181,"props":2691,"children":2692},{},[],{"type":150,"tag":181,"props":2694,"children":2696},{"className":2695},[296],[2697,2702],{"type":150,"tag":181,"props":2698,"children":2700},{"className":2699,"style":2249},[296,301],[2701],{"type":155,"value":1698},{"type":150,"tag":181,"props":2703,"children":2705},{"className":2704},[307],[2706],{"type":150,"tag":181,"props":2707,"children":2709},{"className":2708},[312,313],[2710,2775],{"type":150,"tag":181,"props":2711,"children":2713},{"className":2712},[318],[2714,2770],{"type":150,"tag":181,"props":2715,"children":2717},{"className":2716,"style":1276},[323],[2718,2735],{"type":150,"tag":181,"props":2719,"children":2721},{"style":2720},"top:-2.4231em;margin-left:-0.0315em;margin-right:0.05em;",[2722,2726],{"type":150,"tag":181,"props":2723,"children":2725},{"className":2724,"style":334},[333],[],{"type":150,"tag":181,"props":2727,"children":2729},{"className":2728},[339,340,341,342],[2730],{"type":150,"tag":181,"props":2731,"children":2733},{"className":2732},[296,301,342],[2734],{"type":155,"value":558},{"type":150,"tag":181,"props":2736,"children":2737},{"style":1297},[2738,2742],{"type":150,"tag":181,"props":2739,"children":2741},{"className":2740,"style":334},[333],[],{"type":150,"tag":181,"props":2743,"children":2745},{"className":2744},[339,340,341,342],[2746],{"type":150,"tag":181,"props":2747,"children":2749},{"className":2748},[296,342],[2750,2755,2760,2765],{"type":150,"tag":181,"props":2751,"children":2753},{"className":2752},[824,342],[2754],{"type":155,"value":792},{"type":150,"tag":181,"props":2756,"children":2758},{"className":2757},[296,301,342],[2759],{"type":155,"value":797},{"type":150,"tag":181,"props":2761,"children":2763},{"className":2762},[835,342],[2764],{"type":155,"value":802},{"type":150,"tag":181,"props":2766,"children":2768},{"className":2767},[296,342],[2769],{"type":155,"value":1874},{"type":150,"tag":181,"props":2771,"children":2773},{"className":2772},[352],[2774],{"type":155,"value":355},{"type":150,"tag":181,"props":2776,"children":2778},{"className":2777},[318],[2779],{"type":150,"tag":181,"props":2780,"children":2783},{"className":2781,"style":2782},[323],"height:0.2769em;",[2784],{"type":150,"tag":181,"props":2785,"children":2786},{},[],{"type":150,"tag":181,"props":2788,"children":2790},{"className":2789},[835],[2791],{"type":155,"value":802},{"type":150,"tag":181,"props":2793,"children":2795},{"style":2794},"top:-3.2748em;",[2796,2800],{"type":150,"tag":181,"props":2797,"children":2799},{"className":2798,"style":2508},[333],[],{"type":150,"tag":181,"props":2801,"children":2805},{"className":2802,"style":2804},[2803],"frac-line","border-bottom-width:0.04em;",[],{"type":150,"tag":181,"props":2807,"children":2809},{"style":2808},"top:-3.8297em;",[2810,2814],{"type":150,"tag":181,"props":2811,"children":2813},{"className":2812,"style":2508},[333],[],{"type":150,"tag":181,"props":2815,"children":2817},{"className":2816},[296],[2818,2889,2893,2898,2903,2990,3082,3087],{"type":150,"tag":181,"props":2819,"children":2821},{"className":2820},[2517],[2822,2827],{"type":150,"tag":181,"props":2823,"children":2825},{"className":2824,"style":2524},[2517,2522,2523],[2826],{"type":155,"value":1788},{"type":150,"tag":181,"props":2828,"children":2830},{"className":2829},[307],[2831],{"type":150,"tag":181,"props":2832,"children":2834},{"className":2833},[312,313],[2835,2878],{"type":150,"tag":181,"props":2836,"children":2838},{"className":2837},[318],[2839,2873],{"type":150,"tag":181,"props":2840,"children":2842},{"className":2841,"style":2542},[323],[2843],{"type":150,"tag":181,"props":2844,"children":2845},{"style":2546},[2846,2850],{"type":150,"tag":181,"props":2847,"children":2849},{"className":2848,"style":334},[333],[],{"type":150,"tag":181,"props":2851,"children":2853},{"className":2852},[339,340,341,342],[2854],{"type":150,"tag":181,"props":2855,"children":2857},{"className":2856},[296,342],[2858,2863,2868],{"type":150,"tag":181,"props":2859,"children":2861},{"className":2860},[296,301,342],[2862],{"type":155,"value":558},{"type":150,"tag":181,"props":2864,"children":2866},{"className":2865},[663,342],[2867],{"type":155,"value":1706},{"type":150,"tag":181,"props":2869,"children":2871},{"className":2870},[296,301,342],[2872],{"type":155,"value":1035},{"type":150,"tag":181,"props":2874,"children":2876},{"className":2875},[352],[2877],{"type":155,"value":355},{"type":150,"tag":181,"props":2879,"children":2881},{"className":2880},[318],[2882],{"type":150,"tag":181,"props":2883,"children":2885},{"className":2884,"style":2587},[323],[2886],{"type":150,"tag":181,"props":2887,"children":2888},{},[],{"type":150,"tag":181,"props":2890,"children":2892},{"className":2891,"style":378},[377],[],{"type":150,"tag":181,"props":2894,"children":2896},{"className":2895},[2517],[2897],{"type":155,"value":1808},{"type":150,"tag":181,"props":2899,"children":2901},{"className":2900},[824],[2902],{"type":155,"value":792},{"type":150,"tag":181,"props":2904,"children":2906},{"className":2905},[296],[2907,2912],{"type":150,"tag":181,"props":2908,"children":2910},{"className":2909,"style":2151},[296,301],[2911],{"type":155,"value":1667},{"type":150,"tag":181,"props":2913,"children":2915},{"className":2914},[307],[2916],{"type":150,"tag":181,"props":2917,"children":2919},{"className":2918},[312,313],[2920,2979],{"type":150,"tag":181,"props":2921,"children":2923},{"className":2922},[318],[2924,2974],{"type":150,"tag":181,"props":2925,"children":2927},{"className":2926,"style":1276},[323],[2928,2944],{"type":150,"tag":181,"props":2929,"children":2930},{"style":2172},[2931,2935],{"type":150,"tag":181,"props":2932,"children":2934},{"className":2933,"style":334},[333],[],{"type":150,"tag":181,"props":2936,"children":2938},{"className":2937},[339,340,341,342],[2939],{"type":150,"tag":181,"props":2940,"children":2942},{"className":2941},[296,301,342],[2943],{"type":155,"value":1035},{"type":150,"tag":181,"props":2945,"children":2946},{"style":1297},[2947,2951],{"type":150,"tag":181,"props":2948,"children":2950},{"className":2949,"style":334},[333],[],{"type":150,"tag":181,"props":2952,"children":2954},{"className":2953},[339,340,341,342],[2955],{"type":150,"tag":181,"props":2956,"children":2958},{"className":2957},[296,342],[2959,2964,2969],{"type":150,"tag":181,"props":2960,"children":2962},{"className":2961},[824,342],[2963],{"type":155,"value":792},{"type":150,"tag":181,"props":2965,"children":2967},{"className":2966},[296,301,342],[2968],{"type":155,"value":797},{"type":150,"tag":181,"props":2970,"children":2972},{"className":2971},[835,342],[2973],{"type":155,"value":802},{"type":150,"tag":181,"props":2975,"children":2977},{"className":2976},[352],[2978],{"type":155,"value":355},{"type":150,"tag":181,"props":2980,"children":2982},{"className":2981},[318],[2983],{"type":150,"tag":181,"props":2984,"children":2986},{"className":2985,"style":1338},[323],[2987],{"type":150,"tag":181,"props":2988,"children":2989},{},[],{"type":150,"tag":181,"props":2991,"children":2993},{"className":2992},[296],[2994,2999],{"type":150,"tag":181,"props":2995,"children":2997},{"className":2996,"style":2249},[296,301],[2998],{"type":155,"value":1698},{"type":150,"tag":181,"props":3000,"children":3002},{"className":3001},[307],[3003],{"type":150,"tag":181,"props":3004,"children":3006},{"className":3005},[312,313],[3007,3071],{"type":150,"tag":181,"props":3008,"children":3010},{"className":3009},[318],[3011,3066],{"type":150,"tag":181,"props":3012,"children":3014},{"className":3013,"style":1276},[323],[3015,3031],{"type":150,"tag":181,"props":3016,"children":3017},{"style":2720},[3018,3022],{"type":150,"tag":181,"props":3019,"children":3021},{"className":3020,"style":334},[333],[],{"type":150,"tag":181,"props":3023,"children":3025},{"className":3024},[339,340,341,342],[3026],{"type":150,"tag":181,"props":3027,"children":3029},{"className":3028},[296,301,342],[3030],{"type":155,"value":558},{"type":150,"tag":181,"props":3032,"children":3033},{"style":1297},[3034,3038],{"type":150,"tag":181,"props":3035,"children":3037},{"className":3036,"style":334},[333],[],{"type":150,"tag":181,"props":3039,"children":3041},{"className":3040},[339,340,341,342],[3042],{"type":150,"tag":181,"props":3043,"children":3045},{"className":3044},[296,342],[3046,3051,3056,3061],{"type":150,"tag":181,"props":3047,"children":3049},{"className":3048},[824,342],[3050],{"type":155,"value":792},{"type":150,"tag":181,"props":3052,"children":3054},{"className":3053},[296,301,342],[3055],{"type":155,"value":797},{"type":150,"tag":181,"props":3057,"children":3059},{"className":3058},[835,342],[3060],{"type":155,"value":802},{"type":150,"tag":181,"props":3062,"children":3064},{"className":3063},[296,342],[3065],{"type":155,"value":1874},{"type":150,"tag":181,"props":3067,"children":3069},{"className":3068},[352],[3070],{"type":155,"value":355},{"type":150,"tag":181,"props":3072,"children":3074},{"className":3073},[318],[3075],{"type":150,"tag":181,"props":3076,"children":3078},{"className":3077,"style":2782},[323],[3079],{"type":150,"tag":181,"props":3080,"children":3081},{},[],{"type":150,"tag":181,"props":3083,"children":3085},{"className":3084},[835],[3086],{"type":155,"value":802},{"type":150,"tag":181,"props":3088,"children":3090},{"className":3089},[296],[3091,3096],{"type":150,"tag":181,"props":3092,"children":3094},{"className":3093,"style":2151},[296,301],[3095],{"type":155,"value":1737},{"type":150,"tag":181,"props":3097,"children":3099},{"className":3098},[307],[3100],{"type":150,"tag":181,"props":3101,"children":3103},{"className":3102},[312,313],[3104,3164],{"type":150,"tag":181,"props":3105,"children":3107},{"className":3106},[318],[3108,3159],{"type":150,"tag":181,"props":3109,"children":3111},{"className":3110,"style":1276},[323],[3112,3129],{"type":150,"tag":181,"props":3113,"children":3115},{"style":3114},"top:-2.4231em;margin-left:-0.0359em;margin-right:0.05em;",[3116,3120],{"type":150,"tag":181,"props":3117,"children":3119},{"className":3118,"style":334},[333],[],{"type":150,"tag":181,"props":3121,"children":3123},{"className":3122},[339,340,341,342],[3124],{"type":150,"tag":181,"props":3125,"children":3127},{"className":3126},[296,301,342],[3128],{"type":155,"value":558},{"type":150,"tag":181,"props":3130,"children":3131},{"style":1297},[3132,3136],{"type":150,"tag":181,"props":3133,"children":3135},{"className":3134,"style":334},[333],[],{"type":150,"tag":181,"props":3137,"children":3139},{"className":3138},[339,340,341,342],[3140],{"type":150,"tag":181,"props":3141,"children":3143},{"className":3142},[296,342],[3144,3149,3154],{"type":150,"tag":181,"props":3145,"children":3147},{"className":3146},[824,342],[3148],{"type":155,"value":792},{"type":150,"tag":181,"props":3150,"children":3152},{"className":3151},[296,301,342],[3153],{"type":155,"value":797},{"type":150,"tag":181,"props":3155,"children":3157},{"className":3156},[835,342],[3158],{"type":155,"value":802},{"type":150,"tag":181,"props":3160,"children":3162},{"className":3161},[352],[3163],{"type":155,"value":355},{"type":150,"tag":181,"props":3165,"children":3167},{"className":3166},[318],[3168],{"type":150,"tag":181,"props":3169,"children":3171},{"className":3170,"style":2782},[323],[3172],{"type":150,"tag":181,"props":3173,"children":3174},{},[],{"type":150,"tag":181,"props":3176,"children":3178},{"className":3177},[352],[3179],{"type":155,"value":355},{"type":150,"tag":181,"props":3181,"children":3183},{"className":3182},[318],[3184],{"type":150,"tag":181,"props":3185,"children":3188},{"className":3186,"style":3187},[323],"height:1.3297em;",[3189],{"type":150,"tag":181,"props":3190,"children":3191},{},[],{"type":150,"tag":181,"props":3193,"children":3195},{"className":3194},[835,2482],[],{"type":150,"tag":181,"props":3197,"children":3199},{"className":3198},[1003],[3200],{"type":150,"tag":181,"props":3201,"children":3203},{"className":3202},[184],[3204,3505],{"type":150,"tag":181,"props":3205,"children":3207},{"className":3206},[189],[3208],{"type":150,"tag":192,"props":3209,"children":3210},{"xmlns":194,"display":1015},[3211],{"type":150,"tag":197,"props":3212,"children":3213},{},[3214,3500],{"type":150,"tag":201,"props":3215,"children":3216},{},[3217,3243,3247,3258,3285,3289,3307,3311,3315,3341,3345,3356,3382,3386,3404,3408,3411,3437,3441,3452,3478,3482],{"type":150,"tag":1046,"props":3218,"children":3219},{},[3220,3224,3228],{"type":150,"tag":209,"props":3221,"children":3222},{},[3223],{"type":155,"value":1667},{"type":150,"tag":209,"props":3225,"children":3226},{},[3227],{"type":155,"value":558},{"type":150,"tag":201,"props":3229,"children":3230},{},[3231,3235,3239],{"type":150,"tag":221,"props":3232,"children":3233},{"stretchy":789},[3234],{"type":155,"value":792},{"type":150,"tag":209,"props":3236,"children":3237},{},[3238],{"type":155,"value":797},{"type":150,"tag":221,"props":3240,"children":3241},{"stretchy":789},[3242],{"type":155,"value":802},{"type":150,"tag":221,"props":3244,"children":3245},{},[3246],{"type":155,"value":865},{"type":150,"tag":205,"props":3248,"children":3249},{},[3250,3254],{"type":150,"tag":209,"props":3251,"children":3252},{},[3253],{"type":155,"value":213},{"type":150,"tag":209,"props":3255,"children":3256},{},[3257],{"type":155,"value":558},{"type":150,"tag":1046,"props":3259,"children":3260},{},[3261,3266,3270],{"type":150,"tag":209,"props":3262,"children":3263},{},[3264],{"type":155,"value":3265},"W",{"type":150,"tag":209,"props":3267,"children":3268},{},[3269],{"type":155,"value":1667},{"type":150,"tag":201,"props":3271,"children":3272},{},[3273,3277,3281],{"type":150,"tag":221,"props":3274,"children":3275},{"stretchy":789},[3276],{"type":155,"value":792},{"type":150,"tag":209,"props":3278,"children":3279},{},[3280],{"type":155,"value":797},{"type":150,"tag":221,"props":3282,"children":3283},{"stretchy":789},[3284],{"type":155,"value":802},{"type":150,"tag":221,"props":3286,"children":3287},{},[3288],{"type":155,"value":563},{"type":150,"tag":565,"props":3290,"children":3291},{},[3292,3296],{"type":150,"tag":209,"props":3293,"children":3294},{"mathvariant":570},[3295],{"type":155,"value":573},{"type":150,"tag":205,"props":3297,"children":3298},{},[3299,3303],{"type":150,"tag":209,"props":3300,"children":3301},{},[3302],{"type":155,"value":578},{"type":150,"tag":209,"props":3304,"children":3305},{},[3306],{"type":155,"value":1698},{"type":150,"tag":221,"props":3308,"children":3309},{"separator":223},[3310],{"type":155,"value":226},{"type":150,"tag":377,"props":3312,"children":3314},{"width":3313},"1em",[],{"type":150,"tag":1046,"props":3316,"children":3317},{},[3318,3322,3326],{"type":150,"tag":209,"props":3319,"children":3320},{},[3321],{"type":155,"value":1698},{"type":150,"tag":209,"props":3323,"children":3324},{},[3325],{"type":155,"value":558},{"type":150,"tag":201,"props":3327,"children":3328},{},[3329,3333,3337],{"type":150,"tag":221,"props":3330,"children":3331},{"stretchy":789},[3332],{"type":155,"value":792},{"type":150,"tag":209,"props":3334,"children":3335},{},[3336],{"type":155,"value":797},{"type":150,"tag":221,"props":3338,"children":3339},{"stretchy":789},[3340],{"type":155,"value":802},{"type":150,"tag":221,"props":3342,"children":3343},{},[3344],{"type":155,"value":865},{"type":150,"tag":205,"props":3346,"children":3347},{},[3348,3352],{"type":150,"tag":209,"props":3349,"children":3350},{},[3351],{"type":155,"value":213},{"type":150,"tag":209,"props":3353,"children":3354},{},[3355],{"type":155,"value":558},{"type":150,"tag":1046,"props":3357,"children":3358},{},[3359,3363,3367],{"type":150,"tag":209,"props":3360,"children":3361},{},[3362],{"type":155,"value":3265},{"type":150,"tag":209,"props":3364,"children":3365},{},[3366],{"type":155,"value":1698},{"type":150,"tag":201,"props":3368,"children":3369},{},[3370,3374,3378],{"type":150,"tag":221,"props":3371,"children":3372},{"stretchy":789},[3373],{"type":155,"value":792},{"type":150,"tag":209,"props":3375,"children":3376},{},[3377],{"type":155,"value":797},{"type":150,"tag":221,"props":3379,"children":3380},{"stretchy":789},[3381],{"type":155,"value":802},{"type":150,"tag":221,"props":3383,"children":3384},{},[3385],{"type":155,"value":563},{"type":150,"tag":565,"props":3387,"children":3388},{},[3389,3393],{"type":150,"tag":209,"props":3390,"children":3391},{"mathvariant":570},[3392],{"type":155,"value":573},{"type":150,"tag":205,"props":3394,"children":3395},{},[3396,3400],{"type":150,"tag":209,"props":3397,"children":3398},{},[3399],{"type":155,"value":578},{"type":150,"tag":209,"props":3401,"children":3402},{},[3403],{"type":155,"value":1698},{"type":150,"tag":221,"props":3405,"children":3406},{"separator":223},[3407],{"type":155,"value":226},{"type":150,"tag":377,"props":3409,"children":3410},{"width":3313},[],{"type":150,"tag":1046,"props":3412,"children":3413},{},[3414,3418,3422],{"type":150,"tag":209,"props":3415,"children":3416},{},[3417],{"type":155,"value":1737},{"type":150,"tag":209,"props":3419,"children":3420},{},[3421],{"type":155,"value":558},{"type":150,"tag":201,"props":3423,"children":3424},{},[3425,3429,3433],{"type":150,"tag":221,"props":3426,"children":3427},{"stretchy":789},[3428],{"type":155,"value":792},{"type":150,"tag":209,"props":3430,"children":3431},{},[3432],{"type":155,"value":797},{"type":150,"tag":221,"props":3434,"children":3435},{"stretchy":789},[3436],{"type":155,"value":802},{"type":150,"tag":221,"props":3438,"children":3439},{},[3440],{"type":155,"value":865},{"type":150,"tag":205,"props":3442,"children":3443},{},[3444,3448],{"type":150,"tag":209,"props":3445,"children":3446},{},[3447],{"type":155,"value":213},{"type":150,"tag":209,"props":3449,"children":3450},{},[3451],{"type":155,"value":558},{"type":150,"tag":1046,"props":3453,"children":3454},{},[3455,3459,3463],{"type":150,"tag":209,"props":3456,"children":3457},{},[3458],{"type":155,"value":3265},{"type":150,"tag":209,"props":3460,"children":3461},{},[3462],{"type":155,"value":1737},{"type":150,"tag":201,"props":3464,"children":3465},{},[3466,3470,3474],{"type":150,"tag":221,"props":3467,"children":3468},{"stretchy":789},[3469],{"type":155,"value":792},{"type":150,"tag":209,"props":3471,"children":3472},{},[3473],{"type":155,"value":797},{"type":150,"tag":221,"props":3475,"children":3476},{"stretchy":789},[3477],{"type":155,"value":802},{"type":150,"tag":221,"props":3479,"children":3480},{},[3481],{"type":155,"value":563},{"type":150,"tag":565,"props":3483,"children":3484},{},[3485,3489],{"type":150,"tag":209,"props":3486,"children":3487},{"mathvariant":570},[3488],{"type":155,"value":573},{"type":150,"tag":205,"props":3490,"children":3491},{},[3492,3496],{"type":150,"tag":209,"props":3493,"children":3494},{},[3495],{"type":155,"value":578},{"type":150,"tag":209,"props":3497,"children":3498},{},[3499],{"type":155,"value":1737},{"type":150,"tag":271,"props":3501,"children":3502},{"encoding":273},[3503],{"type":155,"value":3504},"q_i^{(s)} = x_i W_q^{(s)} \\in \\mathbb{R}^{d_k}, \\quad k_i^{(s)} = x_i W_k^{(s)} \\in \\mathbb{R}^{d_k}, \\quad v_i^{(s)} = x_i W_v^{(s)} \\in \\mathbb{R}^{d_v}",{"type":150,"tag":181,"props":3506,"children":3508},{"className":3507,"ariaHidden":223},[280],[3509,3618,3789,4015,4183,4401,4568],{"type":150,"tag":181,"props":3510,"children":3512},{"className":3511},[285],[3513,3518,3605,3609,3614],{"type":150,"tag":181,"props":3514,"children":3517},{"className":3515,"style":3516},[290],"height:1.3217em;vertical-align:-0.2769em;",[],{"type":150,"tag":181,"props":3519,"children":3521},{"className":3520},[296],[3522,3527],{"type":150,"tag":181,"props":3523,"children":3525},{"className":3524,"style":2151},[296,301],[3526],{"type":155,"value":1667},{"type":150,"tag":181,"props":3528,"children":3530},{"className":3529},[307],[3531],{"type":150,"tag":181,"props":3532,"children":3534},{"className":3533},[312,313],[3535,3594],{"type":150,"tag":181,"props":3536,"children":3538},{"className":3537},[318],[3539,3589],{"type":150,"tag":181,"props":3540,"children":3542},{"className":3541,"style":1276},[323],[3543,3559],{"type":150,"tag":181,"props":3544,"children":3545},{"style":3114},[3546,3550],{"type":150,"tag":181,"props":3547,"children":3549},{"className":3548,"style":334},[333],[],{"type":150,"tag":181,"props":3551,"children":3553},{"className":3552},[339,340,341,342],[3554],{"type":150,"tag":181,"props":3555,"children":3557},{"className":3556},[296,301,342],[3558],{"type":155,"value":558},{"type":150,"tag":181,"props":3560,"children":3561},{"style":1297},[3562,3566],{"type":150,"tag":181,"props":3563,"children":3565},{"className":3564,"style":334},[333],[],{"type":150,"tag":181,"props":3567,"children":3569},{"className":3568},[339,340,341,342],[3570],{"type":150,"tag":181,"props":3571,"children":3573},{"className":3572},[296,342],[3574,3579,3584],{"type":150,"tag":181,"props":3575,"children":3577},{"className":3576},[824,342],[3578],{"type":155,"value":792},{"type":150,"tag":181,"props":3580,"children":3582},{"className":3581},[296,301,342],[3583],{"type":155,"value":797},{"type":150,"tag":181,"props":3585,"children":3587},{"className":3586},[835,342],[3588],{"type":155,"value":802},{"type":150,"tag":181,"props":3590,"children":3592},{"className":3591},[352],[3593],{"type":155,"value":355},{"type":150,"tag":181,"props":3595,"children":3597},{"className":3596},[318],[3598],{"type":150,"tag":181,"props":3599,"children":3601},{"className":3600,"style":2782},[323],[3602],{"type":150,"tag":181,"props":3603,"children":3604},{},[],{"type":150,"tag":181,"props":3606,"children":3608},{"className":3607,"style":658},[377],[],{"type":150,"tag":181,"props":3610,"children":3612},{"className":3611},[663],[3613],{"type":155,"value":865},{"type":150,"tag":181,"props":3615,"children":3617},{"className":3616,"style":658},[377],[],{"type":150,"tag":181,"props":3619,"children":3621},{"className":3620},[285],[3622,3627,3684,3776,3780,3785],{"type":150,"tag":181,"props":3623,"children":3626},{"className":3624,"style":3625},[290],"height:1.3211em;vertical-align:-0.3831em;",[],{"type":150,"tag":181,"props":3628,"children":3630},{"className":3629},[296],[3631,3636],{"type":150,"tag":181,"props":3632,"children":3634},{"className":3633},[296,301],[3635],{"type":155,"value":213},{"type":150,"tag":181,"props":3637,"children":3639},{"className":3638},[307],[3640],{"type":150,"tag":181,"props":3641,"children":3643},{"className":3642},[312,313],[3644,3673],{"type":150,"tag":181,"props":3645,"children":3647},{"className":3646},[318],[3648,3668],{"type":150,"tag":181,"props":3649,"children":3651},{"className":3650,"style":621},[323],[3652],{"type":150,"tag":181,"props":3653,"children":3654},{"style":328},[3655,3659],{"type":150,"tag":181,"props":3656,"children":3658},{"className":3657,"style":334},[333],[],{"type":150,"tag":181,"props":3660,"children":3662},{"className":3661},[339,340,341,342],[3663],{"type":150,"tag":181,"props":3664,"children":3666},{"className":3665},[296,301,342],[3667],{"type":155,"value":558},{"type":150,"tag":181,"props":3669,"children":3671},{"className":3670},[352],[3672],{"type":155,"value":355},{"type":150,"tag":181,"props":3674,"children":3676},{"className":3675},[318],[3677],{"type":150,"tag":181,"props":3678,"children":3680},{"className":3679,"style":363},[323],[3681],{"type":150,"tag":181,"props":3682,"children":3683},{},[],{"type":150,"tag":181,"props":3685,"children":3687},{"className":3686},[296],[3688,3694],{"type":150,"tag":181,"props":3689,"children":3692},{"className":3690,"style":3691},[296,301],"margin-right:0.1389em;",[3693],{"type":155,"value":3265},{"type":150,"tag":181,"props":3695,"children":3697},{"className":3696},[307],[3698],{"type":150,"tag":181,"props":3699,"children":3701},{"className":3700},[312,313],[3702,3764],{"type":150,"tag":181,"props":3703,"children":3705},{"className":3704},[318],[3706,3759],{"type":150,"tag":181,"props":3707,"children":3710},{"className":3708,"style":3709},[323],"height:0.938em;",[3711,3728],{"type":150,"tag":181,"props":3712,"children":3714},{"style":3713},"top:-2.453em;margin-left:-0.1389em;margin-right:0.05em;",[3715,3719],{"type":150,"tag":181,"props":3716,"children":3718},{"className":3717,"style":334},[333],[],{"type":150,"tag":181,"props":3720,"children":3722},{"className":3721},[339,340,341,342],[3723],{"type":150,"tag":181,"props":3724,"children":3726},{"className":3725,"style":2151},[296,301,342],[3727],{"type":155,"value":1667},{"type":150,"tag":181,"props":3729,"children":3731},{"style":3730},"top:-3.113em;margin-right:0.05em;",[3732,3736],{"type":150,"tag":181,"props":3733,"children":3735},{"className":3734,"style":334},[333],[],{"type":150,"tag":181,"props":3737,"children":3739},{"className":3738},[339,340,341,342],[3740],{"type":150,"tag":181,"props":3741,"children":3743},{"className":3742},[296,342],[3744,3749,3754],{"type":150,"tag":181,"props":3745,"children":3747},{"className":3746},[824,342],[3748],{"type":155,"value":792},{"type":150,"tag":181,"props":3750,"children":3752},{"className":3751},[296,301,342],[3753],{"type":155,"value":797},{"type":150,"tag":181,"props":3755,"children":3757},{"className":3756},[835,342],[3758],{"type":155,"value":802},{"type":150,"tag":181,"props":3760,"children":3762},{"className":3761},[352],[3763],{"type":155,"value":355},{"type":150,"tag":181,"props":3765,"children":3767},{"className":3766},[318],[3768],{"type":150,"tag":181,"props":3769,"children":3772},{"className":3770,"style":3771},[323],"height:0.3831em;",[3773],{"type":150,"tag":181,"props":3774,"children":3775},{},[],{"type":150,"tag":181,"props":3777,"children":3779},{"className":3778,"style":658},[377],[],{"type":150,"tag":181,"props":3781,"children":3783},{"className":3782},[663],[3784],{"type":155,"value":563},{"type":150,"tag":181,"props":3786,"children":3788},{"className":3787,"style":658},[377],[],{"type":150,"tag":181,"props":3790,"children":3792},{"className":3791},[285],[3793,3797,3901,3906,3911,3915,4002,4006,4011],{"type":150,"tag":181,"props":3794,"children":3796},{"className":3795,"style":3516},[290],[],{"type":150,"tag":181,"props":3798,"children":3800},{"className":3799},[296],[3801,3806],{"type":150,"tag":181,"props":3802,"children":3804},{"className":3803},[296,686],[3805],{"type":155,"value":573},{"type":150,"tag":181,"props":3807,"children":3809},{"className":3808},[307],[3810],{"type":150,"tag":181,"props":3811,"children":3813},{"className":3812},[312],[3814],{"type":150,"tag":181,"props":3815,"children":3817},{"className":3816},[318],[3818],{"type":150,"tag":181,"props":3819,"children":3822},{"className":3820,"style":3821},[323],"height:0.8991em;",[3823],{"type":150,"tag":181,"props":3824,"children":3825},{"style":3730},[3826,3830],{"type":150,"tag":181,"props":3827,"children":3829},{"className":3828,"style":334},[333],[],{"type":150,"tag":181,"props":3831,"children":3833},{"className":3832},[339,340,341,342],[3834],{"type":150,"tag":181,"props":3835,"children":3837},{"className":3836},[296,342],[3838],{"type":150,"tag":181,"props":3839,"children":3841},{"className":3840},[296,342],[3842,3847],{"type":150,"tag":181,"props":3843,"children":3845},{"className":3844},[296,301,342],[3846],{"type":155,"value":578},{"type":150,"tag":181,"props":3848,"children":3850},{"className":3849},[307],[3851],{"type":150,"tag":181,"props":3852,"children":3854},{"className":3853},[312,313],[3855,3889],{"type":150,"tag":181,"props":3856,"children":3858},{"className":3857},[318],[3859,3884],{"type":150,"tag":181,"props":3860,"children":3863},{"className":3861,"style":3862},[323],"height:0.3448em;",[3864],{"type":150,"tag":181,"props":3865,"children":3867},{"style":3866},"top:-2.3488em;margin-left:0em;margin-right:0.0714em;",[3868,3873],{"type":150,"tag":181,"props":3869,"children":3872},{"className":3870,"style":3871},[333],"height:2.5em;",[],{"type":150,"tag":181,"props":3874,"children":3878},{"className":3875},[339,3876,3877,342],"reset-size3","size1",[3879],{"type":150,"tag":181,"props":3880,"children":3882},{"className":3881,"style":2249},[296,301,342],[3883],{"type":155,"value":1698},{"type":150,"tag":181,"props":3885,"children":3887},{"className":3886},[352],[3888],{"type":155,"value":355},{"type":150,"tag":181,"props":3890,"children":3892},{"className":3891},[318],[3893],{"type":150,"tag":181,"props":3894,"children":3897},{"className":3895,"style":3896},[323],"height:0.1512em;",[3898],{"type":150,"tag":181,"props":3899,"children":3900},{},[],{"type":150,"tag":181,"props":3902,"children":3904},{"className":3903},[371],[3905],{"type":155,"value":226},{"type":150,"tag":181,"props":3907,"children":3910},{"className":3908,"style":3909},[377],"margin-right:1em;",[],{"type":150,"tag":181,"props":3912,"children":3914},{"className":3913,"style":378},[377],[],{"type":150,"tag":181,"props":3916,"children":3918},{"className":3917},[296],[3919,3924],{"type":150,"tag":181,"props":3920,"children":3922},{"className":3921,"style":2249},[296,301],[3923],{"type":155,"value":1698},{"type":150,"tag":181,"props":3925,"children":3927},{"className":3926},[307],[3928],{"type":150,"tag":181,"props":3929,"children":3931},{"className":3930},[312,313],[3932,3991],{"type":150,"tag":181,"props":3933,"children":3935},{"className":3934},[318],[3936,3986],{"type":150,"tag":181,"props":3937,"children":3939},{"className":3938,"style":1276},[323],[3940,3956],{"type":150,"tag":181,"props":3941,"children":3942},{"style":2720},[3943,3947],{"type":150,"tag":181,"props":3944,"children":3946},{"className":3945,"style":334},[333],[],{"type":150,"tag":181,"props":3948,"children":3950},{"className":3949},[339,340,341,342],[3951],{"type":150,"tag":181,"props":3952,"children":3954},{"className":3953},[296,301,342],[3955],{"type":155,"value":558},{"type":150,"tag":181,"props":3957,"children":3958},{"style":1297},[3959,3963],{"type":150,"tag":181,"props":3960,"children":3962},{"className":3961,"style":334},[333],[],{"type":150,"tag":181,"props":3964,"children":3966},{"className":3965},[339,340,341,342],[3967],{"type":150,"tag":181,"props":3968,"children":3970},{"className":3969},[296,342],[3971,3976,3981],{"type":150,"tag":181,"props":3972,"children":3974},{"className":3973},[824,342],[3975],{"type":155,"value":792},{"type":150,"tag":181,"props":3977,"children":3979},{"className":3978},[296,301,342],[3980],{"type":155,"value":797},{"type":150,"tag":181,"props":3982,"children":3984},{"className":3983},[835,342],[3985],{"type":155,"value":802},{"type":150,"tag":181,"props":3987,"children":3989},{"className":3988},[352],[3990],{"type":155,"value":355},{"type":150,"tag":181,"props":3992,"children":3994},{"className":3993},[318],[3995],{"type":150,"tag":181,"props":3996,"children":3998},{"className":3997,"style":2782},[323],[3999],{"type":150,"tag":181,"props":4000,"children":4001},{},[],{"type":150,"tag":181,"props":4003,"children":4005},{"className":4004,"style":658},[377],[],{"type":150,"tag":181,"props":4007,"children":4009},{"className":4008},[663],[4010],{"type":155,"value":865},{"type":150,"tag":181,"props":4012,"children":4014},{"className":4013,"style":658},[377],[],{"type":150,"tag":181,"props":4016,"children":4018},{"className":4017},[285],[4019,4024,4081,4170,4174,4179],{"type":150,"tag":181,"props":4020,"children":4023},{"className":4021,"style":4022},[290],"height:1.3461em;vertical-align:-0.3013em;",[],{"type":150,"tag":181,"props":4025,"children":4027},{"className":4026},[296],[4028,4033],{"type":150,"tag":181,"props":4029,"children":4031},{"className":4030},[296,301],[4032],{"type":155,"value":213},{"type":150,"tag":181,"props":4034,"children":4036},{"className":4035},[307],[4037],{"type":150,"tag":181,"props":4038,"children":4040},{"className":4039},[312,313],[4041,4070],{"type":150,"tag":181,"props":4042,"children":4044},{"className":4043},[318],[4045,4065],{"type":150,"tag":181,"props":4046,"children":4048},{"className":4047,"style":621},[323],[4049],{"type":150,"tag":181,"props":4050,"children":4051},{"style":328},[4052,4056],{"type":150,"tag":181,"props":4053,"children":4055},{"className":4054,"style":334},[333],[],{"type":150,"tag":181,"props":4057,"children":4059},{"className":4058},[339,340,341,342],[4060],{"type":150,"tag":181,"props":4061,"children":4063},{"className":4062},[296,301,342],[4064],{"type":155,"value":558},{"type":150,"tag":181,"props":4066,"children":4068},{"className":4067},[352],[4069],{"type":155,"value":355},{"type":150,"tag":181,"props":4071,"children":4073},{"className":4072},[318],[4074],{"type":150,"tag":181,"props":4075,"children":4077},{"className":4076,"style":363},[323],[4078],{"type":150,"tag":181,"props":4079,"children":4080},{},[],{"type":150,"tag":181,"props":4082,"children":4084},{"className":4083},[296],[4085,4090],{"type":150,"tag":181,"props":4086,"children":4088},{"className":4087,"style":3691},[296,301],[4089],{"type":155,"value":3265},{"type":150,"tag":181,"props":4091,"children":4093},{"className":4092},[307],[4094],{"type":150,"tag":181,"props":4095,"children":4097},{"className":4096},[312,313],[4098,4158],{"type":150,"tag":181,"props":4099,"children":4101},{"className":4100},[318],[4102,4153],{"type":150,"tag":181,"props":4103,"children":4105},{"className":4104,"style":1276},[323],[4106,4123],{"type":150,"tag":181,"props":4107,"children":4109},{"style":4108},"top:-2.3987em;margin-left:-0.1389em;margin-right:0.05em;",[4110,4114],{"type":150,"tag":181,"props":4111,"children":4113},{"className":4112,"style":334},[333],[],{"type":150,"tag":181,"props":4115,"children":4117},{"className":4116},[339,340,341,342],[4118],{"type":150,"tag":181,"props":4119,"children":4121},{"className":4120,"style":2249},[296,301,342],[4122],{"type":155,"value":1698},{"type":150,"tag":181,"props":4124,"children":4125},{"style":1297},[4126,4130],{"type":150,"tag":181,"props":4127,"children":4129},{"className":4128,"style":334},[333],[],{"type":150,"tag":181,"props":4131,"children":4133},{"className":4132},[339,340,341,342],[4134],{"type":150,"tag":181,"props":4135,"children":4137},{"className":4136},[296,342],[4138,4143,4148],{"type":150,"tag":181,"props":4139,"children":4141},{"className":4140},[824,342],[4142],{"type":155,"value":792},{"type":150,"tag":181,"props":4144,"children":4146},{"className":4145},[296,301,342],[4147],{"type":155,"value":797},{"type":150,"tag":181,"props":4149,"children":4151},{"className":4150},[835,342],[4152],{"type":155,"value":802},{"type":150,"tag":181,"props":4154,"children":4156},{"className":4155},[352],[4157],{"type":155,"value":355},{"type":150,"tag":181,"props":4159,"children":4161},{"className":4160},[318],[4162],{"type":150,"tag":181,"props":4163,"children":4166},{"className":4164,"style":4165},[323],"height:0.3013em;",[4167],{"type":150,"tag":181,"props":4168,"children":4169},{},[],{"type":150,"tag":181,"props":4171,"children":4173},{"className":4172,"style":658},[377],[],{"type":150,"tag":181,"props":4175,"children":4177},{"className":4176},[663],[4178],{"type":155,"value":563},{"type":150,"tag":181,"props":4180,"children":4182},{"className":4181,"style":658},[377],[],{"type":150,"tag":181,"props":4184,"children":4186},{"className":4185},[285],[4187,4191,4288,4293,4297,4301,4388,4392,4397],{"type":150,"tag":181,"props":4188,"children":4190},{"className":4189,"style":3516},[290],[],{"type":150,"tag":181,"props":4192,"children":4194},{"className":4193},[296],[4195,4200],{"type":150,"tag":181,"props":4196,"children":4198},{"className":4197},[296,686],[4199],{"type":155,"value":573},{"type":150,"tag":181,"props":4201,"children":4203},{"className":4202},[307],[4204],{"type":150,"tag":181,"props":4205,"children":4207},{"className":4206},[312],[4208],{"type":150,"tag":181,"props":4209,"children":4211},{"className":4210},[318],[4212],{"type":150,"tag":181,"props":4213,"children":4215},{"className":4214,"style":3821},[323],[4216],{"type":150,"tag":181,"props":4217,"children":4218},{"style":3730},[4219,4223],{"type":150,"tag":181,"props":4220,"children":4222},{"className":4221,"style":334},[333],[],{"type":150,"tag":181,"props":4224,"children":4226},{"className":4225},[339,340,341,342],[4227],{"type":150,"tag":181,"props":4228,"children":4230},{"className":4229},[296,342],[4231],{"type":150,"tag":181,"props":4232,"children":4234},{"className":4233},[296,342],[4235,4240],{"type":150,"tag":181,"props":4236,"children":4238},{"className":4237},[296,301,342],[4239],{"type":155,"value":578},{"type":150,"tag":181,"props":4241,"children":4243},{"className":4242},[307],[4244],{"type":150,"tag":181,"props":4245,"children":4247},{"className":4246},[312,313],[4248,4277],{"type":150,"tag":181,"props":4249,"children":4251},{"className":4250},[318],[4252,4272],{"type":150,"tag":181,"props":4253,"children":4255},{"className":4254,"style":3862},[323],[4256],{"type":150,"tag":181,"props":4257,"children":4258},{"style":3866},[4259,4263],{"type":150,"tag":181,"props":4260,"children":4262},{"className":4261,"style":3871},[333],[],{"type":150,"tag":181,"props":4264,"children":4266},{"className":4265},[339,3876,3877,342],[4267],{"type":150,"tag":181,"props":4268,"children":4270},{"className":4269,"style":2249},[296,301,342],[4271],{"type":155,"value":1698},{"type":150,"tag":181,"props":4273,"children":4275},{"className":4274},[352],[4276],{"type":155,"value":355},{"type":150,"tag":181,"props":4278,"children":4280},{"className":4279},[318],[4281],{"type":150,"tag":181,"props":4282,"children":4284},{"className":4283,"style":3896},[323],[4285],{"type":150,"tag":181,"props":4286,"children":4287},{},[],{"type":150,"tag":181,"props":4289,"children":4291},{"className":4290},[371],[4292],{"type":155,"value":226},{"type":150,"tag":181,"props":4294,"children":4296},{"className":4295,"style":3909},[377],[],{"type":150,"tag":181,"props":4298,"children":4300},{"className":4299,"style":378},[377],[],{"type":150,"tag":181,"props":4302,"children":4304},{"className":4303},[296],[4305,4310],{"type":150,"tag":181,"props":4306,"children":4308},{"className":4307,"style":2151},[296,301],[4309],{"type":155,"value":1737},{"type":150,"tag":181,"props":4311,"children":4313},{"className":4312},[307],[4314],{"type":150,"tag":181,"props":4315,"children":4317},{"className":4316},[312,313],[4318,4377],{"type":150,"tag":181,"props":4319,"children":4321},{"className":4320},[318],[4322,4372],{"type":150,"tag":181,"props":4323,"children":4325},{"className":4324,"style":1276},[323],[4326,4342],{"type":150,"tag":181,"props":4327,"children":4328},{"style":3114},[4329,4333],{"type":150,"tag":181,"props":4330,"children":4332},{"className":4331,"style":334},[333],[],{"type":150,"tag":181,"props":4334,"children":4336},{"className":4335},[339,340,341,342],[4337],{"type":150,"tag":181,"props":4338,"children":4340},{"className":4339},[296,301,342],[4341],{"type":155,"value":558},{"type":150,"tag":181,"props":4343,"children":4344},{"style":1297},[4345,4349],{"type":150,"tag":181,"props":4346,"children":4348},{"className":4347,"style":334},[333],[],{"type":150,"tag":181,"props":4350,"children":4352},{"className":4351},[339,340,341,342],[4353],{"type":150,"tag":181,"props":4354,"children":4356},{"className":4355},[296,342],[4357,4362,4367],{"type":150,"tag":181,"props":4358,"children":4360},{"className":4359},[824,342],[4361],{"type":155,"value":792},{"type":150,"tag":181,"props":4363,"children":4365},{"className":4364},[296,301,342],[4366],{"type":155,"value":797},{"type":150,"tag":181,"props":4368,"children":4370},{"className":4369},[835,342],[4371],{"type":155,"value":802},{"type":150,"tag":181,"props":4373,"children":4375},{"className":4374},[352],[4376],{"type":155,"value":355},{"type":150,"tag":181,"props":4378,"children":4380},{"className":4379},[318],[4381],{"type":150,"tag":181,"props":4382,"children":4384},{"className":4383,"style":2782},[323],[4385],{"type":150,"tag":181,"props":4386,"children":4387},{},[],{"type":150,"tag":181,"props":4389,"children":4391},{"className":4390,"style":658},[377],[],{"type":150,"tag":181,"props":4393,"children":4395},{"className":4394},[663],[4396],{"type":155,"value":865},{"type":150,"tag":181,"props":4398,"children":4400},{"className":4399,"style":658},[377],[],{"type":150,"tag":181,"props":4402,"children":4404},{"className":4403},[285],[4405,4410,4467,4555,4559,4564],{"type":150,"tag":181,"props":4406,"children":4409},{"className":4407,"style":4408},[290],"height:1.185em;vertical-align:-0.247em;",[],{"type":150,"tag":181,"props":4411,"children":4413},{"className":4412},[296],[4414,4419],{"type":150,"tag":181,"props":4415,"children":4417},{"className":4416},[296,301],[4418],{"type":155,"value":213},{"type":150,"tag":181,"props":4420,"children":4422},{"className":4421},[307],[4423],{"type":150,"tag":181,"props":4424,"children":4426},{"className":4425},[312,313],[4427,4456],{"type":150,"tag":181,"props":4428,"children":4430},{"className":4429},[318],[4431,4451],{"type":150,"tag":181,"props":4432,"children":4434},{"className":4433,"style":621},[323],[4435],{"type":150,"tag":181,"props":4436,"children":4437},{"style":328},[4438,4442],{"type":150,"tag":181,"props":4439,"children":4441},{"className":4440,"style":334},[333],[],{"type":150,"tag":181,"props":4443,"children":4445},{"className":4444},[339,340,341,342],[4446],{"type":150,"tag":181,"props":4447,"children":4449},{"className":4448},[296,301,342],[4450],{"type":155,"value":558},{"type":150,"tag":181,"props":4452,"children":4454},{"className":4453},[352],[4455],{"type":155,"value":355},{"type":150,"tag":181,"props":4457,"children":4459},{"className":4458},[318],[4460],{"type":150,"tag":181,"props":4461,"children":4463},{"className":4462,"style":363},[323],[4464],{"type":150,"tag":181,"props":4465,"children":4466},{},[],{"type":150,"tag":181,"props":4468,"children":4470},{"className":4469},[296],[4471,4476],{"type":150,"tag":181,"props":4472,"children":4474},{"className":4473,"style":3691},[296,301],[4475],{"type":155,"value":3265},{"type":150,"tag":181,"props":4477,"children":4479},{"className":4478},[307],[4480],{"type":150,"tag":181,"props":4481,"children":4483},{"className":4482},[312,313],[4484,4543],{"type":150,"tag":181,"props":4485,"children":4487},{"className":4486},[318],[4488,4538],{"type":150,"tag":181,"props":4489,"children":4491},{"className":4490,"style":3709},[323],[4492,4508],{"type":150,"tag":181,"props":4493,"children":4494},{"style":3713},[4495,4499],{"type":150,"tag":181,"props":4496,"children":4498},{"className":4497,"style":334},[333],[],{"type":150,"tag":181,"props":4500,"children":4502},{"className":4501},[339,340,341,342],[4503],{"type":150,"tag":181,"props":4504,"children":4506},{"className":4505,"style":2151},[296,301,342],[4507],{"type":155,"value":1737},{"type":150,"tag":181,"props":4509,"children":4510},{"style":3730},[4511,4515],{"type":150,"tag":181,"props":4512,"children":4514},{"className":4513,"style":334},[333],[],{"type":150,"tag":181,"props":4516,"children":4518},{"className":4517},[339,340,341,342],[4519],{"type":150,"tag":181,"props":4520,"children":4522},{"className":4521},[296,342],[4523,4528,4533],{"type":150,"tag":181,"props":4524,"children":4526},{"className":4525},[824,342],[4527],{"type":155,"value":792},{"type":150,"tag":181,"props":4529,"children":4531},{"className":4530},[296,301,342],[4532],{"type":155,"value":797},{"type":150,"tag":181,"props":4534,"children":4536},{"className":4535},[835,342],[4537],{"type":155,"value":802},{"type":150,"tag":181,"props":4539,"children":4541},{"className":4540},[352],[4542],{"type":155,"value":355},{"type":150,"tag":181,"props":4544,"children":4546},{"className":4545},[318],[4547],{"type":150,"tag":181,"props":4548,"children":4551},{"className":4549,"style":4550},[323],"height:0.247em;",[4552],{"type":150,"tag":181,"props":4553,"children":4554},{},[],{"type":150,"tag":181,"props":4556,"children":4558},{"className":4557,"style":658},[377],[],{"type":150,"tag":181,"props":4560,"children":4562},{"className":4561},[663],[4563],{"type":155,"value":563},{"type":150,"tag":181,"props":4565,"children":4567},{"className":4566,"style":658},[377],[],{"type":150,"tag":181,"props":4569,"children":4571},{"className":4570},[285],[4572,4576],{"type":150,"tag":181,"props":4573,"children":4575},{"className":4574,"style":3821},[290],[],{"type":150,"tag":181,"props":4577,"children":4579},{"className":4578},[296],[4580,4585],{"type":150,"tag":181,"props":4581,"children":4583},{"className":4582},[296,686],[4584],{"type":155,"value":573},{"type":150,"tag":181,"props":4586,"children":4588},{"className":4587},[307],[4589],{"type":150,"tag":181,"props":4590,"children":4592},{"className":4591},[312],[4593],{"type":150,"tag":181,"props":4594,"children":4596},{"className":4595},[318],[4597],{"type":150,"tag":181,"props":4598,"children":4600},{"className":4599,"style":3821},[323],[4601],{"type":150,"tag":181,"props":4602,"children":4603},{"style":3730},[4604,4608],{"type":150,"tag":181,"props":4605,"children":4607},{"className":4606,"style":334},[333],[],{"type":150,"tag":181,"props":4609,"children":4611},{"className":4610},[339,340,341,342],[4612],{"type":150,"tag":181,"props":4613,"children":4615},{"className":4614},[296,342],[4616],{"type":150,"tag":181,"props":4617,"children":4619},{"className":4618},[296,342],[4620,4625],{"type":150,"tag":181,"props":4621,"children":4623},{"className":4622},[296,301,342],[4624],{"type":155,"value":578},{"type":150,"tag":181,"props":4626,"children":4628},{"className":4627},[307],[4629],{"type":150,"tag":181,"props":4630,"children":4632},{"className":4631},[312,313],[4633,4664],{"type":150,"tag":181,"props":4634,"children":4636},{"className":4635},[318],[4637,4659],{"type":150,"tag":181,"props":4638,"children":4641},{"className":4639,"style":4640},[323],"height:0.1645em;",[4642],{"type":150,"tag":181,"props":4643,"children":4645},{"style":4644},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[4646,4650],{"type":150,"tag":181,"props":4647,"children":4649},{"className":4648,"style":3871},[333],[],{"type":150,"tag":181,"props":4651,"children":4653},{"className":4652},[339,3876,3877,342],[4654],{"type":150,"tag":181,"props":4655,"children":4657},{"className":4656,"style":2151},[296,301,342],[4658],{"type":155,"value":1737},{"type":150,"tag":181,"props":4660,"children":4662},{"className":4661},[352],[4663],{"type":155,"value":355},{"type":150,"tag":181,"props":4665,"children":4667},{"className":4666},[318],[4668],{"type":150,"tag":181,"props":4669,"children":4672},{"className":4670,"style":4671},[323],"height:0.143em;",[4673],{"type":150,"tag":181,"props":4674,"children":4675},{},[],{"type":150,"tag":4677,"props":4678,"children":4679},"note",{},[4680],{"type":150,"tag":151,"props":4681,"children":4682},{},[4683,4685,4935,4937,5019,5021,5184,5186,5349],{"type":155,"value":4684},"（省略了缩放因子。常见设置是 ",{"type":150,"tag":181,"props":4686,"children":4688},{"className":4687},[184],[4689,4750],{"type":150,"tag":181,"props":4690,"children":4692},{"className":4691},[189],[4693],{"type":150,"tag":192,"props":4694,"children":4695},{"xmlns":194},[4696],{"type":150,"tag":197,"props":4697,"children":4698},{},[4699,4745],{"type":150,"tag":201,"props":4700,"children":4701},{},[4702,4713,4717,4728,4732,4736,4741],{"type":150,"tag":205,"props":4703,"children":4704},{},[4705,4709],{"type":150,"tag":209,"props":4706,"children":4707},{},[4708],{"type":155,"value":578},{"type":150,"tag":209,"props":4710,"children":4711},{},[4712],{"type":155,"value":1698},{"type":150,"tag":221,"props":4714,"children":4715},{},[4716],{"type":155,"value":865},{"type":150,"tag":205,"props":4718,"children":4719},{},[4720,4724],{"type":150,"tag":209,"props":4721,"children":4722},{},[4723],{"type":155,"value":578},{"type":150,"tag":209,"props":4725,"children":4726},{},[4727],{"type":155,"value":1737},{"type":150,"tag":221,"props":4729,"children":4730},{},[4731],{"type":155,"value":865},{"type":150,"tag":209,"props":4733,"children":4734},{},[4735],{"type":155,"value":578},{"type":150,"tag":209,"props":4737,"children":4738},{"mathvariant":1618},[4739],{"type":155,"value":4740},"\u002F",{"type":150,"tag":209,"props":4742,"children":4743},{},[4744],{"type":155,"value":745},{"type":150,"tag":271,"props":4746,"children":4747},{"encoding":273},[4748],{"type":155,"value":4749},"d_k = d_v = d\u002Fh",{"type":150,"tag":181,"props":4751,"children":4753},{"className":4752,"ariaHidden":223},[280],[4754,4833,4912],{"type":150,"tag":181,"props":4755,"children":4757},{"className":4756},[285],[4758,4763,4820,4824,4829],{"type":150,"tag":181,"props":4759,"children":4762},{"className":4760,"style":4761},[290],"height:0.8444em;vertical-align:-0.15em;",[],{"type":150,"tag":181,"props":4764,"children":4766},{"className":4765},[296],[4767,4772],{"type":150,"tag":181,"props":4768,"children":4770},{"className":4769},[296,301],[4771],{"type":155,"value":578},{"type":150,"tag":181,"props":4773,"children":4775},{"className":4774},[307],[4776],{"type":150,"tag":181,"props":4777,"children":4779},{"className":4778},[312,313],[4780,4809],{"type":150,"tag":181,"props":4781,"children":4783},{"className":4782},[318],[4784,4804],{"type":150,"tag":181,"props":4785,"children":4787},{"className":4786,"style":493},[323],[4788],{"type":150,"tag":181,"props":4789,"children":4790},{"style":328},[4791,4795],{"type":150,"tag":181,"props":4792,"children":4794},{"className":4793,"style":334},[333],[],{"type":150,"tag":181,"props":4796,"children":4798},{"className":4797},[339,340,341,342],[4799],{"type":150,"tag":181,"props":4800,"children":4802},{"className":4801,"style":2249},[296,301,342],[4803],{"type":155,"value":1698},{"type":150,"tag":181,"props":4805,"children":4807},{"className":4806},[352],[4808],{"type":155,"value":355},{"type":150,"tag":181,"props":4810,"children":4812},{"className":4811},[318],[4813],{"type":150,"tag":181,"props":4814,"children":4816},{"className":4815,"style":363},[323],[4817],{"type":150,"tag":181,"props":4818,"children":4819},{},[],{"type":150,"tag":181,"props":4821,"children":4823},{"className":4822,"style":658},[377],[],{"type":150,"tag":181,"props":4825,"children":4827},{"className":4826},[663],[4828],{"type":155,"value":865},{"type":150,"tag":181,"props":4830,"children":4832},{"className":4831,"style":658},[377],[],{"type":150,"tag":181,"props":4834,"children":4836},{"className":4835},[285],[4837,4841,4899,4903,4908],{"type":150,"tag":181,"props":4838,"children":4840},{"className":4839,"style":4761},[290],[],{"type":150,"tag":181,"props":4842,"children":4844},{"className":4843},[296],[4845,4850],{"type":150,"tag":181,"props":4846,"children":4848},{"className":4847},[296,301],[4849],{"type":155,"value":578},{"type":150,"tag":181,"props":4851,"children":4853},{"className":4852},[307],[4854],{"type":150,"tag":181,"props":4855,"children":4857},{"className":4856},[312,313],[4858,4888],{"type":150,"tag":181,"props":4859,"children":4861},{"className":4860},[318],[4862,4883],{"type":150,"tag":181,"props":4863,"children":4866},{"className":4864,"style":4865},[323],"height:0.1514em;",[4867],{"type":150,"tag":181,"props":4868,"children":4869},{"style":328},[4870,4874],{"type":150,"tag":181,"props":4871,"children":4873},{"className":4872,"style":334},[333],[],{"type":150,"tag":181,"props":4875,"children":4877},{"className":4876},[339,340,341,342],[4878],{"type":150,"tag":181,"props":4879,"children":4881},{"className":4880,"style":2151},[296,301,342],[4882],{"type":155,"value":1737},{"type":150,"tag":181,"props":4884,"children":4886},{"className":4885},[352],[4887],{"type":155,"value":355},{"type":150,"tag":181,"props":4889,"children":4891},{"className":4890},[318],[4892],{"type":150,"tag":181,"props":4893,"children":4895},{"className":4894,"style":363},[323],[4896],{"type":150,"tag":181,"props":4897,"children":4898},{},[],{"type":150,"tag":181,"props":4900,"children":4902},{"className":4901,"style":658},[377],[],{"type":150,"tag":181,"props":4904,"children":4906},{"className":4905},[663],[4907],{"type":155,"value":865},{"type":150,"tag":181,"props":4909,"children":4911},{"className":4910,"style":658},[377],[],{"type":150,"tag":181,"props":4913,"children":4915},{"className":4914},[285],[4916,4920,4925,4930],{"type":150,"tag":181,"props":4917,"children":4919},{"className":4918,"style":819},[290],[],{"type":150,"tag":181,"props":4921,"children":4923},{"className":4922},[296,301],[4924],{"type":155,"value":578},{"type":150,"tag":181,"props":4926,"children":4928},{"className":4927},[296],[4929],{"type":155,"value":4740},{"type":150,"tag":181,"props":4931,"children":4933},{"className":4932},[296,301],[4934],{"type":155,"value":745},{"type":155,"value":4936},"。）关键在因果性。自回归生成时，新预测出来的第 ",{"type":150,"tag":181,"props":4938,"children":4940},{"className":4939},[184],[4941,4972],{"type":150,"tag":181,"props":4942,"children":4944},{"className":4943},[189],[4945],{"type":150,"tag":192,"props":4946,"children":4947},{"xmlns":194},[4948],{"type":150,"tag":197,"props":4949,"children":4950},{},[4951,4967],{"type":150,"tag":201,"props":4952,"children":4953},{},[4954,4958,4963],{"type":150,"tag":209,"props":4955,"children":4956},{},[4957],{"type":155,"value":1035},{"type":150,"tag":221,"props":4959,"children":4960},{},[4961],{"type":155,"value":4962},"+",{"type":150,"tag":215,"props":4964,"children":4965},{},[4966],{"type":155,"value":219},{"type":150,"tag":271,"props":4968,"children":4969},{"encoding":273},[4970],{"type":155,"value":4971},"t+1",{"type":150,"tag":181,"props":4973,"children":4975},{"className":4974,"ariaHidden":223},[280],[4976,5005],{"type":150,"tag":181,"props":4977,"children":4979},{"className":4978},[285],[4980,4985,4990,4995,5001],{"type":150,"tag":181,"props":4981,"children":4984},{"className":4982,"style":4983},[290],"height:0.6984em;vertical-align:-0.0833em;",[],{"type":150,"tag":181,"props":4986,"children":4988},{"className":4987},[296,301],[4989],{"type":155,"value":1035},{"type":150,"tag":181,"props":4991,"children":4994},{"className":4992,"style":4993},[377],"margin-right:0.2222em;",[],{"type":150,"tag":181,"props":4996,"children":4999},{"className":4997},[4998],"mbin",[5000],{"type":155,"value":4962},{"type":150,"tag":181,"props":5002,"children":5004},{"className":5003,"style":4993},[377],[],{"type":150,"tag":181,"props":5006,"children":5008},{"className":5007},[285],[5009,5014],{"type":150,"tag":181,"props":5010,"children":5013},{"className":5011,"style":5012},[290],"height:0.6444em;",[],{"type":150,"tag":181,"props":5015,"children":5017},{"className":5016},[296],[5018],{"type":155,"value":219},{"type":155,"value":5020}," 个 token 不会影响已经算好的 ",{"type":150,"tag":181,"props":5022,"children":5024},{"className":5023},[184],[5025,5076],{"type":150,"tag":181,"props":5026,"children":5028},{"className":5027},[189],[5029],{"type":150,"tag":192,"props":5030,"children":5031},{"xmlns":194},[5032],{"type":150,"tag":197,"props":5033,"children":5034},{},[5035,5071],{"type":150,"tag":201,"props":5036,"children":5037},{},[5038],{"type":150,"tag":1046,"props":5039,"children":5040},{},[5041,5045,5056],{"type":150,"tag":209,"props":5042,"children":5043},{},[5044],{"type":155,"value":1698},{"type":150,"tag":201,"props":5046,"children":5047},{},[5048,5052],{"type":150,"tag":221,"props":5049,"children":5050},{},[5051],{"type":155,"value":1706},{"type":150,"tag":209,"props":5053,"children":5054},{},[5055],{"type":155,"value":1035},{"type":150,"tag":201,"props":5057,"children":5058},{},[5059,5063,5067],{"type":150,"tag":221,"props":5060,"children":5061},{"stretchy":789},[5062],{"type":155,"value":792},{"type":150,"tag":209,"props":5064,"children":5065},{},[5066],{"type":155,"value":797},{"type":150,"tag":221,"props":5068,"children":5069},{"stretchy":789},[5070],{"type":155,"value":802},{"type":150,"tag":271,"props":5072,"children":5073},{"encoding":273},[5074],{"type":155,"value":5075},"k^{(s)}_{\\le t}",{"type":150,"tag":181,"props":5077,"children":5079},{"className":5078,"ariaHidden":223},[280],[5080],{"type":150,"tag":181,"props":5081,"children":5083},{"className":5082},[285],[5084,5088],{"type":150,"tag":181,"props":5085,"children":5087},{"className":5086,"style":2127},[290],[],{"type":150,"tag":181,"props":5089,"children":5091},{"className":5090},[296],[5092,5097],{"type":150,"tag":181,"props":5093,"children":5095},{"className":5094,"style":2249},[296,301],[5096],{"type":155,"value":1698},{"type":150,"tag":181,"props":5098,"children":5100},{"className":5099},[307],[5101],{"type":150,"tag":181,"props":5102,"children":5104},{"className":5103},[312,313],[5105,5173],{"type":150,"tag":181,"props":5106,"children":5108},{"className":5107},[318],[5109,5168],{"type":150,"tag":181,"props":5110,"children":5112},{"className":5111,"style":1276},[323],[5113,5138],{"type":150,"tag":181,"props":5114,"children":5115},{"style":2270},[5116,5120],{"type":150,"tag":181,"props":5117,"children":5119},{"className":5118,"style":334},[333],[],{"type":150,"tag":181,"props":5121,"children":5123},{"className":5122},[339,340,341,342],[5124],{"type":150,"tag":181,"props":5125,"children":5127},{"className":5126},[296,342],[5128,5133],{"type":150,"tag":181,"props":5129,"children":5131},{"className":5130},[663,342],[5132],{"type":155,"value":1706},{"type":150,"tag":181,"props":5134,"children":5136},{"className":5135},[296,301,342],[5137],{"type":155,"value":1035},{"type":150,"tag":181,"props":5139,"children":5140},{"style":1297},[5141,5145],{"type":150,"tag":181,"props":5142,"children":5144},{"className":5143,"style":334},[333],[],{"type":150,"tag":181,"props":5146,"children":5148},{"className":5147},[339,340,341,342],[5149],{"type":150,"tag":181,"props":5150,"children":5152},{"className":5151},[296,342],[5153,5158,5163],{"type":150,"tag":181,"props":5154,"children":5156},{"className":5155},[824,342],[5157],{"type":155,"value":792},{"type":150,"tag":181,"props":5159,"children":5161},{"className":5160},[296,301,342],[5162],{"type":155,"value":797},{"type":150,"tag":181,"props":5164,"children":5166},{"className":5165},[835,342],[5167],{"type":155,"value":802},{"type":150,"tag":181,"props":5169,"children":5171},{"className":5170},[352],[5172],{"type":155,"value":355},{"type":150,"tag":181,"props":5174,"children":5176},{"className":5175},[318],[5177],{"type":150,"tag":181,"props":5178,"children":5180},{"className":5179,"style":2336},[323],[5181],{"type":150,"tag":181,"props":5182,"children":5183},{},[],{"type":155,"value":5185}," 和 ",{"type":150,"tag":181,"props":5187,"children":5189},{"className":5188},[184],[5190,5241],{"type":150,"tag":181,"props":5191,"children":5193},{"className":5192},[189],[5194],{"type":150,"tag":192,"props":5195,"children":5196},{"xmlns":194},[5197],{"type":150,"tag":197,"props":5198,"children":5199},{},[5200,5236],{"type":150,"tag":201,"props":5201,"children":5202},{},[5203],{"type":150,"tag":1046,"props":5204,"children":5205},{},[5206,5210,5221],{"type":150,"tag":209,"props":5207,"children":5208},{},[5209],{"type":155,"value":1737},{"type":150,"tag":201,"props":5211,"children":5212},{},[5213,5217],{"type":150,"tag":221,"props":5214,"children":5215},{},[5216],{"type":155,"value":1706},{"type":150,"tag":209,"props":5218,"children":5219},{},[5220],{"type":155,"value":1035},{"type":150,"tag":201,"props":5222,"children":5223},{},[5224,5228,5232],{"type":150,"tag":221,"props":5225,"children":5226},{"stretchy":789},[5227],{"type":155,"value":792},{"type":150,"tag":209,"props":5229,"children":5230},{},[5231],{"type":155,"value":797},{"type":150,"tag":221,"props":5233,"children":5234},{"stretchy":789},[5235],{"type":155,"value":802},{"type":150,"tag":271,"props":5237,"children":5238},{"encoding":273},[5239],{"type":155,"value":5240},"v^{(s)}_{\\le t}",{"type":150,"tag":181,"props":5242,"children":5244},{"className":5243,"ariaHidden":223},[280],[5245],{"type":150,"tag":181,"props":5246,"children":5248},{"className":5247},[285],[5249,5253],{"type":150,"tag":181,"props":5250,"children":5252},{"className":5251,"style":2127},[290],[],{"type":150,"tag":181,"props":5254,"children":5256},{"className":5255},[296],[5257,5262],{"type":150,"tag":181,"props":5258,"children":5260},{"className":5259,"style":2151},[296,301],[5261],{"type":155,"value":1737},{"type":150,"tag":181,"props":5263,"children":5265},{"className":5264},[307],[5266],{"type":150,"tag":181,"props":5267,"children":5269},{"className":5268},[312,313],[5270,5338],{"type":150,"tag":181,"props":5271,"children":5273},{"className":5272},[318],[5274,5333],{"type":150,"tag":181,"props":5275,"children":5277},{"className":5276,"style":1276},[323],[5278,5303],{"type":150,"tag":181,"props":5279,"children":5280},{"style":2377},[5281,5285],{"type":150,"tag":181,"props":5282,"children":5284},{"className":5283,"style":334},[333],[],{"type":150,"tag":181,"props":5286,"children":5288},{"className":5287},[339,340,341,342],[5289],{"type":150,"tag":181,"props":5290,"children":5292},{"className":5291},[296,342],[5293,5298],{"type":150,"tag":181,"props":5294,"children":5296},{"className":5295},[663,342],[5297],{"type":155,"value":1706},{"type":150,"tag":181,"props":5299,"children":5301},{"className":5300},[296,301,342],[5302],{"type":155,"value":1035},{"type":150,"tag":181,"props":5304,"children":5305},{"style":1297},[5306,5310],{"type":150,"tag":181,"props":5307,"children":5309},{"className":5308,"style":334},[333],[],{"type":150,"tag":181,"props":5311,"children":5313},{"className":5312},[339,340,341,342],[5314],{"type":150,"tag":181,"props":5315,"children":5317},{"className":5316},[296,342],[5318,5323,5328],{"type":150,"tag":181,"props":5319,"children":5321},{"className":5320},[824,342],[5322],{"type":155,"value":792},{"type":150,"tag":181,"props":5324,"children":5326},{"className":5325},[296,301,342],[5327],{"type":155,"value":797},{"type":150,"tag":181,"props":5329,"children":5331},{"className":5330},[835,342],[5332],{"type":155,"value":802},{"type":150,"tag":181,"props":5334,"children":5336},{"className":5335},[352],[5337],{"type":155,"value":355},{"type":150,"tag":181,"props":5339,"children":5341},{"className":5340},[318],[5342],{"type":150,"tag":181,"props":5343,"children":5345},{"className":5344,"style":2336},[323],[5346],{"type":150,"tag":181,"props":5347,"children":5348},{},[],{"type":155,"value":5350},"，所以这部分可以缓存下来，避免重复计算。这就是 KV Cache。它的大小约等于：",{"type":150,"tag":181,"props":5352,"children":5354},{"className":5353},[1003],[5355],{"type":150,"tag":181,"props":5356,"children":5358},{"className":5357},[184],[5359,5432],{"type":150,"tag":181,"props":5360,"children":5362},{"className":5361},[189],[5363],{"type":150,"tag":192,"props":5364,"children":5365},{"xmlns":194,"display":1015},[5366],{"type":150,"tag":197,"props":5367,"children":5368},{},[5369,5427],{"type":150,"tag":201,"props":5370,"children":5371},{},[5372,5376,5381,5386,5390,5394,5398,5409,5413,5418,5422],{"type":150,"tag":215,"props":5373,"children":5374},{},[5375],{"type":155,"value":238},{"type":150,"tag":221,"props":5377,"children":5378},{},[5379],{"type":155,"value":5380},"×",{"type":150,"tag":209,"props":5382,"children":5383},{},[5384],{"type":155,"value":5385},"L",{"type":150,"tag":221,"props":5387,"children":5388},{},[5389],{"type":155,"value":5380},{"type":150,"tag":209,"props":5391,"children":5392},{},[5393],{"type":155,"value":745},{"type":150,"tag":221,"props":5395,"children":5396},{},[5397],{"type":155,"value":5380},{"type":150,"tag":205,"props":5399,"children":5400},{},[5401,5405],{"type":150,"tag":209,"props":5402,"children":5403},{},[5404],{"type":155,"value":578},{"type":150,"tag":209,"props":5406,"children":5407},{},[5408],{"type":155,"value":1698},{"type":150,"tag":221,"props":5410,"children":5411},{},[5412],{"type":155,"value":5380},{"type":150,"tag":209,"props":5414,"children":5415},{},[5416],{"type":155,"value":5417},"N",{"type":150,"tag":221,"props":5419,"children":5420},{},[5421],{"type":155,"value":5380},{"type":150,"tag":249,"props":5423,"children":5424},{},[5425],{"type":155,"value":5426},"bytes",{"type":150,"tag":271,"props":5428,"children":5429},{"encoding":273},[5430],{"type":155,"value":5431},"2 \\times L \\times h \\times d_k \\times N \\times \\text{bytes}",{"type":150,"tag":181,"props":5433,"children":5435},{"className":5434,"ariaHidden":223},[280],[5436,5463,5490,5517,5595,5622],{"type":150,"tag":181,"props":5437,"children":5439},{"className":5438},[285],[5440,5445,5450,5454,5459],{"type":150,"tag":181,"props":5441,"children":5444},{"className":5442,"style":5443},[290],"height:0.7278em;vertical-align:-0.0833em;",[],{"type":150,"tag":181,"props":5446,"children":5448},{"className":5447},[296],[5449],{"type":155,"value":238},{"type":150,"tag":181,"props":5451,"children":5453},{"className":5452,"style":4993},[377],[],{"type":150,"tag":181,"props":5455,"children":5457},{"className":5456},[4998],[5458],{"type":155,"value":5380},{"type":150,"tag":181,"props":5460,"children":5462},{"className":5461,"style":4993},[377],[],{"type":150,"tag":181,"props":5464,"children":5466},{"className":5465},[285],[5467,5472,5477,5481,5486],{"type":150,"tag":181,"props":5468,"children":5471},{"className":5469,"style":5470},[290],"height:0.7667em;vertical-align:-0.0833em;",[],{"type":150,"tag":181,"props":5473,"children":5475},{"className":5474},[296,301],[5476],{"type":155,"value":5385},{"type":150,"tag":181,"props":5478,"children":5480},{"className":5479,"style":4993},[377],[],{"type":150,"tag":181,"props":5482,"children":5484},{"className":5483},[4998],[5485],{"type":155,"value":5380},{"type":150,"tag":181,"props":5487,"children":5489},{"className":5488,"style":4993},[377],[],{"type":150,"tag":181,"props":5491,"children":5493},{"className":5492},[285],[5494,5499,5504,5508,5513],{"type":150,"tag":181,"props":5495,"children":5498},{"className":5496,"style":5497},[290],"height:0.7778em;vertical-align:-0.0833em;",[],{"type":150,"tag":181,"props":5500,"children":5502},{"className":5501},[296,301],[5503],{"type":155,"value":745},{"type":150,"tag":181,"props":5505,"children":5507},{"className":5506,"style":4993},[377],[],{"type":150,"tag":181,"props":5509,"children":5511},{"className":5510},[4998],[5512],{"type":155,"value":5380},{"type":150,"tag":181,"props":5514,"children":5516},{"className":5515,"style":4993},[377],[],{"type":150,"tag":181,"props":5518,"children":5520},{"className":5519},[285],[5521,5525,5582,5586,5591],{"type":150,"tag":181,"props":5522,"children":5524},{"className":5523,"style":4761},[290],[],{"type":150,"tag":181,"props":5526,"children":5528},{"className":5527},[296],[5529,5534],{"type":150,"tag":181,"props":5530,"children":5532},{"className":5531},[296,301],[5533],{"type":155,"value":578},{"type":150,"tag":181,"props":5535,"children":5537},{"className":5536},[307],[5538],{"type":150,"tag":181,"props":5539,"children":5541},{"className":5540},[312,313],[5542,5571],{"type":150,"tag":181,"props":5543,"children":5545},{"className":5544},[318],[5546,5566],{"type":150,"tag":181,"props":5547,"children":5549},{"className":5548,"style":493},[323],[5550],{"type":150,"tag":181,"props":5551,"children":5552},{"style":328},[5553,5557],{"type":150,"tag":181,"props":5554,"children":5556},{"className":5555,"style":334},[333],[],{"type":150,"tag":181,"props":5558,"children":5560},{"className":5559},[339,340,341,342],[5561],{"type":150,"tag":181,"props":5562,"children":5564},{"className":5563,"style":2249},[296,301,342],[5565],{"type":155,"value":1698},{"type":150,"tag":181,"props":5567,"children":5569},{"className":5568},[352],[5570],{"type":155,"value":355},{"type":150,"tag":181,"props":5572,"children":5574},{"className":5573},[318],[5575],{"type":150,"tag":181,"props":5576,"children":5578},{"className":5577,"style":363},[323],[5579],{"type":150,"tag":181,"props":5580,"children":5581},{},[],{"type":150,"tag":181,"props":5583,"children":5585},{"className":5584,"style":4993},[377],[],{"type":150,"tag":181,"props":5587,"children":5589},{"className":5588},[4998],[5590],{"type":155,"value":5380},{"type":150,"tag":181,"props":5592,"children":5594},{"className":5593,"style":4993},[377],[],{"type":150,"tag":181,"props":5596,"children":5598},{"className":5597},[285],[5599,5603,5609,5613,5618],{"type":150,"tag":181,"props":5600,"children":5602},{"className":5601,"style":5470},[290],[],{"type":150,"tag":181,"props":5604,"children":5607},{"className":5605,"style":5606},[296,301],"margin-right:0.109em;",[5608],{"type":155,"value":5417},{"type":150,"tag":181,"props":5610,"children":5612},{"className":5611,"style":4993},[377],[],{"type":150,"tag":181,"props":5614,"children":5616},{"className":5615},[4998],[5617],{"type":155,"value":5380},{"type":150,"tag":181,"props":5619,"children":5621},{"className":5620,"style":4993},[377],[],{"type":150,"tag":181,"props":5623,"children":5625},{"className":5624},[285],[5626,5630],{"type":150,"tag":181,"props":5627,"children":5629},{"className":5628,"style":941},[290],[],{"type":150,"tag":181,"props":5631,"children":5633},{"className":5632},[296,155],[5634],{"type":150,"tag":181,"props":5635,"children":5637},{"className":5636},[296],[5638],{"type":155,"value":5426},{"type":150,"tag":151,"props":5640,"children":5641},{},[5642,5644,5687,5689,5731,5733,5775,5777,5819],{"type":155,"value":5643},"其中 ",{"type":150,"tag":181,"props":5645,"children":5647},{"className":5646},[184],[5648,5669],{"type":150,"tag":181,"props":5649,"children":5651},{"className":5650},[189],[5652],{"type":150,"tag":192,"props":5653,"children":5654},{"xmlns":194},[5655],{"type":150,"tag":197,"props":5656,"children":5657},{},[5658,5665],{"type":150,"tag":201,"props":5659,"children":5660},{},[5661],{"type":150,"tag":209,"props":5662,"children":5663},{},[5664],{"type":155,"value":5385},{"type":150,"tag":271,"props":5666,"children":5667},{"encoding":273},[5668],{"type":155,"value":5385},{"type":150,"tag":181,"props":5670,"children":5672},{"className":5671,"ariaHidden":223},[280],[5673],{"type":150,"tag":181,"props":5674,"children":5676},{"className":5675},[285],[5677,5682],{"type":150,"tag":181,"props":5678,"children":5681},{"className":5679,"style":5680},[290],"height:0.6833em;",[],{"type":150,"tag":181,"props":5683,"children":5685},{"className":5684},[296,301],[5686],{"type":155,"value":5385},{"type":155,"value":5688}," 是层数，",{"type":150,"tag":181,"props":5690,"children":5692},{"className":5691},[184],[5693,5714],{"type":150,"tag":181,"props":5694,"children":5696},{"className":5695},[189],[5697],{"type":150,"tag":192,"props":5698,"children":5699},{"xmlns":194},[5700],{"type":150,"tag":197,"props":5701,"children":5702},{},[5703,5710],{"type":150,"tag":201,"props":5704,"children":5705},{},[5706],{"type":150,"tag":209,"props":5707,"children":5708},{},[5709],{"type":155,"value":5417},{"type":150,"tag":271,"props":5711,"children":5712},{"encoding":273},[5713],{"type":155,"value":5417},{"type":150,"tag":181,"props":5715,"children":5717},{"className":5716,"ariaHidden":223},[280],[5718],{"type":150,"tag":181,"props":5719,"children":5721},{"className":5720},[285],[5722,5726],{"type":150,"tag":181,"props":5723,"children":5725},{"className":5724,"style":5680},[290],[],{"type":150,"tag":181,"props":5727,"children":5729},{"className":5728,"style":5606},[296,301],[5730],{"type":155,"value":5417},{"type":155,"value":5732}," 是序列长度。注意这里面的 ",{"type":150,"tag":181,"props":5734,"children":5736},{"className":5735},[184],[5737,5758],{"type":150,"tag":181,"props":5738,"children":5740},{"className":5739},[189],[5741],{"type":150,"tag":192,"props":5742,"children":5743},{"xmlns":194},[5744],{"type":150,"tag":197,"props":5745,"children":5746},{},[5747,5754],{"type":150,"tag":201,"props":5748,"children":5749},{},[5750],{"type":150,"tag":209,"props":5751,"children":5752},{},[5753],{"type":155,"value":5417},{"type":150,"tag":271,"props":5755,"children":5756},{"encoding":273},[5757],{"type":155,"value":5417},{"type":150,"tag":181,"props":5759,"children":5761},{"className":5760,"ariaHidden":223},[280],[5762],{"type":150,"tag":181,"props":5763,"children":5765},{"className":5764},[285],[5766,5770],{"type":150,"tag":181,"props":5767,"children":5769},{"className":5768,"style":5680},[290],[],{"type":150,"tag":181,"props":5771,"children":5773},{"className":5772,"style":5606},[296,301],[5774],{"type":155,"value":5417},{"type":155,"value":5776},"。模型参数是个常数，选定模型就不再变了；但 KV Cache 随 ",{"type":150,"tag":181,"props":5778,"children":5780},{"className":5779},[184],[5781,5802],{"type":150,"tag":181,"props":5782,"children":5784},{"className":5783},[189],[5785],{"type":150,"tag":192,"props":5786,"children":5787},{"xmlns":194},[5788],{"type":150,"tag":197,"props":5789,"children":5790},{},[5791,5798],{"type":150,"tag":201,"props":5792,"children":5793},{},[5794],{"type":150,"tag":209,"props":5795,"children":5796},{},[5797],{"type":155,"value":5417},{"type":150,"tag":271,"props":5799,"children":5800},{"encoding":273},[5801],{"type":155,"value":5417},{"type":150,"tag":181,"props":5803,"children":5805},{"className":5804,"ariaHidden":223},[280],[5806],{"type":150,"tag":181,"props":5807,"children":5809},{"className":5808},[285],[5810,5814],{"type":150,"tag":181,"props":5811,"children":5813},{"className":5812,"style":5680},[290],[],{"type":150,"tag":181,"props":5815,"children":5817},{"className":5816,"style":5606},[296,301],[5818],{"type":155,"value":5417},{"type":155,"value":5820}," 线性增长。Context 足够长的时候，它会变成显存里的主导项，可能超出一张卡甚至一台机（8 张卡）的总显存。举个数。按 GPT-3 的配置算：96 层，96 个 attention head，每个 head 维度 128。单个 token 的 KV Cache 是",{"type":150,"tag":181,"props":5822,"children":5824},{"className":5823},[1003],[5825],{"type":150,"tag":181,"props":5826,"children":5828},{"className":5827},[184],[5829,5892],{"type":150,"tag":181,"props":5830,"children":5832},{"className":5831},[189],[5833],{"type":150,"tag":192,"props":5834,"children":5835},{"xmlns":194,"display":1015},[5836],{"type":150,"tag":197,"props":5837,"children":5838},{},[5839,5887],{"type":150,"tag":201,"props":5840,"children":5841},{},[5842,5846,5850,5855,5859,5863,5867,5872,5877,5882],{"type":150,"tag":215,"props":5843,"children":5844},{},[5845],{"type":155,"value":238},{"type":150,"tag":221,"props":5847,"children":5848},{},[5849],{"type":155,"value":5380},{"type":150,"tag":215,"props":5851,"children":5852},{},[5853],{"type":155,"value":5854},"96",{"type":150,"tag":221,"props":5856,"children":5857},{},[5858],{"type":155,"value":5380},{"type":150,"tag":215,"props":5860,"children":5861},{},[5862],{"type":155,"value":5854},{"type":150,"tag":221,"props":5864,"children":5865},{},[5866],{"type":155,"value":5380},{"type":150,"tag":215,"props":5868,"children":5869},{},[5870],{"type":155,"value":5871},"128",{"type":150,"tag":221,"props":5873,"children":5874},{},[5875],{"type":155,"value":5876},"≈",{"type":150,"tag":215,"props":5878,"children":5879},{},[5880],{"type":155,"value":5881},"2.36",{"type":150,"tag":249,"props":5883,"children":5884},{},[5885],{"type":155,"value":5886},"M",{"type":150,"tag":271,"props":5888,"children":5889},{"encoding":273},[5890],{"type":155,"value":5891},"2 \\times 96 \\times 96 \\times 128 \\approx 2.36\\text{M}",{"type":150,"tag":181,"props":5893,"children":5895},{"className":5894,"ariaHidden":223},[280],[5896,5922,5948,5974,6000],{"type":150,"tag":181,"props":5897,"children":5899},{"className":5898},[285],[5900,5904,5909,5913,5918],{"type":150,"tag":181,"props":5901,"children":5903},{"className":5902,"style":5443},[290],[],{"type":150,"tag":181,"props":5905,"children":5907},{"className":5906},[296],[5908],{"type":155,"value":238},{"type":150,"tag":181,"props":5910,"children":5912},{"className":5911,"style":4993},[377],[],{"type":150,"tag":181,"props":5914,"children":5916},{"className":5915},[4998],[5917],{"type":155,"value":5380},{"type":150,"tag":181,"props":5919,"children":5921},{"className":5920,"style":4993},[377],[],{"type":150,"tag":181,"props":5923,"children":5925},{"className":5924},[285],[5926,5930,5935,5939,5944],{"type":150,"tag":181,"props":5927,"children":5929},{"className":5928,"style":5443},[290],[],{"type":150,"tag":181,"props":5931,"children":5933},{"className":5932},[296],[5934],{"type":155,"value":5854},{"type":150,"tag":181,"props":5936,"children":5938},{"className":5937,"style":4993},[377],[],{"type":150,"tag":181,"props":5940,"children":5942},{"className":5941},[4998],[5943],{"type":155,"value":5380},{"type":150,"tag":181,"props":5945,"children":5947},{"className":5946,"style":4993},[377],[],{"type":150,"tag":181,"props":5949,"children":5951},{"className":5950},[285],[5952,5956,5961,5965,5970],{"type":150,"tag":181,"props":5953,"children":5955},{"className":5954,"style":5443},[290],[],{"type":150,"tag":181,"props":5957,"children":5959},{"className":5958},[296],[5960],{"type":155,"value":5854},{"type":150,"tag":181,"props":5962,"children":5964},{"className":5963,"style":4993},[377],[],{"type":150,"tag":181,"props":5966,"children":5968},{"className":5967},[4998],[5969],{"type":155,"value":5380},{"type":150,"tag":181,"props":5971,"children":5973},{"className":5972,"style":4993},[377],[],{"type":150,"tag":181,"props":5975,"children":5977},{"className":5976},[285],[5978,5982,5987,5991,5996],{"type":150,"tag":181,"props":5979,"children":5981},{"className":5980,"style":5012},[290],[],{"type":150,"tag":181,"props":5983,"children":5985},{"className":5984},[296],[5986],{"type":155,"value":5871},{"type":150,"tag":181,"props":5988,"children":5990},{"className":5989,"style":658},[377],[],{"type":150,"tag":181,"props":5992,"children":5994},{"className":5993},[663],[5995],{"type":155,"value":5876},{"type":150,"tag":181,"props":5997,"children":5999},{"className":5998,"style":658},[377],[],{"type":150,"tag":181,"props":6001,"children":6003},{"className":6002},[285],[6004,6008,6013],{"type":150,"tag":181,"props":6005,"children":6007},{"className":6006,"style":5680},[290],[],{"type":150,"tag":181,"props":6009,"children":6011},{"className":6010},[296],[6012],{"type":155,"value":5881},{"type":150,"tag":181,"props":6014,"children":6016},{"className":6015},[296,155],[6017],{"type":150,"tag":181,"props":6018,"children":6020},{"className":6019},[296],[6021],{"type":155,"value":5886},{"type":150,"tag":151,"props":6023,"children":6024},{},[6025],{"type":155,"value":6026},"个参数。按 2 字节精度存，就是 4.7 MB。一个 token 4.7 MB。128k context 就是 600 GB 量级。这个数字基本说明了为什么长上下文不是\"再多加个位置编码\"就能解决的。它是个显存和带宽问题。而 GPU 部署有个很不讲道理的原则：能一张卡放下的，就不要跨多张卡；能一台机放下的，就不要跨多台机。因为卡内通信带宽 > 卡间通信带宽 > 机间通信带宽。木桶效应，跨的设备越多，被最慢那段拖累得越狠。",{"type":150,"tag":151,"props":6028,"children":6029},{},[6030],{"type":155,"value":6031},"单卡 H100 内 SRAM 到 HBM 的带宽已经有 3 TB\u002Fs，对 short context 来说这依然是瓶颈，更不用说更慢的卡间和机间了。",{"type":150,"tag":169,"props":6033,"children":6035},{"id":6034},"prefill-与-decode",[6036],{"type":155,"value":6037},"Prefill 与 Decode",{"type":150,"tag":151,"props":6039,"children":6040},{},[6041,6043,6048,6050,6055,6056,6061],{"type":155,"value":6042},"LLM 推理分两个阶段，它们的成本结构不同。",{"type":150,"tag":158,"props":6044,"children":6045},{},[6046],{"type":155,"value":6047},"Prefill",{"type":155,"value":6049},"：把输入的所有 token 并行算一遍，同时把 KV Cache 建起来。所有 token 一起算，矩阵运算排得很满，算力基本能吃满。它是 ",{"type":150,"tag":158,"props":6051,"children":6052},{},[6053],{"type":155,"value":6054},"compute bound",{"type":155,"value":723},{"type":150,"tag":158,"props":6057,"children":6058},{},[6059],{"type":155,"value":6060},"Decode",{"type":155,"value":6062},"：之后逐个生成 token。这里有个容易漏掉的点。",{"type":150,"tag":151,"props":6064,"children":6065},{},[6066,6068,6073],{"type":155,"value":6067},"每生成一个 token，计算量很小，但要读的东西一点都不少：",{"type":150,"tag":158,"props":6069,"children":6070},{},[6071],{"type":155,"value":6072},"模型权重",{"type":155,"value":6074},"要整体过一遍，KV Cache 也要读一遍。",{"type":150,"tag":151,"props":6076,"children":6077},{},[6078,6080,6085],{"type":155,"value":6079},"不是读一部分权重，也不是只读 attention 那几层。每一步都要把参与计算的权重从 HBM 搬进 SM。算得少，搬得多。所以 decode 是 ",{"type":150,"tag":158,"props":6081,"children":6082},{},[6083],{"type":155,"value":6084},"memory bound",{"type":155,"value":6086},"。同一个模型，两个阶段的瓶颈完全不同。如果一个架构在两种形式下数学等价，那就可以 prefill 用一种算，decode 用另一种算，各取所需。MLA 就是这么干的。",{"type":150,"tag":151,"props":6088,"children":6089},{},[6090],{"type":155,"value":6091},"后面 V4.1 那一节会把这两个瓶颈各自的账算一遍。最后把账单写全：",{"type":150,"tag":181,"props":6093,"children":6095},{"className":6094},[1003],[6096],{"type":150,"tag":181,"props":6097,"children":6099},{"className":6098},[184],[6100,6179],{"type":150,"tag":181,"props":6101,"children":6103},{"className":6102},[189],[6104],{"type":150,"tag":192,"props":6105,"children":6106},{"xmlns":194,"display":1015},[6107],{"type":150,"tag":197,"props":6108,"children":6109},{},[6110,6174],{"type":150,"tag":201,"props":6111,"children":6112},{},[6113,6126,6130,6142,6146,6158,6162],{"type":150,"tag":205,"props":6114,"children":6115},{},[6116,6121],{"type":150,"tag":209,"props":6117,"children":6118},{},[6119],{"type":155,"value":6120},"C",{"type":150,"tag":249,"props":6122,"children":6123},{},[6124],{"type":155,"value":6125},"total",{"type":150,"tag":221,"props":6127,"children":6128},{},[6129],{"type":155,"value":865},{"type":150,"tag":205,"props":6131,"children":6132},{},[6133,6137],{"type":150,"tag":209,"props":6134,"children":6135},{},[6136],{"type":155,"value":6120},{"type":150,"tag":249,"props":6138,"children":6139},{},[6140],{"type":155,"value":6141},"attention",{"type":150,"tag":221,"props":6143,"children":6144},{},[6145],{"type":155,"value":4962},{"type":150,"tag":205,"props":6147,"children":6148},{},[6149,6153],{"type":150,"tag":209,"props":6150,"children":6151},{},[6152],{"type":155,"value":6120},{"type":150,"tag":249,"props":6154,"children":6155},{},[6156],{"type":155,"value":6157},"FFN\u002FMoE",{"type":150,"tag":221,"props":6159,"children":6160},{},[6161],{"type":155,"value":4962},{"type":150,"tag":205,"props":6163,"children":6164},{},[6165,6169],{"type":150,"tag":209,"props":6166,"children":6167},{},[6168],{"type":155,"value":6120},{"type":150,"tag":249,"props":6170,"children":6171},{},[6172],{"type":155,"value":6173},"memory movement",{"type":150,"tag":271,"props":6175,"children":6176},{"encoding":273},[6177],{"type":155,"value":6178},"C_{\\text{total}} = C_{\\text{attention}} + C_{\\text{FFN\u002FMoE}} + C_{\\text{memory movement}}",{"type":150,"tag":181,"props":6180,"children":6182},{"className":6181,"ariaHidden":223},[280],[6183,6272,6359,6448],{"type":150,"tag":181,"props":6184,"children":6186},{"className":6185},[285],[6187,6192,6259,6263,6268],{"type":150,"tag":181,"props":6188,"children":6191},{"className":6189,"style":6190},[290],"height:0.8333em;vertical-align:-0.15em;",[],{"type":150,"tag":181,"props":6193,"children":6195},{"className":6194},[296],[6196,6202],{"type":150,"tag":181,"props":6197,"children":6200},{"className":6198,"style":6199},[296,301],"margin-right:0.0715em;",[6201],{"type":155,"value":6120},{"type":150,"tag":181,"props":6203,"children":6205},{"className":6204},[307],[6206],{"type":150,"tag":181,"props":6207,"children":6209},{"className":6208},[312,313],[6210,6248],{"type":150,"tag":181,"props":6211,"children":6213},{"className":6212},[318],[6214,6243],{"type":150,"tag":181,"props":6215,"children":6217},{"className":6216,"style":493},[323],[6218],{"type":150,"tag":181,"props":6219,"children":6221},{"style":6220},"top:-2.55em;margin-left:-0.0715em;margin-right:0.05em;",[6222,6226],{"type":150,"tag":181,"props":6223,"children":6225},{"className":6224,"style":334},[333],[],{"type":150,"tag":181,"props":6227,"children":6229},{"className":6228},[339,340,341,342],[6230],{"type":150,"tag":181,"props":6231,"children":6233},{"className":6232},[296,342],[6234],{"type":150,"tag":181,"props":6235,"children":6237},{"className":6236},[296,155,342],[6238],{"type":150,"tag":181,"props":6239,"children":6241},{"className":6240},[296,342],[6242],{"type":155,"value":6125},{"type":150,"tag":181,"props":6244,"children":6246},{"className":6245},[352],[6247],{"type":155,"value":355},{"type":150,"tag":181,"props":6249,"children":6251},{"className":6250},[318],[6252],{"type":150,"tag":181,"props":6253,"children":6255},{"className":6254,"style":363},[323],[6256],{"type":150,"tag":181,"props":6257,"children":6258},{},[],{"type":150,"tag":181,"props":6260,"children":6262},{"className":6261,"style":658},[377],[],{"type":150,"tag":181,"props":6264,"children":6266},{"className":6265},[663],[6267],{"type":155,"value":865},{"type":150,"tag":181,"props":6269,"children":6271},{"className":6270,"style":658},[377],[],{"type":150,"tag":181,"props":6273,"children":6275},{"className":6274},[285],[6276,6280,6346,6350,6355],{"type":150,"tag":181,"props":6277,"children":6279},{"className":6278,"style":6190},[290],[],{"type":150,"tag":181,"props":6281,"children":6283},{"className":6282},[296],[6284,6289],{"type":150,"tag":181,"props":6285,"children":6287},{"className":6286,"style":6199},[296,301],[6288],{"type":155,"value":6120},{"type":150,"tag":181,"props":6290,"children":6292},{"className":6291},[307],[6293],{"type":150,"tag":181,"props":6294,"children":6296},{"className":6295},[312,313],[6297,6335],{"type":150,"tag":181,"props":6298,"children":6300},{"className":6299},[318],[6301,6330],{"type":150,"tag":181,"props":6302,"children":6305},{"className":6303,"style":6304},[323],"height:0.3175em;",[6306],{"type":150,"tag":181,"props":6307,"children":6308},{"style":6220},[6309,6313],{"type":150,"tag":181,"props":6310,"children":6312},{"className":6311,"style":334},[333],[],{"type":150,"tag":181,"props":6314,"children":6316},{"className":6315},[339,340,341,342],[6317],{"type":150,"tag":181,"props":6318,"children":6320},{"className":6319},[296,342],[6321],{"type":150,"tag":181,"props":6322,"children":6324},{"className":6323},[296,155,342],[6325],{"type":150,"tag":181,"props":6326,"children":6328},{"className":6327},[296,342],[6329],{"type":155,"value":6141},{"type":150,"tag":181,"props":6331,"children":6333},{"className":6332},[352],[6334],{"type":155,"value":355},{"type":150,"tag":181,"props":6336,"children":6338},{"className":6337},[318],[6339],{"type":150,"tag":181,"props":6340,"children":6342},{"className":6341,"style":363},[323],[6343],{"type":150,"tag":181,"props":6344,"children":6345},{},[],{"type":150,"tag":181,"props":6347,"children":6349},{"className":6348,"style":4993},[377],[],{"type":150,"tag":181,"props":6351,"children":6353},{"className":6352},[4998],[6354],{"type":155,"value":4962},{"type":150,"tag":181,"props":6356,"children":6358},{"className":6357,"style":4993},[377],[],{"type":150,"tag":181,"props":6360,"children":6362},{"className":6361},[285],[6363,6368,6435,6439,6444],{"type":150,"tag":181,"props":6364,"children":6367},{"className":6365,"style":6366},[290],"height:1.0385em;vertical-align:-0.3552em;",[],{"type":150,"tag":181,"props":6369,"children":6371},{"className":6370},[296],[6372,6377],{"type":150,"tag":181,"props":6373,"children":6375},{"className":6374,"style":6199},[296,301],[6376],{"type":155,"value":6120},{"type":150,"tag":181,"props":6378,"children":6380},{"className":6379},[307],[6381],{"type":150,"tag":181,"props":6382,"children":6384},{"className":6383},[312,313],[6385,6423],{"type":150,"tag":181,"props":6386,"children":6388},{"className":6387},[318],[6389,6418],{"type":150,"tag":181,"props":6390,"children":6392},{"className":6391,"style":3862},[323],[6393],{"type":150,"tag":181,"props":6394,"children":6396},{"style":6395},"top:-2.5198em;margin-left:-0.0715em;margin-right:0.05em;",[6397,6401],{"type":150,"tag":181,"props":6398,"children":6400},{"className":6399,"style":334},[333],[],{"type":150,"tag":181,"props":6402,"children":6404},{"className":6403},[339,340,341,342],[6405],{"type":150,"tag":181,"props":6406,"children":6408},{"className":6407},[296,342],[6409],{"type":150,"tag":181,"props":6410,"children":6412},{"className":6411},[296,155,342],[6413],{"type":150,"tag":181,"props":6414,"children":6416},{"className":6415},[296,342],[6417],{"type":155,"value":6157},{"type":150,"tag":181,"props":6419,"children":6421},{"className":6420},[352],[6422],{"type":155,"value":355},{"type":150,"tag":181,"props":6424,"children":6426},{"className":6425},[318],[6427],{"type":150,"tag":181,"props":6428,"children":6431},{"className":6429,"style":6430},[323],"height:0.3552em;",[6432],{"type":150,"tag":181,"props":6433,"children":6434},{},[],{"type":150,"tag":181,"props":6436,"children":6438},{"className":6437,"style":4993},[377],[],{"type":150,"tag":181,"props":6440,"children":6442},{"className":6441},[4998],[6443],{"type":155,"value":4962},{"type":150,"tag":181,"props":6445,"children":6447},{"className":6446,"style":4993},[377],[],{"type":150,"tag":181,"props":6449,"children":6451},{"className":6450},[285],[6452,6457],{"type":150,"tag":181,"props":6453,"children":6456},{"className":6454,"style":6455},[290],"height:0.9694em;vertical-align:-0.2861em;",[],{"type":150,"tag":181,"props":6458,"children":6460},{"className":6459},[296],[6461,6466],{"type":150,"tag":181,"props":6462,"children":6464},{"className":6463,"style":6199},[296,301],[6465],{"type":155,"value":6120},{"type":150,"tag":181,"props":6467,"children":6469},{"className":6468},[307],[6470],{"type":150,"tag":181,"props":6471,"children":6473},{"className":6472},[312,313],[6474,6511],{"type":150,"tag":181,"props":6475,"children":6477},{"className":6476},[318],[6478,6506],{"type":150,"tag":181,"props":6479,"children":6481},{"className":6480,"style":1191},[323],[6482],{"type":150,"tag":181,"props":6483,"children":6484},{"style":6220},[6485,6489],{"type":150,"tag":181,"props":6486,"children":6488},{"className":6487,"style":334},[333],[],{"type":150,"tag":181,"props":6490,"children":6492},{"className":6491},[339,340,341,342],[6493],{"type":150,"tag":181,"props":6494,"children":6496},{"className":6495},[296,342],[6497],{"type":150,"tag":181,"props":6498,"children":6500},{"className":6499},[296,155,342],[6501],{"type":150,"tag":181,"props":6502,"children":6504},{"className":6503},[296,342],[6505],{"type":155,"value":6173},{"type":150,"tag":181,"props":6507,"children":6509},{"className":6508},[352],[6510],{"type":155,"value":355},{"type":150,"tag":181,"props":6512,"children":6514},{"className":6513},[318],[6515],{"type":150,"tag":181,"props":6516,"children":6519},{"className":6517,"style":6518},[323],"height:0.2861em;",[6520],{"type":150,"tag":181,"props":6521,"children":6522},{},[],{"type":150,"tag":151,"props":6524,"children":6525},{},[6526,6528,6638,6640,6682,6684,6726,6728,6854,6856,6978,6980,7022,7024,7119,7121,7216,7218,7260],{"type":155,"value":6527},"账单上还有另外两项，而且它们经常被无视。第二项 ",{"type":150,"tag":181,"props":6529,"children":6531},{"className":6530},[184],[6532,6561],{"type":150,"tag":181,"props":6533,"children":6535},{"className":6534},[189],[6536],{"type":150,"tag":192,"props":6537,"children":6538},{"xmlns":194},[6539],{"type":150,"tag":197,"props":6540,"children":6541},{},[6542,6556],{"type":150,"tag":201,"props":6543,"children":6544},{},[6545],{"type":150,"tag":205,"props":6546,"children":6547},{},[6548,6552],{"type":150,"tag":209,"props":6549,"children":6550},{},[6551],{"type":155,"value":6120},{"type":150,"tag":249,"props":6553,"children":6554},{},[6555],{"type":155,"value":6157},{"type":150,"tag":271,"props":6557,"children":6558},{"encoding":273},[6559],{"type":155,"value":6560},"C_{\\text{FFN\u002FMoE}}",{"type":150,"tag":181,"props":6562,"children":6564},{"className":6563,"ariaHidden":223},[280],[6565],{"type":150,"tag":181,"props":6566,"children":6568},{"className":6567},[285],[6569,6573],{"type":150,"tag":181,"props":6570,"children":6572},{"className":6571,"style":6366},[290],[],{"type":150,"tag":181,"props":6574,"children":6576},{"className":6575},[296],[6577,6582],{"type":150,"tag":181,"props":6578,"children":6580},{"className":6579,"style":6199},[296,301],[6581],{"type":155,"value":6120},{"type":150,"tag":181,"props":6583,"children":6585},{"className":6584},[307],[6586],{"type":150,"tag":181,"props":6587,"children":6589},{"className":6588},[312,313],[6590,6627],{"type":150,"tag":181,"props":6591,"children":6593},{"className":6592},[318],[6594,6622],{"type":150,"tag":181,"props":6595,"children":6597},{"className":6596,"style":3862},[323],[6598],{"type":150,"tag":181,"props":6599,"children":6600},{"style":6395},[6601,6605],{"type":150,"tag":181,"props":6602,"children":6604},{"className":6603,"style":334},[333],[],{"type":150,"tag":181,"props":6606,"children":6608},{"className":6607},[339,340,341,342],[6609],{"type":150,"tag":181,"props":6610,"children":6612},{"className":6611},[296,342],[6613],{"type":150,"tag":181,"props":6614,"children":6616},{"className":6615},[296,155,342],[6617],{"type":150,"tag":181,"props":6618,"children":6620},{"className":6619},[296,342],[6621],{"type":155,"value":6157},{"type":150,"tag":181,"props":6623,"children":6625},{"className":6624},[352],[6626],{"type":155,"value":355},{"type":150,"tag":181,"props":6628,"children":6630},{"className":6629},[318],[6631],{"type":150,"tag":181,"props":6632,"children":6634},{"className":6633,"style":6430},[323],[6635],{"type":150,"tag":181,"props":6636,"children":6637},{},[],{"type":155,"value":6639}," 是前馈层的计算。MoE（Mixture of Experts）就是把这层换成多个专家网络，每个 token 只激活其中少数几个。它跟序列长度 ",{"type":150,"tag":181,"props":6641,"children":6643},{"className":6642},[184],[6644,6665],{"type":150,"tag":181,"props":6645,"children":6647},{"className":6646},[189],[6648],{"type":150,"tag":192,"props":6649,"children":6650},{"xmlns":194},[6651],{"type":150,"tag":197,"props":6652,"children":6653},{},[6654,6661],{"type":150,"tag":201,"props":6655,"children":6656},{},[6657],{"type":150,"tag":209,"props":6658,"children":6659},{},[6660],{"type":155,"value":5417},{"type":150,"tag":271,"props":6662,"children":6663},{"encoding":273},[6664],{"type":155,"value":5417},{"type":150,"tag":181,"props":6666,"children":6668},{"className":6667,"ariaHidden":223},[280],[6669],{"type":150,"tag":181,"props":6670,"children":6672},{"className":6671},[285],[6673,6677],{"type":150,"tag":181,"props":6674,"children":6676},{"className":6675,"style":5680},[290],[],{"type":150,"tag":181,"props":6678,"children":6680},{"className":6679,"style":5606},[296,301],[6681],{"type":155,"value":5417},{"type":155,"value":6683}," 成正比，跟模型宽度 ",{"type":150,"tag":181,"props":6685,"children":6687},{"className":6686},[184],[6688,6709],{"type":150,"tag":181,"props":6689,"children":6691},{"className":6690},[189],[6692],{"type":150,"tag":192,"props":6693,"children":6694},{"xmlns":194},[6695],{"type":150,"tag":197,"props":6696,"children":6697},{},[6698,6705],{"type":150,"tag":201,"props":6699,"children":6700},{},[6701],{"type":150,"tag":209,"props":6702,"children":6703},{},[6704],{"type":155,"value":578},{"type":150,"tag":271,"props":6706,"children":6707},{"encoding":273},[6708],{"type":155,"value":578},{"type":150,"tag":181,"props":6710,"children":6712},{"className":6711,"ariaHidden":223},[280],[6713],{"type":150,"tag":181,"props":6714,"children":6716},{"className":6715},[285],[6717,6721],{"type":150,"tag":181,"props":6718,"children":6720},{"className":6719,"style":761},[290],[],{"type":150,"tag":181,"props":6722,"children":6724},{"className":6723},[296,301],[6725],{"type":155,"value":578},{"type":155,"value":6727}," 的平方成正比，所以量级是 ",{"type":150,"tag":181,"props":6729,"children":6731},{"className":6730},[184],[6732,6778],{"type":150,"tag":181,"props":6733,"children":6735},{"className":6734},[189],[6736],{"type":150,"tag":192,"props":6737,"children":6738},{"xmlns":194},[6739],{"type":150,"tag":197,"props":6740,"children":6741},{},[6742,6773],{"type":150,"tag":201,"props":6743,"children":6744},{},[6745,6750,6754,6758,6769],{"type":150,"tag":209,"props":6746,"children":6747},{},[6748],{"type":155,"value":6749},"O",{"type":150,"tag":221,"props":6751,"children":6752},{"stretchy":789},[6753],{"type":155,"value":792},{"type":150,"tag":209,"props":6755,"children":6756},{},[6757],{"type":155,"value":5417},{"type":150,"tag":565,"props":6759,"children":6760},{},[6761,6765],{"type":150,"tag":209,"props":6762,"children":6763},{},[6764],{"type":155,"value":578},{"type":150,"tag":215,"props":6766,"children":6767},{},[6768],{"type":155,"value":238},{"type":150,"tag":221,"props":6770,"children":6771},{"stretchy":789},[6772],{"type":155,"value":802},{"type":150,"tag":271,"props":6774,"children":6775},{"encoding":273},[6776],{"type":155,"value":6777},"O(Nd^2)",{"type":150,"tag":181,"props":6779,"children":6781},{"className":6780,"ariaHidden":223},[280],[6782],{"type":150,"tag":181,"props":6783,"children":6785},{"className":6784},[285],[6786,6791,6797,6802,6807,6849],{"type":150,"tag":181,"props":6787,"children":6790},{"className":6788,"style":6789},[290],"height:1.0641em;vertical-align:-0.25em;",[],{"type":150,"tag":181,"props":6792,"children":6795},{"className":6793,"style":6794},[296,301],"margin-right:0.0278em;",[6796],{"type":155,"value":6749},{"type":150,"tag":181,"props":6798,"children":6800},{"className":6799},[824],[6801],{"type":155,"value":792},{"type":150,"tag":181,"props":6803,"children":6805},{"className":6804,"style":5606},[296,301],[6806],{"type":155,"value":5417},{"type":150,"tag":181,"props":6808,"children":6810},{"className":6809},[296],[6811,6816],{"type":150,"tag":181,"props":6812,"children":6814},{"className":6813},[296,301],[6815],{"type":155,"value":578},{"type":150,"tag":181,"props":6817,"children":6819},{"className":6818},[307],[6820],{"type":150,"tag":181,"props":6821,"children":6823},{"className":6822},[312],[6824],{"type":150,"tag":181,"props":6825,"children":6827},{"className":6826},[318],[6828],{"type":150,"tag":181,"props":6829,"children":6832},{"className":6830,"style":6831},[323],"height:0.8141em;",[6833],{"type":150,"tag":181,"props":6834,"children":6835},{"style":707},[6836,6840],{"type":150,"tag":181,"props":6837,"children":6839},{"className":6838,"style":334},[333],[],{"type":150,"tag":181,"props":6841,"children":6843},{"className":6842},[339,340,341,342],[6844],{"type":150,"tag":181,"props":6845,"children":6847},{"className":6846},[296,342],[6848],{"type":155,"value":238},{"type":150,"tag":181,"props":6850,"children":6852},{"className":6851},[835],[6853],{"type":155,"value":802},{"type":155,"value":6855},"。attention 那项是 ",{"type":150,"tag":181,"props":6857,"children":6859},{"className":6858},[184],[6860,6905],{"type":150,"tag":181,"props":6861,"children":6863},{"className":6862},[189],[6864],{"type":150,"tag":192,"props":6865,"children":6866},{"xmlns":194},[6867],{"type":150,"tag":197,"props":6868,"children":6869},{},[6870,6900],{"type":150,"tag":201,"props":6871,"children":6872},{},[6873,6877,6881,6892,6896],{"type":150,"tag":209,"props":6874,"children":6875},{},[6876],{"type":155,"value":6749},{"type":150,"tag":221,"props":6878,"children":6879},{"stretchy":789},[6880],{"type":155,"value":792},{"type":150,"tag":565,"props":6882,"children":6883},{},[6884,6888],{"type":150,"tag":209,"props":6885,"children":6886},{},[6887],{"type":155,"value":5417},{"type":150,"tag":215,"props":6889,"children":6890},{},[6891],{"type":155,"value":238},{"type":150,"tag":209,"props":6893,"children":6894},{},[6895],{"type":155,"value":578},{"type":150,"tag":221,"props":6897,"children":6898},{"stretchy":789},[6899],{"type":155,"value":802},{"type":150,"tag":271,"props":6901,"children":6902},{"encoding":273},[6903],{"type":155,"value":6904},"O(N^2 d)",{"type":150,"tag":181,"props":6906,"children":6908},{"className":6907,"ariaHidden":223},[280],[6909],{"type":150,"tag":181,"props":6910,"children":6912},{"className":6911},[285],[6913,6917,6922,6927,6968,6973],{"type":150,"tag":181,"props":6914,"children":6916},{"className":6915,"style":6789},[290],[],{"type":150,"tag":181,"props":6918,"children":6920},{"className":6919,"style":6794},[296,301],[6921],{"type":155,"value":6749},{"type":150,"tag":181,"props":6923,"children":6925},{"className":6924},[824],[6926],{"type":155,"value":792},{"type":150,"tag":181,"props":6928,"children":6930},{"className":6929},[296],[6931,6936],{"type":150,"tag":181,"props":6932,"children":6934},{"className":6933,"style":5606},[296,301],[6935],{"type":155,"value":5417},{"type":150,"tag":181,"props":6937,"children":6939},{"className":6938},[307],[6940],{"type":150,"tag":181,"props":6941,"children":6943},{"className":6942},[312],[6944],{"type":150,"tag":181,"props":6945,"children":6947},{"className":6946},[318],[6948],{"type":150,"tag":181,"props":6949,"children":6951},{"className":6950,"style":6831},[323],[6952],{"type":150,"tag":181,"props":6953,"children":6954},{"style":707},[6955,6959],{"type":150,"tag":181,"props":6956,"children":6958},{"className":6957,"style":334},[333],[],{"type":150,"tag":181,"props":6960,"children":6962},{"className":6961},[339,340,341,342],[6963],{"type":150,"tag":181,"props":6964,"children":6966},{"className":6965},[296,342],[6967],{"type":155,"value":238},{"type":150,"tag":181,"props":6969,"children":6971},{"className":6970},[296,301],[6972],{"type":155,"value":578},{"type":150,"tag":181,"props":6974,"children":6976},{"className":6975},[835],[6977],{"type":155,"value":802},{"type":155,"value":6979},"。注意这两个量级里的指数位置不一样：",{"type":150,"tag":181,"props":6981,"children":6983},{"className":6982},[184],[6984,7005],{"type":150,"tag":181,"props":6985,"children":6987},{"className":6986},[189],[6988],{"type":150,"tag":192,"props":6989,"children":6990},{"xmlns":194},[6991],{"type":150,"tag":197,"props":6992,"children":6993},{},[6994,7001],{"type":150,"tag":201,"props":6995,"children":6996},{},[6997],{"type":150,"tag":209,"props":6998,"children":6999},{},[7000],{"type":155,"value":5417},{"type":150,"tag":271,"props":7002,"children":7003},{"encoding":273},[7004],{"type":155,"value":5417},{"type":150,"tag":181,"props":7006,"children":7008},{"className":7007,"ariaHidden":223},[280],[7009],{"type":150,"tag":181,"props":7010,"children":7012},{"className":7011},[285],[7013,7017],{"type":150,"tag":181,"props":7014,"children":7016},{"className":7015,"style":5680},[290],[],{"type":150,"tag":181,"props":7018,"children":7020},{"className":7019,"style":5606},[296,301],[7021],{"type":155,"value":5417},{"type":155,"value":7023}," 小的时候，",{"type":150,"tag":181,"props":7025,"children":7027},{"className":7026},[184],[7028,7061],{"type":150,"tag":181,"props":7029,"children":7031},{"className":7030},[189],[7032],{"type":150,"tag":192,"props":7033,"children":7034},{"xmlns":194},[7035],{"type":150,"tag":197,"props":7036,"children":7037},{},[7038,7056],{"type":150,"tag":201,"props":7039,"children":7040},{},[7041,7052],{"type":150,"tag":565,"props":7042,"children":7043},{},[7044,7048],{"type":150,"tag":209,"props":7045,"children":7046},{},[7047],{"type":155,"value":5417},{"type":150,"tag":215,"props":7049,"children":7050},{},[7051],{"type":155,"value":238},{"type":150,"tag":209,"props":7053,"children":7054},{},[7055],{"type":155,"value":578},{"type":150,"tag":271,"props":7057,"children":7058},{"encoding":273},[7059],{"type":155,"value":7060},"N^2 d",{"type":150,"tag":181,"props":7062,"children":7064},{"className":7063,"ariaHidden":223},[280],[7065],{"type":150,"tag":181,"props":7066,"children":7068},{"className":7067},[285],[7069,7073,7114],{"type":150,"tag":181,"props":7070,"children":7072},{"className":7071,"style":6831},[290],[],{"type":150,"tag":181,"props":7074,"children":7076},{"className":7075},[296],[7077,7082],{"type":150,"tag":181,"props":7078,"children":7080},{"className":7079,"style":5606},[296,301],[7081],{"type":155,"value":5417},{"type":150,"tag":181,"props":7083,"children":7085},{"className":7084},[307],[7086],{"type":150,"tag":181,"props":7087,"children":7089},{"className":7088},[312],[7090],{"type":150,"tag":181,"props":7091,"children":7093},{"className":7092},[318],[7094],{"type":150,"tag":181,"props":7095,"children":7097},{"className":7096,"style":6831},[323],[7098],{"type":150,"tag":181,"props":7099,"children":7100},{"style":707},[7101,7105],{"type":150,"tag":181,"props":7102,"children":7104},{"className":7103,"style":334},[333],[],{"type":150,"tag":181,"props":7106,"children":7108},{"className":7107},[339,340,341,342],[7109],{"type":150,"tag":181,"props":7110,"children":7112},{"className":7111},[296,342],[7113],{"type":155,"value":238},{"type":150,"tag":181,"props":7115,"children":7117},{"className":7116},[296,301],[7118],{"type":155,"value":578},{"type":155,"value":7120}," 打不过 ",{"type":150,"tag":181,"props":7122,"children":7124},{"className":7123},[184],[7125,7158],{"type":150,"tag":181,"props":7126,"children":7128},{"className":7127},[189],[7129],{"type":150,"tag":192,"props":7130,"children":7131},{"xmlns":194},[7132],{"type":150,"tag":197,"props":7133,"children":7134},{},[7135,7153],{"type":150,"tag":201,"props":7136,"children":7137},{},[7138,7142],{"type":150,"tag":209,"props":7139,"children":7140},{},[7141],{"type":155,"value":5417},{"type":150,"tag":565,"props":7143,"children":7144},{},[7145,7149],{"type":150,"tag":209,"props":7146,"children":7147},{},[7148],{"type":155,"value":578},{"type":150,"tag":215,"props":7150,"children":7151},{},[7152],{"type":155,"value":238},{"type":150,"tag":271,"props":7154,"children":7155},{"encoding":273},[7156],{"type":155,"value":7157},"Nd^2",{"type":150,"tag":181,"props":7159,"children":7161},{"className":7160,"ariaHidden":223},[280],[7162],{"type":150,"tag":181,"props":7163,"children":7165},{"className":7164},[285],[7166,7170,7175],{"type":150,"tag":181,"props":7167,"children":7169},{"className":7168,"style":6831},[290],[],{"type":150,"tag":181,"props":7171,"children":7173},{"className":7172,"style":5606},[296,301],[7174],{"type":155,"value":5417},{"type":150,"tag":181,"props":7176,"children":7178},{"className":7177},[296],[7179,7184],{"type":150,"tag":181,"props":7180,"children":7182},{"className":7181},[296,301],[7183],{"type":155,"value":578},{"type":150,"tag":181,"props":7185,"children":7187},{"className":7186},[307],[7188],{"type":150,"tag":181,"props":7189,"children":7191},{"className":7190},[312],[7192],{"type":150,"tag":181,"props":7193,"children":7195},{"className":7194},[318],[7196],{"type":150,"tag":181,"props":7197,"children":7199},{"className":7198,"style":6831},[323],[7200],{"type":150,"tag":181,"props":7201,"children":7202},{"style":707},[7203,7207],{"type":150,"tag":181,"props":7204,"children":7206},{"className":7205,"style":334},[333],[],{"type":150,"tag":181,"props":7208,"children":7210},{"className":7209},[339,340,341,342],[7211],{"type":150,"tag":181,"props":7212,"children":7214},{"className":7213},[296,342],[7215],{"type":155,"value":238},{"type":155,"value":7217},"；",{"type":150,"tag":181,"props":7219,"children":7221},{"className":7220},[184],[7222,7243],{"type":150,"tag":181,"props":7223,"children":7225},{"className":7224},[189],[7226],{"type":150,"tag":192,"props":7227,"children":7228},{"xmlns":194},[7229],{"type":150,"tag":197,"props":7230,"children":7231},{},[7232,7239],{"type":150,"tag":201,"props":7233,"children":7234},{},[7235],{"type":150,"tag":209,"props":7236,"children":7237},{},[7238],{"type":155,"value":5417},{"type":150,"tag":271,"props":7240,"children":7241},{"encoding":273},[7242],{"type":155,"value":5417},{"type":150,"tag":181,"props":7244,"children":7246},{"className":7245,"ariaHidden":223},[280],[7247],{"type":150,"tag":181,"props":7248,"children":7250},{"className":7249},[285],[7251,7255],{"type":150,"tag":181,"props":7252,"children":7254},{"className":7253,"style":5680},[290],[],{"type":150,"tag":181,"props":7256,"children":7258},{"className":7257,"style":5606},[296,301],[7259],{"type":155,"value":5417},{"type":155,"value":7261}," 大起来之后才反过来。",{"type":150,"tag":151,"props":7263,"children":7264},{},[7265,7267,7309,7311,7397,7399,7520,7522,7632],{"type":155,"value":7266},"这也是为什么在短 context 时代大家理所当然地只盯 attention。那时候它确实是瓶颈。但当 ",{"type":150,"tag":181,"props":7268,"children":7270},{"className":7269},[184],[7271,7292],{"type":150,"tag":181,"props":7272,"children":7274},{"className":7273},[189],[7275],{"type":150,"tag":192,"props":7276,"children":7277},{"xmlns":194},[7278],{"type":150,"tag":197,"props":7279,"children":7280},{},[7281,7288],{"type":150,"tag":201,"props":7282,"children":7283},{},[7284],{"type":150,"tag":209,"props":7285,"children":7286},{},[7287],{"type":155,"value":5417},{"type":150,"tag":271,"props":7289,"children":7290},{"encoding":273},[7291],{"type":155,"value":5417},{"type":150,"tag":181,"props":7293,"children":7295},{"className":7294,"ariaHidden":223},[280],[7296],{"type":150,"tag":181,"props":7297,"children":7299},{"className":7298},[285],[7300,7304],{"type":150,"tag":181,"props":7301,"children":7303},{"className":7302,"style":5680},[290],[],{"type":150,"tag":181,"props":7305,"children":7307},{"className":7306,"style":5606},[296,301],[7308],{"type":155,"value":5417},{"type":155,"value":7310}," 从 4k 涨到 1M，情况变了；再等到 sparse attention 把 ",{"type":150,"tag":181,"props":7312,"children":7314},{"className":7313},[184],[7315,7344],{"type":150,"tag":181,"props":7316,"children":7318},{"className":7317},[189],[7319],{"type":150,"tag":192,"props":7320,"children":7321},{"xmlns":194},[7322],{"type":150,"tag":197,"props":7323,"children":7324},{},[7325,7339],{"type":150,"tag":201,"props":7326,"children":7327},{},[7328],{"type":150,"tag":565,"props":7329,"children":7330},{},[7331,7335],{"type":150,"tag":209,"props":7332,"children":7333},{},[7334],{"type":155,"value":5417},{"type":150,"tag":215,"props":7336,"children":7337},{},[7338],{"type":155,"value":238},{"type":150,"tag":271,"props":7340,"children":7341},{"encoding":273},[7342],{"type":155,"value":7343},"N^2",{"type":150,"tag":181,"props":7345,"children":7347},{"className":7346,"ariaHidden":223},[280],[7348],{"type":150,"tag":181,"props":7349,"children":7351},{"className":7350},[285],[7352,7356],{"type":150,"tag":181,"props":7353,"children":7355},{"className":7354,"style":6831},[290],[],{"type":150,"tag":181,"props":7357,"children":7359},{"className":7358},[296],[7360,7365],{"type":150,"tag":181,"props":7361,"children":7363},{"className":7362,"style":5606},[296,301],[7364],{"type":155,"value":5417},{"type":150,"tag":181,"props":7366,"children":7368},{"className":7367},[307],[7369],{"type":150,"tag":181,"props":7370,"children":7372},{"className":7371},[312],[7373],{"type":150,"tag":181,"props":7374,"children":7376},{"className":7375},[318],[7377],{"type":150,"tag":181,"props":7378,"children":7380},{"className":7379,"style":6831},[323],[7381],{"type":150,"tag":181,"props":7382,"children":7383},{"style":707},[7384,7388],{"type":150,"tag":181,"props":7385,"children":7387},{"className":7386,"style":334},[333],[],{"type":150,"tag":181,"props":7389,"children":7391},{"className":7390},[339,340,341,342],[7392],{"type":150,"tag":181,"props":7393,"children":7395},{"className":7394},[296,342],[7396],{"type":155,"value":238},{"type":155,"value":7398}," 压下去，",{"type":150,"tag":181,"props":7400,"children":7402},{"className":7401},[184],[7403,7447],{"type":150,"tag":181,"props":7404,"children":7406},{"className":7405},[189],[7407],{"type":150,"tag":192,"props":7408,"children":7409},{"xmlns":194},[7410],{"type":150,"tag":197,"props":7411,"children":7412},{},[7413,7443],{"type":150,"tag":201,"props":7414,"children":7415},{},[7416,7420,7424,7428,7439],{"type":150,"tag":209,"props":7417,"children":7418},{},[7419],{"type":155,"value":6749},{"type":150,"tag":221,"props":7421,"children":7422},{"stretchy":789},[7423],{"type":155,"value":792},{"type":150,"tag":209,"props":7425,"children":7426},{},[7427],{"type":155,"value":5417},{"type":150,"tag":565,"props":7429,"children":7430},{},[7431,7435],{"type":150,"tag":209,"props":7432,"children":7433},{},[7434],{"type":155,"value":578},{"type":150,"tag":215,"props":7436,"children":7437},{},[7438],{"type":155,"value":238},{"type":150,"tag":221,"props":7440,"children":7441},{"stretchy":789},[7442],{"type":155,"value":802},{"type":150,"tag":271,"props":7444,"children":7445},{"encoding":273},[7446],{"type":155,"value":6777},{"type":150,"tag":181,"props":7448,"children":7450},{"className":7449,"ariaHidden":223},[280],[7451],{"type":150,"tag":181,"props":7452,"children":7454},{"className":7453},[285],[7455,7459,7464,7469,7474,7515],{"type":150,"tag":181,"props":7456,"children":7458},{"className":7457,"style":6789},[290],[],{"type":150,"tag":181,"props":7460,"children":7462},{"className":7461,"style":6794},[296,301],[7463],{"type":155,"value":6749},{"type":150,"tag":181,"props":7465,"children":7467},{"className":7466},[824],[7468],{"type":155,"value":792},{"type":150,"tag":181,"props":7470,"children":7472},{"className":7471,"style":5606},[296,301],[7473],{"type":155,"value":5417},{"type":150,"tag":181,"props":7475,"children":7477},{"className":7476},[296],[7478,7483],{"type":150,"tag":181,"props":7479,"children":7481},{"className":7480},[296,301],[7482],{"type":155,"value":578},{"type":150,"tag":181,"props":7484,"children":7486},{"className":7485},[307],[7487],{"type":150,"tag":181,"props":7488,"children":7490},{"className":7489},[312],[7491],{"type":150,"tag":181,"props":7492,"children":7494},{"className":7493},[318],[7495],{"type":150,"tag":181,"props":7496,"children":7498},{"className":7497,"style":6831},[323],[7499],{"type":150,"tag":181,"props":7500,"children":7501},{"style":707},[7502,7506],{"type":150,"tag":181,"props":7503,"children":7505},{"className":7504,"style":334},[333],[],{"type":150,"tag":181,"props":7507,"children":7509},{"className":7508},[339,340,341,342],[7510],{"type":150,"tag":181,"props":7511,"children":7513},{"className":7512},[296,342],[7514],{"type":155,"value":238},{"type":150,"tag":181,"props":7516,"children":7518},{"className":7517},[835],[7519],{"type":155,"value":802},{"type":155,"value":7521}," 这一项就浮到水面上了。第三项 ",{"type":150,"tag":181,"props":7523,"children":7525},{"className":7524},[184],[7526,7555],{"type":150,"tag":181,"props":7527,"children":7529},{"className":7528},[189],[7530],{"type":150,"tag":192,"props":7531,"children":7532},{"xmlns":194},[7533],{"type":150,"tag":197,"props":7534,"children":7535},{},[7536,7550],{"type":150,"tag":201,"props":7537,"children":7538},{},[7539],{"type":150,"tag":205,"props":7540,"children":7541},{},[7542,7546],{"type":150,"tag":209,"props":7543,"children":7544},{},[7545],{"type":155,"value":6120},{"type":150,"tag":249,"props":7547,"children":7548},{},[7549],{"type":155,"value":6173},{"type":150,"tag":271,"props":7551,"children":7552},{"encoding":273},[7553],{"type":155,"value":7554},"C_{\\text{memory movement}}",{"type":150,"tag":181,"props":7556,"children":7558},{"className":7557,"ariaHidden":223},[280],[7559],{"type":150,"tag":181,"props":7560,"children":7562},{"className":7561},[285],[7563,7567],{"type":150,"tag":181,"props":7564,"children":7566},{"className":7565,"style":6455},[290],[],{"type":150,"tag":181,"props":7568,"children":7570},{"className":7569},[296],[7571,7576],{"type":150,"tag":181,"props":7572,"children":7574},{"className":7573,"style":6199},[296,301],[7575],{"type":155,"value":6120},{"type":150,"tag":181,"props":7577,"children":7579},{"className":7578},[307],[7580],{"type":150,"tag":181,"props":7581,"children":7583},{"className":7582},[312,313],[7584,7621],{"type":150,"tag":181,"props":7585,"children":7587},{"className":7586},[318],[7588,7616],{"type":150,"tag":181,"props":7589,"children":7591},{"className":7590,"style":1191},[323],[7592],{"type":150,"tag":181,"props":7593,"children":7594},{"style":6220},[7595,7599],{"type":150,"tag":181,"props":7596,"children":7598},{"className":7597,"style":334},[333],[],{"type":150,"tag":181,"props":7600,"children":7602},{"className":7601},[339,340,341,342],[7603],{"type":150,"tag":181,"props":7604,"children":7606},{"className":7605},[296,342],[7607],{"type":150,"tag":181,"props":7608,"children":7610},{"className":7609},[296,155,342],[7611],{"type":150,"tag":181,"props":7612,"children":7614},{"className":7613},[296,342],[7615],{"type":155,"value":6173},{"type":150,"tag":181,"props":7617,"children":7619},{"className":7618},[352],[7620],{"type":155,"value":355},{"type":150,"tag":181,"props":7622,"children":7624},{"className":7623},[318],[7625],{"type":150,"tag":181,"props":7626,"children":7628},{"className":7627,"style":6518},[323],[7629],{"type":150,"tag":181,"props":7630,"children":7631},{},[],{"type":155,"value":7633}," 是最容易漏的。GPU 上算一个 FLOP 很快，搬一次数据慢得多，所以\"减少 KV Cache\"的收益很多时候落在省搬运，而不在省显存。但这件事有个限度：decode 每步要读的权重是固定的，搬不动，KV Cache 压到比它更小之后，继续压的边际收益就下降。这条限度解释了后面很多设计的取舍，也解释了为什么 1M context 之后，战场转向每层的计算和权重：账单上还能砍的地方换位置了。",{"type":150,"tag":169,"props":7635,"children":7637},{"id":7636},"mqa-与-gqa",[7638],{"type":155,"value":7639},"MQA 与 GQA",{"type":150,"tag":151,"props":7641,"children":7642},{},[7643,7645,7710,7712,7772,7774,7831],{"type":155,"value":7644},"既然 KV Cache 是问题，最朴素的想法就是：少存点。MQA（Multi-Query Attention）直接让所有 attention head 共享同一份 K、V。用公式说，就是把所有 ",{"type":150,"tag":181,"props":7646,"children":7648},{"className":7647},[184],[7649,7679],{"type":150,"tag":181,"props":7650,"children":7652},{"className":7651},[189],[7653],{"type":150,"tag":192,"props":7654,"children":7655},{"xmlns":194},[7656],{"type":150,"tag":197,"props":7657,"children":7658},{},[7659,7674],{"type":150,"tag":201,"props":7660,"children":7661},{},[7662,7666,7670],{"type":150,"tag":209,"props":7663,"children":7664},{},[7665],{"type":155,"value":1698},{"type":150,"tag":221,"props":7667,"children":7668},{"separator":223},[7669],{"type":155,"value":226},{"type":150,"tag":209,"props":7671,"children":7672},{},[7673],{"type":155,"value":1737},{"type":150,"tag":271,"props":7675,"children":7676},{"encoding":273},[7677],{"type":155,"value":7678},"k, v",{"type":150,"tag":181,"props":7680,"children":7682},{"className":7681,"ariaHidden":223},[280],[7683],{"type":150,"tag":181,"props":7684,"children":7686},{"className":7685},[285],[7687,7691,7696,7701,7705],{"type":150,"tag":181,"props":7688,"children":7690},{"className":7689,"style":941},[290],[],{"type":150,"tag":181,"props":7692,"children":7694},{"className":7693,"style":2249},[296,301],[7695],{"type":155,"value":1698},{"type":150,"tag":181,"props":7697,"children":7699},{"className":7698},[371],[7700],{"type":155,"value":226},{"type":150,"tag":181,"props":7702,"children":7704},{"className":7703,"style":378},[377],[],{"type":150,"tag":181,"props":7706,"children":7708},{"className":7707,"style":2151},[296,301],[7709],{"type":155,"value":1737},{"type":155,"value":7711}," 的上标 ",{"type":150,"tag":181,"props":7713,"children":7715},{"className":7714},[184],[7716,7745],{"type":150,"tag":181,"props":7717,"children":7719},{"className":7718},[189],[7720],{"type":150,"tag":192,"props":7721,"children":7722},{"xmlns":194},[7723],{"type":150,"tag":197,"props":7724,"children":7725},{},[7726,7741],{"type":150,"tag":201,"props":7727,"children":7728},{},[7729,7733,7737],{"type":150,"tag":221,"props":7730,"children":7731},{"stretchy":789},[7732],{"type":155,"value":792},{"type":150,"tag":209,"props":7734,"children":7735},{},[7736],{"type":155,"value":797},{"type":150,"tag":221,"props":7738,"children":7739},{"stretchy":789},[7740],{"type":155,"value":802},{"type":150,"tag":271,"props":7742,"children":7743},{"encoding":273},[7744],{"type":155,"value":807},{"type":150,"tag":181,"props":7746,"children":7748},{"className":7747,"ariaHidden":223},[280],[7749],{"type":150,"tag":181,"props":7750,"children":7752},{"className":7751},[285],[7753,7757,7762,7767],{"type":150,"tag":181,"props":7754,"children":7756},{"className":7755,"style":819},[290],[],{"type":150,"tag":181,"props":7758,"children":7760},{"className":7759},[824],[7761],{"type":155,"value":792},{"type":150,"tag":181,"props":7763,"children":7765},{"className":7764},[296,301],[7766],{"type":155,"value":797},{"type":150,"tag":181,"props":7768,"children":7770},{"className":7769},[835],[7771],{"type":155,"value":802},{"type":155,"value":7773}," 去掉。它是 2019 年提出来的。那会儿 LLM 还没火，可见减少 KV Cache 早就已经是热门课题了。KV Cache 直接降到原来的 ",{"type":150,"tag":181,"props":7775,"children":7777},{"className":7776},[184],[7778,7808],{"type":150,"tag":181,"props":7779,"children":7781},{"className":7780},[189],[7782],{"type":150,"tag":192,"props":7783,"children":7784},{"xmlns":194},[7785],{"type":150,"tag":197,"props":7786,"children":7787},{},[7788,7803],{"type":150,"tag":201,"props":7789,"children":7790},{},[7791,7795,7799],{"type":150,"tag":215,"props":7792,"children":7793},{},[7794],{"type":155,"value":219},{"type":150,"tag":209,"props":7796,"children":7797},{"mathvariant":1618},[7798],{"type":155,"value":4740},{"type":150,"tag":209,"props":7800,"children":7801},{},[7802],{"type":155,"value":745},{"type":150,"tag":271,"props":7804,"children":7805},{"encoding":273},[7806],{"type":155,"value":7807},"1\u002Fh",{"type":150,"tag":181,"props":7809,"children":7811},{"className":7810,"ariaHidden":223},[280],[7812],{"type":150,"tag":181,"props":7813,"children":7815},{"className":7814},[285],[7816,7820,7826],{"type":150,"tag":181,"props":7817,"children":7819},{"className":7818,"style":819},[290],[],{"type":150,"tag":181,"props":7821,"children":7823},{"className":7822},[296],[7824],{"type":155,"value":7825},"1\u002F",{"type":150,"tag":181,"props":7827,"children":7829},{"className":7828},[296,301],[7830],{"type":155,"value":745},{"type":155,"value":7832},"。单从省显存的角度看，这已经是天花板了。",{"type":150,"tag":151,"props":7834,"children":7835},{},[7836],{"type":155,"value":7837},"代价也很直白：原来一个 token 的每个 head 都有各自的 K、V，现在被压成一份，信息损失不小。",{"type":150,"tag":151,"props":7839,"children":7840},{},[7841],{"type":155,"value":7842},"不过 MQA 的支持者认为这个损失被高估了。理由是不同 head 学到的 K、V 本来就高度相关，共享一份丢不掉太多有效信息；而且这部分损失可以通过继续训练补回来。这个论证后来被证明有一定道理，但\"一定道理\"和\"可以无条件依赖\"是两回事。GQA 的出现本身就说明，业界并不敢把压缩率开到 MQA 那么极端。",{"type":150,"tag":151,"props":7844,"children":7845},{},[7846,7848,7890,7892,7935],{"type":155,"value":7847},"有个细节容易被忽略。MQA 共享 K、V 之后，attention 部分的参数量少了将近一半。为了保持总参数量不变，通常会相应增大 FFN 的规模。这能补回来一部分效果损失。但还有人担心压得太狠，影响学习效率和最终效果。于是有了 GQA（Grouped-Query Attention），那是 2023 年的事。GQA 把 ",{"type":150,"tag":181,"props":7849,"children":7851},{"className":7850},[184],[7852,7873],{"type":150,"tag":181,"props":7853,"children":7855},{"className":7854},[189],[7856],{"type":150,"tag":192,"props":7857,"children":7858},{"xmlns":194},[7859],{"type":150,"tag":197,"props":7860,"children":7861},{},[7862,7869],{"type":150,"tag":201,"props":7863,"children":7864},{},[7865],{"type":150,"tag":209,"props":7866,"children":7867},{},[7868],{"type":155,"value":745},{"type":150,"tag":271,"props":7870,"children":7871},{"encoding":273},[7872],{"type":155,"value":745},{"type":150,"tag":181,"props":7874,"children":7876},{"className":7875,"ariaHidden":223},[280],[7877],{"type":150,"tag":181,"props":7878,"children":7880},{"className":7879},[285],[7881,7885],{"type":150,"tag":181,"props":7882,"children":7884},{"className":7883,"style":761},[290],[],{"type":150,"tag":181,"props":7886,"children":7888},{"className":7887},[296,301],[7889],{"type":155,"value":745},{"type":155,"value":7891}," 个 head 分成 ",{"type":150,"tag":181,"props":7893,"children":7895},{"className":7894},[184],[7896,7918],{"type":150,"tag":181,"props":7897,"children":7899},{"className":7898},[189],[7900],{"type":150,"tag":192,"props":7901,"children":7902},{"xmlns":194},[7903],{"type":150,"tag":197,"props":7904,"children":7905},{},[7906,7914],{"type":150,"tag":201,"props":7907,"children":7908},{},[7909],{"type":150,"tag":209,"props":7910,"children":7911},{},[7912],{"type":155,"value":7913},"g",{"type":150,"tag":271,"props":7915,"children":7916},{"encoding":273},[7917],{"type":155,"value":7913},{"type":150,"tag":181,"props":7919,"children":7921},{"className":7920,"ariaHidden":223},[280],[7922],{"type":150,"tag":181,"props":7923,"children":7925},{"className":7924},[285],[7926,7930],{"type":150,"tag":181,"props":7927,"children":7929},{"className":7928,"style":291},[290],[],{"type":150,"tag":181,"props":7931,"children":7933},{"className":7932,"style":2151},[296,301],[7934],{"type":155,"value":7913},{"type":155,"value":7936}," 组，每组共享同一对 K、V。具体怎么分？",{"type":150,"tag":151,"props":7938,"children":7939},{},[7940,7942,7984,7986,8058],{"type":155,"value":7941},"记第 ",{"type":150,"tag":181,"props":7943,"children":7945},{"className":7944},[184],[7946,7967],{"type":150,"tag":181,"props":7947,"children":7949},{"className":7948},[189],[7950],{"type":150,"tag":192,"props":7951,"children":7952},{"xmlns":194},[7953],{"type":150,"tag":197,"props":7954,"children":7955},{},[7956,7963],{"type":150,"tag":201,"props":7957,"children":7958},{},[7959],{"type":150,"tag":209,"props":7960,"children":7961},{},[7962],{"type":155,"value":797},{"type":150,"tag":271,"props":7964,"children":7965},{"encoding":273},[7966],{"type":155,"value":797},{"type":150,"tag":181,"props":7968,"children":7970},{"className":7969,"ariaHidden":223},[280],[7971],{"type":150,"tag":181,"props":7972,"children":7974},{"className":7973},[285],[7975,7979],{"type":150,"tag":181,"props":7976,"children":7978},{"className":7977,"style":914},[290],[],{"type":150,"tag":181,"props":7980,"children":7982},{"className":7981},[296,301],[7983],{"type":155,"value":797},{"type":155,"value":7985}," 个 head 属于第 ",{"type":150,"tag":181,"props":7987,"children":7989},{"className":7988},[184],[7990,8025],{"type":150,"tag":181,"props":7991,"children":7993},{"className":7992},[189],[7994],{"type":150,"tag":192,"props":7995,"children":7996},{"xmlns":194},[7997],{"type":150,"tag":197,"props":7998,"children":7999},{},[8000,8020],{"type":150,"tag":201,"props":8001,"children":8002},{},[8003,8008,8012,8016],{"type":150,"tag":209,"props":8004,"children":8005},{},[8006],{"type":155,"value":8007},"γ",{"type":150,"tag":221,"props":8009,"children":8010},{"stretchy":789},[8011],{"type":155,"value":792},{"type":150,"tag":209,"props":8013,"children":8014},{},[8015],{"type":155,"value":797},{"type":150,"tag":221,"props":8017,"children":8018},{"stretchy":789},[8019],{"type":155,"value":802},{"type":150,"tag":271,"props":8021,"children":8022},{"encoding":273},[8023],{"type":155,"value":8024},"\\gamma(s)",{"type":150,"tag":181,"props":8026,"children":8028},{"className":8027,"ariaHidden":223},[280],[8029],{"type":150,"tag":181,"props":8030,"children":8032},{"className":8031},[285],[8033,8037,8043,8048,8053],{"type":150,"tag":181,"props":8034,"children":8036},{"className":8035,"style":819},[290],[],{"type":150,"tag":181,"props":8038,"children":8041},{"className":8039,"style":8040},[296,301],"margin-right:0.0556em;",[8042],{"type":155,"value":8007},{"type":150,"tag":181,"props":8044,"children":8046},{"className":8045},[824],[8047],{"type":155,"value":792},{"type":150,"tag":181,"props":8049,"children":8051},{"className":8050},[296,301],[8052],{"type":155,"value":797},{"type":150,"tag":181,"props":8054,"children":8056},{"className":8055},[835],[8057],{"type":155,"value":802},{"type":155,"value":8059}," 组：",{"type":150,"tag":181,"props":8061,"children":8063},{"className":8062},[1003],[8064],{"type":150,"tag":181,"props":8065,"children":8067},{"className":8066},[184],[8068,8137],{"type":150,"tag":181,"props":8069,"children":8071},{"className":8070},[189],[8072],{"type":150,"tag":192,"props":8073,"children":8074},{"xmlns":194,"display":1015},[8075],{"type":150,"tag":197,"props":8076,"children":8077},{},[8078,8132],{"type":150,"tag":201,"props":8079,"children":8080},{},[8081,8085,8089,8093,8097,8101],{"type":150,"tag":209,"props":8082,"children":8083},{},[8084],{"type":155,"value":8007},{"type":150,"tag":221,"props":8086,"children":8087},{"stretchy":789},[8088],{"type":155,"value":792},{"type":150,"tag":209,"props":8090,"children":8091},{},[8092],{"type":155,"value":797},{"type":150,"tag":221,"props":8094,"children":8095},{"stretchy":789},[8096],{"type":155,"value":802},{"type":150,"tag":221,"props":8098,"children":8099},{},[8100],{"type":155,"value":865},{"type":150,"tag":201,"props":8102,"children":8103},{},[8104,8109,8127],{"type":150,"tag":221,"props":8105,"children":8106},{"fence":223},[8107],{"type":155,"value":8108},"⌈",{"type":150,"tag":1774,"props":8110,"children":8111},{},[8112,8123],{"type":150,"tag":201,"props":8113,"children":8114},{},[8115,8119],{"type":150,"tag":209,"props":8116,"children":8117},{},[8118],{"type":155,"value":797},{"type":150,"tag":209,"props":8120,"children":8121},{},[8122],{"type":155,"value":7913},{"type":150,"tag":209,"props":8124,"children":8125},{},[8126],{"type":155,"value":745},{"type":150,"tag":221,"props":8128,"children":8129},{"fence":223},[8130],{"type":155,"value":8131},"⌉",{"type":150,"tag":271,"props":8133,"children":8134},{"encoding":273},[8135],{"type":155,"value":8136},"\\gamma(s) = \\left\\lceil \\frac{sg}{h} \\right\\rceil",{"type":150,"tag":181,"props":8138,"children":8140},{"className":8139,"ariaHidden":223},[280],[8141,8182],{"type":150,"tag":181,"props":8142,"children":8144},{"className":8143},[285],[8145,8149,8154,8159,8164,8169,8173,8178],{"type":150,"tag":181,"props":8146,"children":8148},{"className":8147,"style":819},[290],[],{"type":150,"tag":181,"props":8150,"children":8152},{"className":8151,"style":8040},[296,301],[8153],{"type":155,"value":8007},{"type":150,"tag":181,"props":8155,"children":8157},{"className":8156},[824],[8158],{"type":155,"value":792},{"type":150,"tag":181,"props":8160,"children":8162},{"className":8161},[296,301],[8163],{"type":155,"value":797},{"type":150,"tag":181,"props":8165,"children":8167},{"className":8166},[835],[8168],{"type":155,"value":802},{"type":150,"tag":181,"props":8170,"children":8172},{"className":8171,"style":658},[377],[],{"type":150,"tag":181,"props":8174,"children":8176},{"className":8175},[663],[8177],{"type":155,"value":865},{"type":150,"tag":181,"props":8179,"children":8181},{"className":8180,"style":658},[377],[],{"type":150,"tag":181,"props":8183,"children":8185},{"className":8184},[285],[8186,8191],{"type":150,"tag":181,"props":8187,"children":8190},{"className":8188,"style":8189},[290],"height:1.836em;vertical-align:-0.686em;",[],{"type":150,"tag":181,"props":8192,"children":8194},{"className":8193},[449],[8195,8208,8306],{"type":150,"tag":181,"props":8196,"children":8200},{"className":8197,"style":8199},[824,8198],"delimcenter","top:0em;",[8201],{"type":150,"tag":181,"props":8202,"children":8206},{"className":8203},[8204,8205],"delimsizing","size2",[8207],{"type":155,"value":8108},{"type":150,"tag":181,"props":8209,"children":8211},{"className":8210},[296],[8212,8216,8302],{"type":150,"tag":181,"props":8213,"children":8215},{"className":8214},[824,2482],[],{"type":150,"tag":181,"props":8217,"children":8219},{"className":8218},[1774],[8220],{"type":150,"tag":181,"props":8221,"children":8223},{"className":8222},[312,313],[8224,8290],{"type":150,"tag":181,"props":8225,"children":8227},{"className":8226},[318],[8228,8285],{"type":150,"tag":181,"props":8229,"children":8232},{"className":8230,"style":8231},[323],"height:1.1076em;",[8233,8251,8263],{"type":150,"tag":181,"props":8234,"children":8236},{"style":8235},"top:-2.314em;",[8237,8242],{"type":150,"tag":181,"props":8238,"children":8241},{"className":8239,"style":8240},[333],"height:3em;",[],{"type":150,"tag":181,"props":8243,"children":8245},{"className":8244},[296],[8246],{"type":150,"tag":181,"props":8247,"children":8249},{"className":8248},[296,301],[8250],{"type":155,"value":745},{"type":150,"tag":181,"props":8252,"children":8254},{"style":8253},"top:-3.23em;",[8255,8259],{"type":150,"tag":181,"props":8256,"children":8258},{"className":8257,"style":8240},[333],[],{"type":150,"tag":181,"props":8260,"children":8262},{"className":8261,"style":2804},[2803],[],{"type":150,"tag":181,"props":8264,"children":8266},{"style":8265},"top:-3.677em;",[8267,8271],{"type":150,"tag":181,"props":8268,"children":8270},{"className":8269,"style":8240},[333],[],{"type":150,"tag":181,"props":8272,"children":8274},{"className":8273},[296],[8275,8280],{"type":150,"tag":181,"props":8276,"children":8278},{"className":8277},[296,301],[8279],{"type":155,"value":797},{"type":150,"tag":181,"props":8281,"children":8283},{"className":8282,"style":2151},[296,301],[8284],{"type":155,"value":7913},{"type":150,"tag":181,"props":8286,"children":8288},{"className":8287},[352],[8289],{"type":155,"value":355},{"type":150,"tag":181,"props":8291,"children":8293},{"className":8292},[318],[8294],{"type":150,"tag":181,"props":8295,"children":8298},{"className":8296,"style":8297},[323],"height:0.686em;",[8299],{"type":150,"tag":181,"props":8300,"children":8301},{},[],{"type":150,"tag":181,"props":8303,"children":8305},{"className":8304},[835,2482],[],{"type":150,"tag":181,"props":8307,"children":8309},{"className":8308,"style":8199},[835,8198],[8310],{"type":150,"tag":181,"props":8311,"children":8313},{"className":8312},[8204,8205],[8314],{"type":155,"value":8131},{"type":150,"tag":151,"props":8316,"children":8317},{},[8318,8320,8381,8383,8443],{"type":155,"value":8319},"（就是按顺序平均切开：前 ",{"type":150,"tag":181,"props":8321,"children":8323},{"className":8322},[184],[8324,8354],{"type":150,"tag":181,"props":8325,"children":8327},{"className":8326},[189],[8328],{"type":150,"tag":192,"props":8329,"children":8330},{"xmlns":194},[8331],{"type":150,"tag":197,"props":8332,"children":8333},{},[8334,8349],{"type":150,"tag":201,"props":8335,"children":8336},{},[8337,8341,8345],{"type":150,"tag":209,"props":8338,"children":8339},{},[8340],{"type":155,"value":745},{"type":150,"tag":209,"props":8342,"children":8343},{"mathvariant":1618},[8344],{"type":155,"value":4740},{"type":150,"tag":209,"props":8346,"children":8347},{},[8348],{"type":155,"value":7913},{"type":150,"tag":271,"props":8350,"children":8351},{"encoding":273},[8352],{"type":155,"value":8353},"h\u002Fg",{"type":150,"tag":181,"props":8355,"children":8357},{"className":8356,"ariaHidden":223},[280],[8358],{"type":150,"tag":181,"props":8359,"children":8361},{"className":8360},[285],[8362,8366,8371,8376],{"type":150,"tag":181,"props":8363,"children":8365},{"className":8364,"style":819},[290],[],{"type":150,"tag":181,"props":8367,"children":8369},{"className":8368},[296,301],[8370],{"type":155,"value":745},{"type":150,"tag":181,"props":8372,"children":8374},{"className":8373},[296],[8375],{"type":155,"value":4740},{"type":150,"tag":181,"props":8377,"children":8379},{"className":8378,"style":2151},[296,301],[8380],{"type":155,"value":7913},{"type":155,"value":8382}," 个 head 一组，接着 ",{"type":150,"tag":181,"props":8384,"children":8386},{"className":8385},[184],[8387,8416],{"type":150,"tag":181,"props":8388,"children":8390},{"className":8389},[189],[8391],{"type":150,"tag":192,"props":8392,"children":8393},{"xmlns":194},[8394],{"type":150,"tag":197,"props":8395,"children":8396},{},[8397,8412],{"type":150,"tag":201,"props":8398,"children":8399},{},[8400,8404,8408],{"type":150,"tag":209,"props":8401,"children":8402},{},[8403],{"type":155,"value":745},{"type":150,"tag":209,"props":8405,"children":8406},{"mathvariant":1618},[8407],{"type":155,"value":4740},{"type":150,"tag":209,"props":8409,"children":8410},{},[8411],{"type":155,"value":7913},{"type":150,"tag":271,"props":8413,"children":8414},{"encoding":273},[8415],{"type":155,"value":8353},{"type":150,"tag":181,"props":8417,"children":8419},{"className":8418,"ariaHidden":223},[280],[8420],{"type":150,"tag":181,"props":8421,"children":8423},{"className":8422},[285],[8424,8428,8433,8438],{"type":150,"tag":181,"props":8425,"children":8427},{"className":8426,"style":819},[290],[],{"type":150,"tag":181,"props":8429,"children":8431},{"className":8430},[296,301],[8432],{"type":155,"value":745},{"type":150,"tag":181,"props":8434,"children":8436},{"className":8435},[296],[8437],{"type":155,"value":4740},{"type":150,"tag":181,"props":8439,"children":8441},{"className":8440,"style":2151},[296,301],[8442],{"type":155,"value":7913},{"type":155,"value":8444}," 个一组，以此类推。）于是 attention 写成：",{"type":150,"tag":181,"props":8446,"children":8448},{"className":8447},[1003],[8449],{"type":150,"tag":181,"props":8450,"children":8452},{"className":8451},[184],[8453,8684],{"type":150,"tag":181,"props":8454,"children":8456},{"className":8455},[189],[8457],{"type":150,"tag":192,"props":8458,"children":8459},{"xmlns":194,"display":1015},[8460],{"type":150,"tag":197,"props":8461,"children":8462},{},[8463,8679],{"type":150,"tag":201,"props":8464,"children":8465},{},[8466,8492,8496,8535],{"type":150,"tag":1046,"props":8467,"children":8468},{},[8469,8473,8477],{"type":150,"tag":209,"props":8470,"children":8471},{},[8472],{"type":155,"value":1030},{"type":150,"tag":209,"props":8474,"children":8475},{},[8476],{"type":155,"value":1035},{"type":150,"tag":201,"props":8478,"children":8479},{},[8480,8484,8488],{"type":150,"tag":221,"props":8481,"children":8482},{"stretchy":789},[8483],{"type":155,"value":792},{"type":150,"tag":209,"props":8485,"children":8486},{},[8487],{"type":155,"value":797},{"type":150,"tag":221,"props":8489,"children":8490},{"stretchy":789},[8491],{"type":155,"value":802},{"type":150,"tag":221,"props":8493,"children":8494},{},[8495],{"type":155,"value":865},{"type":150,"tag":201,"props":8497,"children":8498},{},[8499,8503,8507,8511,8515,8519,8523,8527,8531],{"type":150,"tag":209,"props":8500,"children":8501},{"mathvariant":1618},[8502],{"type":155,"value":1621},{"type":150,"tag":209,"props":8504,"children":8505},{"mathvariant":1618},[8506],{"type":155,"value":1035},{"type":150,"tag":209,"props":8508,"children":8509},{"mathvariant":1618},[8510],{"type":155,"value":1035},{"type":150,"tag":209,"props":8512,"children":8513},{"mathvariant":1618},[8514],{"type":155,"value":1634},{"type":150,"tag":209,"props":8516,"children":8517},{"mathvariant":1618},[8518],{"type":155,"value":1639},{"type":150,"tag":209,"props":8520,"children":8521},{"mathvariant":1618},[8522],{"type":155,"value":1035},{"type":150,"tag":209,"props":8524,"children":8525},{"mathvariant":1618},[8526],{"type":155,"value":558},{"type":150,"tag":209,"props":8528,"children":8529},{"mathvariant":1618},[8530],{"type":155,"value":1030},{"type":150,"tag":209,"props":8532,"children":8533},{"mathvariant":1618},[8534],{"type":155,"value":1639},{"type":150,"tag":201,"props":8536,"children":8537},{},[8538,8542,8568,8572,8577,8622,8626,8630,8675],{"type":150,"tag":221,"props":8539,"children":8540},{"fence":223},[8541],{"type":155,"value":792},{"type":150,"tag":1046,"props":8543,"children":8544},{},[8545,8549,8553],{"type":150,"tag":209,"props":8546,"children":8547},{},[8548],{"type":155,"value":1667},{"type":150,"tag":209,"props":8550,"children":8551},{},[8552],{"type":155,"value":1035},{"type":150,"tag":201,"props":8554,"children":8555},{},[8556,8560,8564],{"type":150,"tag":221,"props":8557,"children":8558},{"stretchy":789},[8559],{"type":155,"value":792},{"type":150,"tag":209,"props":8561,"children":8562},{},[8563],{"type":155,"value":797},{"type":150,"tag":221,"props":8565,"children":8566},{"stretchy":789},[8567],{"type":155,"value":802},{"type":150,"tag":221,"props":8569,"children":8570},{"separator":223},[8571],{"type":155,"value":226},{"type":150,"tag":249,"props":8573,"children":8574},{},[8575],{"type":155,"value":8576}," ",{"type":150,"tag":1046,"props":8578,"children":8579},{},[8580,8584,8595],{"type":150,"tag":209,"props":8581,"children":8582},{},[8583],{"type":155,"value":1698},{"type":150,"tag":201,"props":8585,"children":8586},{},[8587,8591],{"type":150,"tag":221,"props":8588,"children":8589},{},[8590],{"type":155,"value":1706},{"type":150,"tag":209,"props":8592,"children":8593},{},[8594],{"type":155,"value":1035},{"type":150,"tag":201,"props":8596,"children":8597},{},[8598,8602,8606,8610,8614,8618],{"type":150,"tag":221,"props":8599,"children":8600},{"stretchy":789},[8601],{"type":155,"value":792},{"type":150,"tag":209,"props":8603,"children":8604},{},[8605],{"type":155,"value":8007},{"type":150,"tag":221,"props":8607,"children":8608},{"stretchy":789},[8609],{"type":155,"value":792},{"type":150,"tag":209,"props":8611,"children":8612},{},[8613],{"type":155,"value":797},{"type":150,"tag":221,"props":8615,"children":8616},{"stretchy":789},[8617],{"type":155,"value":802},{"type":150,"tag":221,"props":8619,"children":8620},{"stretchy":789},[8621],{"type":155,"value":802},{"type":150,"tag":221,"props":8623,"children":8624},{"separator":223},[8625],{"type":155,"value":226},{"type":150,"tag":249,"props":8627,"children":8628},{},[8629],{"type":155,"value":8576},{"type":150,"tag":1046,"props":8631,"children":8632},{},[8633,8637,8648],{"type":150,"tag":209,"props":8634,"children":8635},{},[8636],{"type":155,"value":1737},{"type":150,"tag":201,"props":8638,"children":8639},{},[8640,8644],{"type":150,"tag":221,"props":8641,"children":8642},{},[8643],{"type":155,"value":1706},{"type":150,"tag":209,"props":8645,"children":8646},{},[8647],{"type":155,"value":1035},{"type":150,"tag":201,"props":8649,"children":8650},{},[8651,8655,8659,8663,8667,8671],{"type":150,"tag":221,"props":8652,"children":8653},{"stretchy":789},[8654],{"type":155,"value":792},{"type":150,"tag":209,"props":8656,"children":8657},{},[8658],{"type":155,"value":8007},{"type":150,"tag":221,"props":8660,"children":8661},{"stretchy":789},[8662],{"type":155,"value":792},{"type":150,"tag":209,"props":8664,"children":8665},{},[8666],{"type":155,"value":797},{"type":150,"tag":221,"props":8668,"children":8669},{"stretchy":789},[8670],{"type":155,"value":802},{"type":150,"tag":221,"props":8672,"children":8673},{"stretchy":789},[8674],{"type":155,"value":802},{"type":150,"tag":221,"props":8676,"children":8677},{"fence":223},[8678],{"type":155,"value":802},{"type":150,"tag":271,"props":8680,"children":8681},{"encoding":273},[8682],{"type":155,"value":8683},"o_t^{(s)} = \\mathrm{Attention}\\left(q_t^{(s)},\\ k^{(\\gamma(s))}_{\\le t},\\ v^{(\\gamma(s))}_{\\le t}\\right)",{"type":150,"tag":181,"props":8685,"children":8687},{"className":8686,"ariaHidden":223},[280],[8688,8796],{"type":150,"tag":181,"props":8689,"children":8691},{"className":8690},[285],[8692,8696,8783,8787,8792],{"type":150,"tag":181,"props":8693,"children":8695},{"className":8694,"style":2018},[290],[],{"type":150,"tag":181,"props":8697,"children":8699},{"className":8698},[296],[8700,8705],{"type":150,"tag":181,"props":8701,"children":8703},{"className":8702},[296,301],[8704],{"type":155,"value":1030},{"type":150,"tag":181,"props":8706,"children":8708},{"className":8707},[307],[8709],{"type":150,"tag":181,"props":8710,"children":8712},{"className":8711},[312,313],[8713,8772],{"type":150,"tag":181,"props":8714,"children":8716},{"className":8715},[318],[8717,8767],{"type":150,"tag":181,"props":8718,"children":8720},{"className":8719,"style":1276},[323],[8721,8737],{"type":150,"tag":181,"props":8722,"children":8723},{"style":1280},[8724,8728],{"type":150,"tag":181,"props":8725,"children":8727},{"className":8726,"style":334},[333],[],{"type":150,"tag":181,"props":8729,"children":8731},{"className":8730},[339,340,341,342],[8732],{"type":150,"tag":181,"props":8733,"children":8735},{"className":8734},[296,301,342],[8736],{"type":155,"value":1035},{"type":150,"tag":181,"props":8738,"children":8739},{"style":1297},[8740,8744],{"type":150,"tag":181,"props":8741,"children":8743},{"className":8742,"style":334},[333],[],{"type":150,"tag":181,"props":8745,"children":8747},{"className":8746},[339,340,341,342],[8748],{"type":150,"tag":181,"props":8749,"children":8751},{"className":8750},[296,342],[8752,8757,8762],{"type":150,"tag":181,"props":8753,"children":8755},{"className":8754},[824,342],[8756],{"type":155,"value":792},{"type":150,"tag":181,"props":8758,"children":8760},{"className":8759},[296,301,342],[8761],{"type":155,"value":797},{"type":150,"tag":181,"props":8763,"children":8765},{"className":8764},[835,342],[8766],{"type":155,"value":802},{"type":150,"tag":181,"props":8768,"children":8770},{"className":8769},[352],[8771],{"type":155,"value":355},{"type":150,"tag":181,"props":8773,"children":8775},{"className":8774},[318],[8776],{"type":150,"tag":181,"props":8777,"children":8779},{"className":8778,"style":1338},[323],[8780],{"type":150,"tag":181,"props":8781,"children":8782},{},[],{"type":150,"tag":181,"props":8784,"children":8786},{"className":8785,"style":658},[377],[],{"type":150,"tag":181,"props":8788,"children":8790},{"className":8789},[663],[8791],{"type":155,"value":865},{"type":150,"tag":181,"props":8793,"children":8795},{"className":8794,"style":658},[377],[],{"type":150,"tag":181,"props":8797,"children":8799},{"className":8798},[285],[8800,8805,8814,8818],{"type":150,"tag":181,"props":8801,"children":8804},{"className":8802,"style":8803},[290],"height:1.8em;vertical-align:-0.65em;",[],{"type":150,"tag":181,"props":8806,"children":8808},{"className":8807},[296],[8809],{"type":150,"tag":181,"props":8810,"children":8812},{"className":8811},[296,2136],[8813],{"type":155,"value":12},{"type":150,"tag":181,"props":8815,"children":8817},{"className":8816,"style":378},[377],[],{"type":150,"tag":181,"props":8819,"children":8821},{"className":8820},[449],[8822,8831,8918,8923,8928,8932,9039,9044,9049,9053,9159],{"type":150,"tag":181,"props":8823,"children":8825},{"className":8824,"style":8199},[824,8198],[8826],{"type":150,"tag":181,"props":8827,"children":8829},{"className":8828},[8204,8205],[8830],{"type":155,"value":792},{"type":150,"tag":181,"props":8832,"children":8834},{"className":8833},[296],[8835,8840],{"type":150,"tag":181,"props":8836,"children":8838},{"className":8837,"style":2151},[296,301],[8839],{"type":155,"value":1667},{"type":150,"tag":181,"props":8841,"children":8843},{"className":8842},[307],[8844],{"type":150,"tag":181,"props":8845,"children":8847},{"className":8846},[312,313],[8848,8907],{"type":150,"tag":181,"props":8849,"children":8851},{"className":8850},[318],[8852,8902],{"type":150,"tag":181,"props":8853,"children":8855},{"className":8854,"style":1276},[323],[8856,8872],{"type":150,"tag":181,"props":8857,"children":8858},{"style":2172},[8859,8863],{"type":150,"tag":181,"props":8860,"children":8862},{"className":8861,"style":334},[333],[],{"type":150,"tag":181,"props":8864,"children":8866},{"className":8865},[339,340,341,342],[8867],{"type":150,"tag":181,"props":8868,"children":8870},{"className":8869},[296,301,342],[8871],{"type":155,"value":1035},{"type":150,"tag":181,"props":8873,"children":8874},{"style":1297},[8875,8879],{"type":150,"tag":181,"props":8876,"children":8878},{"className":8877,"style":334},[333],[],{"type":150,"tag":181,"props":8880,"children":8882},{"className":8881},[339,340,341,342],[8883],{"type":150,"tag":181,"props":8884,"children":8886},{"className":8885},[296,342],[8887,8892,8897],{"type":150,"tag":181,"props":8888,"children":8890},{"className":8889},[824,342],[8891],{"type":155,"value":792},{"type":150,"tag":181,"props":8893,"children":8895},{"className":8894},[296,301,342],[8896],{"type":155,"value":797},{"type":150,"tag":181,"props":8898,"children":8900},{"className":8899},[835,342],[8901],{"type":155,"value":802},{"type":150,"tag":181,"props":8903,"children":8905},{"className":8904},[352],[8906],{"type":155,"value":355},{"type":150,"tag":181,"props":8908,"children":8910},{"className":8909},[318],[8911],{"type":150,"tag":181,"props":8912,"children":8914},{"className":8913,"style":1338},[323],[8915],{"type":150,"tag":181,"props":8916,"children":8917},{},[],{"type":150,"tag":181,"props":8919,"children":8921},{"className":8920},[371],[8922],{"type":155,"value":226},{"type":150,"tag":181,"props":8924,"children":8926},{"className":8925},[377],[8927],{"type":155,"value":8576},{"type":150,"tag":181,"props":8929,"children":8931},{"className":8930,"style":378},[377],[],{"type":150,"tag":181,"props":8933,"children":8935},{"className":8934},[296],[8936,8941],{"type":150,"tag":181,"props":8937,"children":8939},{"className":8938,"style":2249},[296,301],[8940],{"type":155,"value":1698},{"type":150,"tag":181,"props":8942,"children":8944},{"className":8943},[307],[8945],{"type":150,"tag":181,"props":8946,"children":8948},{"className":8947},[312,313],[8949,9028],{"type":150,"tag":181,"props":8950,"children":8952},{"className":8951},[318],[8953,9023],{"type":150,"tag":181,"props":8954,"children":8956},{"className":8955,"style":1276},[323],[8957,8982],{"type":150,"tag":181,"props":8958,"children":8959},{"style":2270},[8960,8964],{"type":150,"tag":181,"props":8961,"children":8963},{"className":8962,"style":334},[333],[],{"type":150,"tag":181,"props":8965,"children":8967},{"className":8966},[339,340,341,342],[8968],{"type":150,"tag":181,"props":8969,"children":8971},{"className":8970},[296,342],[8972,8977],{"type":150,"tag":181,"props":8973,"children":8975},{"className":8974},[663,342],[8976],{"type":155,"value":1706},{"type":150,"tag":181,"props":8978,"children":8980},{"className":8979},[296,301,342],[8981],{"type":155,"value":1035},{"type":150,"tag":181,"props":8983,"children":8984},{"style":1297},[8985,8989],{"type":150,"tag":181,"props":8986,"children":8988},{"className":8987,"style":334},[333],[],{"type":150,"tag":181,"props":8990,"children":8992},{"className":8991},[339,340,341,342],[8993],{"type":150,"tag":181,"props":8994,"children":8996},{"className":8995},[296,342],[8997,9002,9007,9012,9017],{"type":150,"tag":181,"props":8998,"children":9000},{"className":8999},[824,342],[9001],{"type":155,"value":792},{"type":150,"tag":181,"props":9003,"children":9005},{"className":9004,"style":8040},[296,301,342],[9006],{"type":155,"value":8007},{"type":150,"tag":181,"props":9008,"children":9010},{"className":9009},[824,342],[9011],{"type":155,"value":792},{"type":150,"tag":181,"props":9013,"children":9015},{"className":9014},[296,301,342],[9016],{"type":155,"value":797},{"type":150,"tag":181,"props":9018,"children":9020},{"className":9019},[835,342],[9021],{"type":155,"value":9022},"))",{"type":150,"tag":181,"props":9024,"children":9026},{"className":9025},[352],[9027],{"type":155,"value":355},{"type":150,"tag":181,"props":9029,"children":9031},{"className":9030},[318],[9032],{"type":150,"tag":181,"props":9033,"children":9035},{"className":9034,"style":2336},[323],[9036],{"type":150,"tag":181,"props":9037,"children":9038},{},[],{"type":150,"tag":181,"props":9040,"children":9042},{"className":9041},[371],[9043],{"type":155,"value":226},{"type":150,"tag":181,"props":9045,"children":9047},{"className":9046},[377],[9048],{"type":155,"value":8576},{"type":150,"tag":181,"props":9050,"children":9052},{"className":9051,"style":378},[377],[],{"type":150,"tag":181,"props":9054,"children":9056},{"className":9055},[296],[9057,9062],{"type":150,"tag":181,"props":9058,"children":9060},{"className":9059,"style":2151},[296,301],[9061],{"type":155,"value":1737},{"type":150,"tag":181,"props":9063,"children":9065},{"className":9064},[307],[9066],{"type":150,"tag":181,"props":9067,"children":9069},{"className":9068},[312,313],[9070,9148],{"type":150,"tag":181,"props":9071,"children":9073},{"className":9072},[318],[9074,9143],{"type":150,"tag":181,"props":9075,"children":9077},{"className":9076,"style":1276},[323],[9078,9103],{"type":150,"tag":181,"props":9079,"children":9080},{"style":2377},[9081,9085],{"type":150,"tag":181,"props":9082,"children":9084},{"className":9083,"style":334},[333],[],{"type":150,"tag":181,"props":9086,"children":9088},{"className":9087},[339,340,341,342],[9089],{"type":150,"tag":181,"props":9090,"children":9092},{"className":9091},[296,342],[9093,9098],{"type":150,"tag":181,"props":9094,"children":9096},{"className":9095},[663,342],[9097],{"type":155,"value":1706},{"type":150,"tag":181,"props":9099,"children":9101},{"className":9100},[296,301,342],[9102],{"type":155,"value":1035},{"type":150,"tag":181,"props":9104,"children":9105},{"style":1297},[9106,9110],{"type":150,"tag":181,"props":9107,"children":9109},{"className":9108,"style":334},[333],[],{"type":150,"tag":181,"props":9111,"children":9113},{"className":9112},[339,340,341,342],[9114],{"type":150,"tag":181,"props":9115,"children":9117},{"className":9116},[296,342],[9118,9123,9128,9133,9138],{"type":150,"tag":181,"props":9119,"children":9121},{"className":9120},[824,342],[9122],{"type":155,"value":792},{"type":150,"tag":181,"props":9124,"children":9126},{"className":9125,"style":8040},[296,301,342],[9127],{"type":155,"value":8007},{"type":150,"tag":181,"props":9129,"children":9131},{"className":9130},[824,342],[9132],{"type":155,"value":792},{"type":150,"tag":181,"props":9134,"children":9136},{"className":9135},[296,301,342],[9137],{"type":155,"value":797},{"type":150,"tag":181,"props":9139,"children":9141},{"className":9140},[835,342],[9142],{"type":155,"value":9022},{"type":150,"tag":181,"props":9144,"children":9146},{"className":9145},[352],[9147],{"type":155,"value":355},{"type":150,"tag":181,"props":9149,"children":9151},{"className":9150},[318],[9152],{"type":150,"tag":181,"props":9153,"children":9155},{"className":9154,"style":2336},[323],[9156],{"type":150,"tag":181,"props":9157,"children":9158},{},[],{"type":150,"tag":181,"props":9160,"children":9162},{"className":9161,"style":8199},[835,8198],[9163],{"type":150,"tag":181,"props":9164,"children":9166},{"className":9165},[8204,8205],[9167],{"type":155,"value":802},{"type":150,"tag":151,"props":9169,"children":9170},{},[9171,9173,9251,9253,9330,9332,9409,9411,9488],{"type":155,"value":9172},"注意 Q 仍然是每个 head 一份，只有 K、V 被分组共用。这个不对称就是 GQA 的全部内容。举个具体的数。",{"type":150,"tag":181,"props":9174,"children":9176},{"className":9175},[184],[9177,9208],{"type":150,"tag":181,"props":9178,"children":9180},{"className":9179},[189],[9181],{"type":150,"tag":192,"props":9182,"children":9183},{"xmlns":194},[9184],{"type":150,"tag":197,"props":9185,"children":9186},{},[9187,9203],{"type":150,"tag":201,"props":9188,"children":9189},{},[9190,9194,9198],{"type":150,"tag":209,"props":9191,"children":9192},{},[9193],{"type":155,"value":745},{"type":150,"tag":221,"props":9195,"children":9196},{},[9197],{"type":155,"value":865},{"type":150,"tag":215,"props":9199,"children":9200},{},[9201],{"type":155,"value":9202},"8",{"type":150,"tag":271,"props":9204,"children":9205},{"encoding":273},[9206],{"type":155,"value":9207},"h = 8",{"type":150,"tag":181,"props":9209,"children":9211},{"className":9210,"ariaHidden":223},[280],[9212,9238],{"type":150,"tag":181,"props":9213,"children":9215},{"className":9214},[285],[9216,9220,9225,9229,9234],{"type":150,"tag":181,"props":9217,"children":9219},{"className":9218,"style":761},[290],[],{"type":150,"tag":181,"props":9221,"children":9223},{"className":9222},[296,301],[9224],{"type":155,"value":745},{"type":150,"tag":181,"props":9226,"children":9228},{"className":9227,"style":658},[377],[],{"type":150,"tag":181,"props":9230,"children":9232},{"className":9231},[663],[9233],{"type":155,"value":865},{"type":150,"tag":181,"props":9235,"children":9237},{"className":9236,"style":658},[377],[],{"type":150,"tag":181,"props":9239,"children":9241},{"className":9240},[285],[9242,9246],{"type":150,"tag":181,"props":9243,"children":9245},{"className":9244,"style":5012},[290],[],{"type":150,"tag":181,"props":9247,"children":9249},{"className":9248},[296],[9250],{"type":155,"value":9202},{"type":155,"value":9252},"、",{"type":150,"tag":181,"props":9254,"children":9256},{"className":9255},[184],[9257,9287],{"type":150,"tag":181,"props":9258,"children":9260},{"className":9259},[189],[9261],{"type":150,"tag":192,"props":9262,"children":9263},{"xmlns":194},[9264],{"type":150,"tag":197,"props":9265,"children":9266},{},[9267,9282],{"type":150,"tag":201,"props":9268,"children":9269},{},[9270,9274,9278],{"type":150,"tag":209,"props":9271,"children":9272},{},[9273],{"type":155,"value":7913},{"type":150,"tag":221,"props":9275,"children":9276},{},[9277],{"type":155,"value":865},{"type":150,"tag":215,"props":9279,"children":9280},{},[9281],{"type":155,"value":238},{"type":150,"tag":271,"props":9283,"children":9284},{"encoding":273},[9285],{"type":155,"value":9286},"g = 2",{"type":150,"tag":181,"props":9288,"children":9290},{"className":9289,"ariaHidden":223},[280],[9291,9317],{"type":150,"tag":181,"props":9292,"children":9294},{"className":9293},[285],[9295,9299,9304,9308,9313],{"type":150,"tag":181,"props":9296,"children":9298},{"className":9297,"style":291},[290],[],{"type":150,"tag":181,"props":9300,"children":9302},{"className":9301,"style":2151},[296,301],[9303],{"type":155,"value":7913},{"type":150,"tag":181,"props":9305,"children":9307},{"className":9306,"style":658},[377],[],{"type":150,"tag":181,"props":9309,"children":9311},{"className":9310},[663],[9312],{"type":155,"value":865},{"type":150,"tag":181,"props":9314,"children":9316},{"className":9315,"style":658},[377],[],{"type":150,"tag":181,"props":9318,"children":9320},{"className":9319},[285],[9321,9325],{"type":150,"tag":181,"props":9322,"children":9324},{"className":9323,"style":5012},[290],[],{"type":150,"tag":181,"props":9326,"children":9328},{"className":9327},[296],[9329],{"type":155,"value":238},{"type":155,"value":9331}," 时，第 1 到第 4 个 head 共用一对 K、V，第 5 到第 8 个 head 共用另一对。",{"type":150,"tag":181,"props":9333,"children":9335},{"className":9334},[184],[9336,9366],{"type":150,"tag":181,"props":9337,"children":9339},{"className":9338},[189],[9340],{"type":150,"tag":192,"props":9341,"children":9342},{"xmlns":194},[9343],{"type":150,"tag":197,"props":9344,"children":9345},{},[9346,9361],{"type":150,"tag":201,"props":9347,"children":9348},{},[9349,9353,9357],{"type":150,"tag":209,"props":9350,"children":9351},{},[9352],{"type":155,"value":7913},{"type":150,"tag":221,"props":9354,"children":9355},{},[9356],{"type":155,"value":865},{"type":150,"tag":209,"props":9358,"children":9359},{},[9360],{"type":155,"value":745},{"type":150,"tag":271,"props":9362,"children":9363},{"encoding":273},[9364],{"type":155,"value":9365},"g = h",{"type":150,"tag":181,"props":9367,"children":9369},{"className":9368,"ariaHidden":223},[280],[9370,9396],{"type":150,"tag":181,"props":9371,"children":9373},{"className":9372},[285],[9374,9378,9383,9387,9392],{"type":150,"tag":181,"props":9375,"children":9377},{"className":9376,"style":291},[290],[],{"type":150,"tag":181,"props":9379,"children":9381},{"className":9380,"style":2151},[296,301],[9382],{"type":155,"value":7913},{"type":150,"tag":181,"props":9384,"children":9386},{"className":9385,"style":658},[377],[],{"type":150,"tag":181,"props":9388,"children":9390},{"className":9389},[663],[9391],{"type":155,"value":865},{"type":150,"tag":181,"props":9393,"children":9395},{"className":9394,"style":658},[377],[],{"type":150,"tag":181,"props":9397,"children":9399},{"className":9398},[285],[9400,9404],{"type":150,"tag":181,"props":9401,"children":9403},{"className":9402,"style":761},[290],[],{"type":150,"tag":181,"props":9405,"children":9407},{"className":9406},[296,301],[9408],{"type":155,"value":745},{"type":155,"value":9410}," 时每个 head 各自一对，退化成 MHA；",{"type":150,"tag":181,"props":9412,"children":9414},{"className":9413},[184],[9415,9445],{"type":150,"tag":181,"props":9416,"children":9418},{"className":9417},[189],[9419],{"type":150,"tag":192,"props":9420,"children":9421},{"xmlns":194},[9422],{"type":150,"tag":197,"props":9423,"children":9424},{},[9425,9440],{"type":150,"tag":201,"props":9426,"children":9427},{},[9428,9432,9436],{"type":150,"tag":209,"props":9429,"children":9430},{},[9431],{"type":155,"value":7913},{"type":150,"tag":221,"props":9433,"children":9434},{},[9435],{"type":155,"value":865},{"type":150,"tag":215,"props":9437,"children":9438},{},[9439],{"type":155,"value":219},{"type":150,"tag":271,"props":9441,"children":9442},{"encoding":273},[9443],{"type":155,"value":9444},"g = 1",{"type":150,"tag":181,"props":9446,"children":9448},{"className":9447,"ariaHidden":223},[280],[9449,9475],{"type":150,"tag":181,"props":9450,"children":9452},{"className":9451},[285],[9453,9457,9462,9466,9471],{"type":150,"tag":181,"props":9454,"children":9456},{"className":9455,"style":291},[290],[],{"type":150,"tag":181,"props":9458,"children":9460},{"className":9459,"style":2151},[296,301],[9461],{"type":155,"value":7913},{"type":150,"tag":181,"props":9463,"children":9465},{"className":9464,"style":658},[377],[],{"type":150,"tag":181,"props":9467,"children":9469},{"className":9468},[663],[9470],{"type":155,"value":865},{"type":150,"tag":181,"props":9472,"children":9474},{"className":9473,"style":658},[377],[],{"type":150,"tag":181,"props":9476,"children":9478},{"className":9477},[285],[9479,9483],{"type":150,"tag":181,"props":9480,"children":9482},{"className":9481,"style":5012},[290],[],{"type":150,"tag":181,"props":9484,"children":9486},{"className":9485},[296],[9487],{"type":155,"value":219},{"type":155,"value":9489}," 时所有 head 共用一对，退化成 MQA。",{"type":150,"tag":151,"props":9491,"children":9492},{},[9493],{"type":155,"value":9494},"它提供了 MHA 到 MQA 的自然过渡，压缩率不如 MQA，但自由度和效果保证更好。",{"type":150,"tag":151,"props":9496,"children":9497},{},[9498,9500,9577,9579,9655],{"type":155,"value":9499},"llama2\u002F3-70B 用的 ",{"type":150,"tag":181,"props":9501,"children":9503},{"className":9502},[184],[9504,9534],{"type":150,"tag":181,"props":9505,"children":9507},{"className":9506},[189],[9508],{"type":150,"tag":192,"props":9509,"children":9510},{"xmlns":194},[9511],{"type":150,"tag":197,"props":9512,"children":9513},{},[9514,9529],{"type":150,"tag":201,"props":9515,"children":9516},{},[9517,9521,9525],{"type":150,"tag":209,"props":9518,"children":9519},{},[9520],{"type":155,"value":7913},{"type":150,"tag":221,"props":9522,"children":9523},{},[9524],{"type":155,"value":865},{"type":150,"tag":215,"props":9526,"children":9527},{},[9528],{"type":155,"value":9202},{"type":150,"tag":271,"props":9530,"children":9531},{"encoding":273},[9532],{"type":155,"value":9533},"g = 8",{"type":150,"tag":181,"props":9535,"children":9537},{"className":9536,"ariaHidden":223},[280],[9538,9564],{"type":150,"tag":181,"props":9539,"children":9541},{"className":9540},[285],[9542,9546,9551,9555,9560],{"type":150,"tag":181,"props":9543,"children":9545},{"className":9544,"style":291},[290],[],{"type":150,"tag":181,"props":9547,"children":9549},{"className":9548,"style":2151},[296,301],[9550],{"type":155,"value":7913},{"type":150,"tag":181,"props":9552,"children":9554},{"className":9553,"style":658},[377],[],{"type":150,"tag":181,"props":9556,"children":9558},{"className":9557},[663],[9559],{"type":155,"value":865},{"type":150,"tag":181,"props":9561,"children":9563},{"className":9562,"style":658},[377],[],{"type":150,"tag":181,"props":9565,"children":9567},{"className":9566},[285],[9568,9572],{"type":150,"tag":181,"props":9569,"children":9571},{"className":9570,"style":5012},[290],[],{"type":150,"tag":181,"props":9573,"children":9575},{"className":9574},[296],[9576],{"type":155,"value":9202},{"type":155,"value":9578},"。早年有个很整齐的解释：70B 这个体量放不进单卡，得单机 8 卡，而 attention 的 head 本来就是独立运算再拼接的，",{"type":150,"tag":181,"props":9580,"children":9582},{"className":9581},[184],[9583,9612],{"type":150,"tag":181,"props":9584,"children":9586},{"className":9585},[189],[9587],{"type":150,"tag":192,"props":9588,"children":9589},{"xmlns":194},[9590],{"type":150,"tag":197,"props":9591,"children":9592},{},[9593,9608],{"type":150,"tag":201,"props":9594,"children":9595},{},[9596,9600,9604],{"type":150,"tag":209,"props":9597,"children":9598},{},[9599],{"type":155,"value":7913},{"type":150,"tag":221,"props":9601,"children":9602},{},[9603],{"type":155,"value":865},{"type":150,"tag":215,"props":9605,"children":9606},{},[9607],{"type":155,"value":9202},{"type":150,"tag":271,"props":9609,"children":9610},{"encoding":273},[9611],{"type":155,"value":9533},{"type":150,"tag":181,"props":9613,"children":9615},{"className":9614,"ariaHidden":223},[280],[9616,9642],{"type":150,"tag":181,"props":9617,"children":9619},{"className":9618},[285],[9620,9624,9629,9633,9638],{"type":150,"tag":181,"props":9621,"children":9623},{"className":9622,"style":291},[290],[],{"type":150,"tag":181,"props":9625,"children":9627},{"className":9626,"style":2151},[296,301],[9628],{"type":155,"value":7913},{"type":150,"tag":181,"props":9630,"children":9632},{"className":9631,"style":658},[377],[],{"type":150,"tag":181,"props":9634,"children":9636},{"className":9635},[663],[9637],{"type":155,"value":865},{"type":150,"tag":181,"props":9639,"children":9641},{"className":9640,"style":658},[377],[],{"type":150,"tag":181,"props":9643,"children":9645},{"className":9644},[285],[9646,9650],{"type":150,"tag":181,"props":9647,"children":9649},{"className":9648,"style":5012},[290],[],{"type":150,"tag":181,"props":9651,"children":9653},{"className":9652},[296],[9654],{"type":155,"value":9202},{"type":155,"value":9656}," 正好让每张卡负责一组 K、V 对应的 head，把卡间通信压到最小。这个解释现在不太站得住了。量化让更大的模型能进单卡，MoE 让 activated 参数远小于总参数，跨机推理也早就不是禁区。",{"type":150,"tag":151,"props":9658,"children":9659},{},[9660,9662,9671],{"type":155,"value":9661},"两个现成的例子：Qwen3.8-Flash-Next 是 125B 的模型、每个 token 只激活 6B，在 B200 上用 TP4 就能跑（",{"type":150,"tag":9663,"props":9664,"children":9668},"a",{"href":9665,"rel":9666},"https:\u002F\u002Fwww.lmsys.org\u002Fblog\u002F2026-08-26-qwen-flash-next",[9667],"nofollow",[9669],{"type":155,"value":9670},"SGLang 的 day-0 支持",{"type":155,"value":9672},"）；DeepSeek-V4-Flash-FP8 同样 TP4 就能跑，H200 上就够。",{"type":150,"tag":151,"props":9674,"children":9675},{},[9676,9678,9720,9722,9798],{"type":155,"value":9677},"所以更准确的说法是：",{"type":150,"tag":181,"props":9679,"children":9681},{"className":9680},[184],[9682,9703],{"type":150,"tag":181,"props":9683,"children":9685},{"className":9684},[189],[9686],{"type":150,"tag":192,"props":9687,"children":9688},{"xmlns":194},[9689],{"type":150,"tag":197,"props":9690,"children":9691},{},[9692,9699],{"type":150,"tag":201,"props":9693,"children":9694},{},[9695],{"type":150,"tag":209,"props":9696,"children":9697},{},[9698],{"type":155,"value":7913},{"type":150,"tag":271,"props":9700,"children":9701},{"encoding":273},[9702],{"type":155,"value":7913},{"type":150,"tag":181,"props":9704,"children":9706},{"className":9705,"ariaHidden":223},[280],[9707],{"type":150,"tag":181,"props":9708,"children":9710},{"className":9709},[285],[9711,9715],{"type":150,"tag":181,"props":9712,"children":9714},{"className":9713,"style":291},[290],[],{"type":150,"tag":181,"props":9716,"children":9718},{"className":9717,"style":2151},[296,301],[9719],{"type":155,"value":7913},{"type":155,"value":9721}," 跟并行度对齐仍然是个好主意，但\"必须单机 8 卡\"这种硬约束已经没有了。",{"type":150,"tag":181,"props":9723,"children":9725},{"className":9724},[184],[9726,9755],{"type":150,"tag":181,"props":9727,"children":9729},{"className":9728},[189],[9730],{"type":150,"tag":192,"props":9731,"children":9732},{"xmlns":194},[9733],{"type":150,"tag":197,"props":9734,"children":9735},{},[9736,9751],{"type":150,"tag":201,"props":9737,"children":9738},{},[9739,9743,9747],{"type":150,"tag":209,"props":9740,"children":9741},{},[9742],{"type":155,"value":7913},{"type":150,"tag":221,"props":9744,"children":9745},{},[9746],{"type":155,"value":865},{"type":150,"tag":215,"props":9748,"children":9749},{},[9750],{"type":155,"value":9202},{"type":150,"tag":271,"props":9752,"children":9753},{"encoding":273},[9754],{"type":155,"value":9533},{"type":150,"tag":181,"props":9756,"children":9758},{"className":9757,"ariaHidden":223},[280],[9759,9785],{"type":150,"tag":181,"props":9760,"children":9762},{"className":9761},[285],[9763,9767,9772,9776,9781],{"type":150,"tag":181,"props":9764,"children":9766},{"className":9765,"style":291},[290],[],{"type":150,"tag":181,"props":9768,"children":9770},{"className":9769,"style":2151},[296,301],[9771],{"type":155,"value":7913},{"type":150,"tag":181,"props":9773,"children":9775},{"className":9774,"style":658},[377],[],{"type":150,"tag":181,"props":9777,"children":9779},{"className":9778},[663],[9780],{"type":155,"value":865},{"type":150,"tag":181,"props":9782,"children":9784},{"className":9783,"style":658},[377],[],{"type":150,"tag":181,"props":9786,"children":9788},{"className":9787},[285],[9789,9793],{"type":150,"tag":181,"props":9790,"children":9792},{"className":9791,"style":5012},[290],[],{"type":150,"tag":181,"props":9794,"children":9796},{"className":9795},[296],[9797],{"type":155,"value":9202},{"type":155,"value":9799}," 能留下来，一半是因为并行度，一半是习惯。但 MQA 和 GQA 有一个共同的性质，值得单独点出来：",{"type":150,"tag":151,"props":9801,"children":9802},{},[9803],{"type":150,"tag":158,"props":9804,"children":9805},{},[9806],{"type":155,"value":9807},"它们压的是 head 的数量，不是 head 的内容。",{"type":150,"tag":151,"props":9809,"children":9810},{},[9811,9813,9818],{"type":155,"value":9812},"被保留下来的那些 KV，每一个仍然是完整的、未经压缩的向量。共享是",{"type":150,"tag":158,"props":9814,"children":9815},{},[9816],{"type":155,"value":9817},"离散",{"type":155,"value":9819},"的。要么整份共用，要么各自一份，中间没有别的可能。下一节就是打破这个限制。",{"type":150,"tag":169,"props":9821,"children":9823},{"id":9822},"mla",[9824],{"type":155,"value":13},{"type":150,"tag":151,"props":9826,"children":9827},{},[9828],{"type":155,"value":9829},"MLA（Multi-head Latent Attention）是 DeepSeek-V2 在 2024 年 5 月提出来的，到现在两年多。技术报告是从低秩投影的角度引入的，以至于有人问：LoRA 提出这么久了，为什么直到 MLA 才有人对 KV Cache 做低秩分解？我认为低秩投影这个角度不贴近本质。因为要说低秩投影的话，只要把 GQA 的所有 K、V 拼成一个长向量，就会发现 GQA 本身也是低秩投影。",{"type":150,"tag":151,"props":9831,"children":9832},{},[9833,9835,9938,9940,9982],{"type":155,"value":9834},"把这个拼起来的向量记作 ",{"type":150,"tag":181,"props":9836,"children":9838},{"className":9837},[184],[9839,9869],{"type":150,"tag":181,"props":9840,"children":9842},{"className":9841},[189],[9843],{"type":150,"tag":192,"props":9844,"children":9845},{"xmlns":194},[9846],{"type":150,"tag":197,"props":9847,"children":9848},{},[9849,9864],{"type":150,"tag":201,"props":9850,"children":9851},{},[9852],{"type":150,"tag":205,"props":9853,"children":9854},{},[9855,9860],{"type":150,"tag":209,"props":9856,"children":9857},{},[9858],{"type":155,"value":9859},"c",{"type":150,"tag":209,"props":9861,"children":9862},{},[9863],{"type":155,"value":558},{"type":150,"tag":271,"props":9865,"children":9866},{"encoding":273},[9867],{"type":155,"value":9868},"c_i",{"type":150,"tag":181,"props":9870,"children":9872},{"className":9871,"ariaHidden":223},[280],[9873],{"type":150,"tag":181,"props":9874,"children":9876},{"className":9875},[285],[9877,9881],{"type":150,"tag":181,"props":9878,"children":9880},{"className":9879,"style":1165},[290],[],{"type":150,"tag":181,"props":9882,"children":9884},{"className":9883},[296],[9885,9890],{"type":150,"tag":181,"props":9886,"children":9888},{"className":9887},[296,301],[9889],{"type":155,"value":9859},{"type":150,"tag":181,"props":9891,"children":9893},{"className":9892},[307],[9894],{"type":150,"tag":181,"props":9895,"children":9897},{"className":9896},[312,313],[9898,9927],{"type":150,"tag":181,"props":9899,"children":9901},{"className":9900},[318],[9902,9922],{"type":150,"tag":181,"props":9903,"children":9905},{"className":9904,"style":621},[323],[9906],{"type":150,"tag":181,"props":9907,"children":9908},{"style":328},[9909,9913],{"type":150,"tag":181,"props":9910,"children":9912},{"className":9911,"style":334},[333],[],{"type":150,"tag":181,"props":9914,"children":9916},{"className":9915},[339,340,341,342],[9917],{"type":150,"tag":181,"props":9918,"children":9920},{"className":9919},[296,301,342],[9921],{"type":155,"value":558},{"type":150,"tag":181,"props":9923,"children":9925},{"className":9924},[352],[9926],{"type":155,"value":355},{"type":150,"tag":181,"props":9928,"children":9930},{"className":9929},[318],[9931],{"type":150,"tag":181,"props":9932,"children":9934},{"className":9933,"style":363},[323],[9935],{"type":150,"tag":181,"props":9936,"children":9937},{},[],{"type":155,"value":9939},"。它由 ",{"type":150,"tag":181,"props":9941,"children":9943},{"className":9942},[184],[9944,9965],{"type":150,"tag":181,"props":9945,"children":9947},{"className":9946},[189],[9948],{"type":150,"tag":192,"props":9949,"children":9950},{"xmlns":194},[9951],{"type":150,"tag":197,"props":9952,"children":9953},{},[9954,9961],{"type":150,"tag":201,"props":9955,"children":9956},{},[9957],{"type":150,"tag":209,"props":9958,"children":9959},{},[9960],{"type":155,"value":7913},{"type":150,"tag":271,"props":9962,"children":9963},{"encoding":273},[9964],{"type":155,"value":7913},{"type":150,"tag":181,"props":9966,"children":9968},{"className":9967,"ariaHidden":223},[280],[9969],{"type":150,"tag":181,"props":9970,"children":9972},{"className":9971},[285],[9973,9977],{"type":150,"tag":181,"props":9974,"children":9976},{"className":9975,"style":291},[290],[],{"type":150,"tag":181,"props":9978,"children":9980},{"className":9979,"style":2151},[296,301],[9981],{"type":155,"value":7913},{"type":155,"value":9983}," 组 K、V 依次首尾相接而成：",{"type":150,"tag":181,"props":9985,"children":9987},{"className":9986},[1003],[9988],{"type":150,"tag":181,"props":9989,"children":9991},{"className":9990},[184],[9992,10225],{"type":150,"tag":181,"props":9993,"children":9995},{"className":9994},[189],[9996],{"type":150,"tag":192,"props":9997,"children":9998},{"xmlns":194,"display":1015},[9999],{"type":150,"tag":197,"props":10000,"children":10001},{},[10002,10220],{"type":150,"tag":201,"props":10003,"children":10004},{},[10005,10016,10020,10024,10050,10054,10058,10062,10066,10092,10096,10122,10126,10130,10134,10138,10164,10168,10172],{"type":150,"tag":205,"props":10006,"children":10007},{},[10008,10012],{"type":150,"tag":209,"props":10009,"children":10010},{},[10011],{"type":155,"value":9859},{"type":150,"tag":209,"props":10013,"children":10014},{},[10015],{"type":155,"value":558},{"type":150,"tag":221,"props":10017,"children":10018},{},[10019],{"type":155,"value":865},{"type":150,"tag":221,"props":10021,"children":10022},{"stretchy":789},[10023],{"type":155,"value":1044},{"type":150,"tag":1046,"props":10025,"children":10026},{},[10027,10031,10035],{"type":150,"tag":209,"props":10028,"children":10029},{},[10030],{"type":155,"value":1698},{"type":150,"tag":209,"props":10032,"children":10033},{},[10034],{"type":155,"value":558},{"type":150,"tag":201,"props":10036,"children":10037},{},[10038,10042,10046],{"type":150,"tag":221,"props":10039,"children":10040},{"stretchy":789},[10041],{"type":155,"value":792},{"type":150,"tag":215,"props":10043,"children":10044},{},[10045],{"type":155,"value":219},{"type":150,"tag":221,"props":10047,"children":10048},{"stretchy":789},[10049],{"type":155,"value":802},{"type":150,"tag":221,"props":10051,"children":10052},{"separator":223},[10053],{"type":155,"value":226},{"type":150,"tag":221,"props":10055,"children":10056},{},[10057],{"type":155,"value":247},{"type":150,"tag":249,"props":10059,"children":10060},{},[10061],{"type":155,"value":253},{"type":150,"tag":221,"props":10063,"children":10064},{"separator":223},[10065],{"type":155,"value":226},{"type":150,"tag":1046,"props":10067,"children":10068},{},[10069,10073,10077],{"type":150,"tag":209,"props":10070,"children":10071},{},[10072],{"type":155,"value":1698},{"type":150,"tag":209,"props":10074,"children":10075},{},[10076],{"type":155,"value":558},{"type":150,"tag":201,"props":10078,"children":10079},{},[10080,10084,10088],{"type":150,"tag":221,"props":10081,"children":10082},{"stretchy":789},[10083],{"type":155,"value":792},{"type":150,"tag":209,"props":10085,"children":10086},{},[10087],{"type":155,"value":7913},{"type":150,"tag":221,"props":10089,"children":10090},{"stretchy":789},[10091],{"type":155,"value":802},{"type":150,"tag":221,"props":10093,"children":10094},{"separator":223},[10095],{"type":155,"value":226},{"type":150,"tag":1046,"props":10097,"children":10098},{},[10099,10103,10107],{"type":150,"tag":209,"props":10100,"children":10101},{},[10102],{"type":155,"value":1737},{"type":150,"tag":209,"props":10104,"children":10105},{},[10106],{"type":155,"value":558},{"type":150,"tag":201,"props":10108,"children":10109},{},[10110,10114,10118],{"type":150,"tag":221,"props":10111,"children":10112},{"stretchy":789},[10113],{"type":155,"value":792},{"type":150,"tag":215,"props":10115,"children":10116},{},[10117],{"type":155,"value":219},{"type":150,"tag":221,"props":10119,"children":10120},{"stretchy":789},[10121],{"type":155,"value":802},{"type":150,"tag":221,"props":10123,"children":10124},{"separator":223},[10125],{"type":155,"value":226},{"type":150,"tag":221,"props":10127,"children":10128},{},[10129],{"type":155,"value":247},{"type":150,"tag":249,"props":10131,"children":10132},{},[10133],{"type":155,"value":253},{"type":150,"tag":221,"props":10135,"children":10136},{"separator":223},[10137],{"type":155,"value":226},{"type":150,"tag":1046,"props":10139,"children":10140},{},[10141,10145,10149],{"type":150,"tag":209,"props":10142,"children":10143},{},[10144],{"type":155,"value":1737},{"type":150,"tag":209,"props":10146,"children":10147},{},[10148],{"type":155,"value":558},{"type":150,"tag":201,"props":10150,"children":10151},{},[10152,10156,10160],{"type":150,"tag":221,"props":10153,"children":10154},{"stretchy":789},[10155],{"type":155,"value":792},{"type":150,"tag":209,"props":10157,"children":10158},{},[10159],{"type":155,"value":7913},{"type":150,"tag":221,"props":10161,"children":10162},{"stretchy":789},[10163],{"type":155,"value":802},{"type":150,"tag":221,"props":10165,"children":10166},{"stretchy":789},[10167],{"type":155,"value":1148},{"type":150,"tag":221,"props":10169,"children":10170},{},[10171],{"type":155,"value":563},{"type":150,"tag":565,"props":10173,"children":10174},{},[10175,10179],{"type":150,"tag":209,"props":10176,"children":10177},{"mathvariant":570},[10178],{"type":155,"value":573},{"type":150,"tag":201,"props":10180,"children":10181},{},[10182,10186,10190,10201,10205,10216],{"type":150,"tag":209,"props":10183,"children":10184},{},[10185],{"type":155,"value":7913},{"type":150,"tag":221,"props":10187,"children":10188},{"stretchy":789},[10189],{"type":155,"value":792},{"type":150,"tag":205,"props":10191,"children":10192},{},[10193,10197],{"type":150,"tag":209,"props":10194,"children":10195},{},[10196],{"type":155,"value":578},{"type":150,"tag":209,"props":10198,"children":10199},{},[10200],{"type":155,"value":1698},{"type":150,"tag":221,"props":10202,"children":10203},{},[10204],{"type":155,"value":4962},{"type":150,"tag":205,"props":10206,"children":10207},{},[10208,10212],{"type":150,"tag":209,"props":10209,"children":10210},{},[10211],{"type":155,"value":578},{"type":150,"tag":209,"props":10213,"children":10214},{},[10215],{"type":155,"value":1737},{"type":150,"tag":221,"props":10217,"children":10218},{"stretchy":789},[10219],{"type":155,"value":802},{"type":150,"tag":271,"props":10221,"children":10222},{"encoding":273},[10223],{"type":155,"value":10224},"c_i = [k_i^{(1)}, \\cdots, k_i^{(g)}, v_i^{(1)}, \\cdots, v_i^{(g)}] \\in \\mathbb{R}^{g(d_k + d_v)}",{"type":150,"tag":181,"props":10226,"children":10228},{"className":10227,"ariaHidden":223},[280],[10229,10307,10757],{"type":150,"tag":181,"props":10230,"children":10232},{"className":10231},[285],[10233,10237,10294,10298,10303],{"type":150,"tag":181,"props":10234,"children":10236},{"className":10235,"style":1165},[290],[],{"type":150,"tag":181,"props":10238,"children":10240},{"className":10239},[296],[10241,10246],{"type":150,"tag":181,"props":10242,"children":10244},{"className":10243},[296,301],[10245],{"type":155,"value":9859},{"type":150,"tag":181,"props":10247,"children":10249},{"className":10248},[307],[10250],{"type":150,"tag":181,"props":10251,"children":10253},{"className":10252},[312,313],[10254,10283],{"type":150,"tag":181,"props":10255,"children":10257},{"className":10256},[318],[10258,10278],{"type":150,"tag":181,"props":10259,"children":10261},{"className":10260,"style":621},[323],[10262],{"type":150,"tag":181,"props":10263,"children":10264},{"style":328},[10265,10269],{"type":150,"tag":181,"props":10266,"children":10268},{"className":10267,"style":334},[333],[],{"type":150,"tag":181,"props":10270,"children":10272},{"className":10271},[339,340,341,342],[10273],{"type":150,"tag":181,"props":10274,"children":10276},{"className":10275},[296,301,342],[10277],{"type":155,"value":558},{"type":150,"tag":181,"props":10279,"children":10281},{"className":10280},[352],[10282],{"type":155,"value":355},{"type":150,"tag":181,"props":10284,"children":10286},{"className":10285},[318],[10287],{"type":150,"tag":181,"props":10288,"children":10290},{"className":10289,"style":363},[323],[10291],{"type":150,"tag":181,"props":10292,"children":10293},{},[],{"type":150,"tag":181,"props":10295,"children":10297},{"className":10296,"style":658},[377],[],{"type":150,"tag":181,"props":10299,"children":10301},{"className":10300},[663],[10302],{"type":155,"value":865},{"type":150,"tag":181,"props":10304,"children":10306},{"className":10305,"style":658},[377],[],{"type":150,"tag":181,"props":10308,"children":10310},{"className":10309},[285],[10311,10315,10320,10407,10412,10416,10421,10425,10429,10434,10438,10525,10530,10534,10621,10626,10630,10635,10639,10643,10648,10652,10739,10744,10748,10753],{"type":150,"tag":181,"props":10312,"children":10314},{"className":10313,"style":3516},[290],[],{"type":150,"tag":181,"props":10316,"children":10318},{"className":10317},[824],[10319],{"type":155,"value":1044},{"type":150,"tag":181,"props":10321,"children":10323},{"className":10322},[296],[10324,10329],{"type":150,"tag":181,"props":10325,"children":10327},{"className":10326,"style":2249},[296,301],[10328],{"type":155,"value":1698},{"type":150,"tag":181,"props":10330,"children":10332},{"className":10331},[307],[10333],{"type":150,"tag":181,"props":10334,"children":10336},{"className":10335},[312,313],[10337,10396],{"type":150,"tag":181,"props":10338,"children":10340},{"className":10339},[318],[10341,10391],{"type":150,"tag":181,"props":10342,"children":10344},{"className":10343,"style":1276},[323],[10345,10361],{"type":150,"tag":181,"props":10346,"children":10347},{"style":2720},[10348,10352],{"type":150,"tag":181,"props":10349,"children":10351},{"className":10350,"style":334},[333],[],{"type":150,"tag":181,"props":10353,"children":10355},{"className":10354},[339,340,341,342],[10356],{"type":150,"tag":181,"props":10357,"children":10359},{"className":10358},[296,301,342],[10360],{"type":155,"value":558},{"type":150,"tag":181,"props":10362,"children":10363},{"style":1297},[10364,10368],{"type":150,"tag":181,"props":10365,"children":10367},{"className":10366,"style":334},[333],[],{"type":150,"tag":181,"props":10369,"children":10371},{"className":10370},[339,340,341,342],[10372],{"type":150,"tag":181,"props":10373,"children":10375},{"className":10374},[296,342],[10376,10381,10386],{"type":150,"tag":181,"props":10377,"children":10379},{"className":10378},[824,342],[10380],{"type":155,"value":792},{"type":150,"tag":181,"props":10382,"children":10384},{"className":10383},[296,342],[10385],{"type":155,"value":219},{"type":150,"tag":181,"props":10387,"children":10389},{"className":10388},[835,342],[10390],{"type":155,"value":802},{"type":150,"tag":181,"props":10392,"children":10394},{"className":10393},[352],[10395],{"type":155,"value":355},{"type":150,"tag":181,"props":10397,"children":10399},{"className":10398},[318],[10400],{"type":150,"tag":181,"props":10401,"children":10403},{"className":10402,"style":2782},[323],[10404],{"type":150,"tag":181,"props":10405,"children":10406},{},[],{"type":150,"tag":181,"props":10408,"children":10410},{"className":10409},[371],[10411],{"type":155,"value":226},{"type":150,"tag":181,"props":10413,"children":10415},{"className":10414,"style":378},[377],[],{"type":150,"tag":181,"props":10417,"children":10419},{"className":10418},[449],[10420],{"type":155,"value":247},{"type":150,"tag":181,"props":10422,"children":10424},{"className":10423,"style":378},[377],[],{"type":150,"tag":181,"props":10426,"children":10428},{"className":10427,"style":378},[377],[],{"type":150,"tag":181,"props":10430,"children":10432},{"className":10431},[371],[10433],{"type":155,"value":226},{"type":150,"tag":181,"props":10435,"children":10437},{"className":10436,"style":378},[377],[],{"type":150,"tag":181,"props":10439,"children":10441},{"className":10440},[296],[10442,10447],{"type":150,"tag":181,"props":10443,"children":10445},{"className":10444,"style":2249},[296,301],[10446],{"type":155,"value":1698},{"type":150,"tag":181,"props":10448,"children":10450},{"className":10449},[307],[10451],{"type":150,"tag":181,"props":10452,"children":10454},{"className":10453},[312,313],[10455,10514],{"type":150,"tag":181,"props":10456,"children":10458},{"className":10457},[318],[10459,10509],{"type":150,"tag":181,"props":10460,"children":10462},{"className":10461,"style":1276},[323],[10463,10479],{"type":150,"tag":181,"props":10464,"children":10465},{"style":2720},[10466,10470],{"type":150,"tag":181,"props":10467,"children":10469},{"className":10468,"style":334},[333],[],{"type":150,"tag":181,"props":10471,"children":10473},{"className":10472},[339,340,341,342],[10474],{"type":150,"tag":181,"props":10475,"children":10477},{"className":10476},[296,301,342],[10478],{"type":155,"value":558},{"type":150,"tag":181,"props":10480,"children":10481},{"style":1297},[10482,10486],{"type":150,"tag":181,"props":10483,"children":10485},{"className":10484,"style":334},[333],[],{"type":150,"tag":181,"props":10487,"children":10489},{"className":10488},[339,340,341,342],[10490],{"type":150,"tag":181,"props":10491,"children":10493},{"className":10492},[296,342],[10494,10499,10504],{"type":150,"tag":181,"props":10495,"children":10497},{"className":10496},[824,342],[10498],{"type":155,"value":792},{"type":150,"tag":181,"props":10500,"children":10502},{"className":10501,"style":2151},[296,301,342],[10503],{"type":155,"value":7913},{"type":150,"tag":181,"props":10505,"children":10507},{"className":10506},[835,342],[10508],{"type":155,"value":802},{"type":150,"tag":181,"props":10510,"children":10512},{"className":10511},[352],[10513],{"type":155,"value":355},{"type":150,"tag":181,"props":10515,"children":10517},{"className":10516},[318],[10518],{"type":150,"tag":181,"props":10519,"children":10521},{"className":10520,"style":2782},[323],[10522],{"type":150,"tag":181,"props":10523,"children":10524},{},[],{"type":150,"tag":181,"props":10526,"children":10528},{"className":10527},[371],[10529],{"type":155,"value":226},{"type":150,"tag":181,"props":10531,"children":10533},{"className":10532,"style":378},[377],[],{"type":150,"tag":181,"props":10535,"children":10537},{"className":10536},[296],[10538,10543],{"type":150,"tag":181,"props":10539,"children":10541},{"className":10540,"style":2151},[296,301],[10542],{"type":155,"value":1737},{"type":150,"tag":181,"props":10544,"children":10546},{"className":10545},[307],[10547],{"type":150,"tag":181,"props":10548,"children":10550},{"className":10549},[312,313],[10551,10610],{"type":150,"tag":181,"props":10552,"children":10554},{"className":10553},[318],[10555,10605],{"type":150,"tag":181,"props":10556,"children":10558},{"className":10557,"style":1276},[323],[10559,10575],{"type":150,"tag":181,"props":10560,"children":10561},{"style":3114},[10562,10566],{"type":150,"tag":181,"props":10563,"children":10565},{"className":10564,"style":334},[333],[],{"type":150,"tag":181,"props":10567,"children":10569},{"className":10568},[339,340,341,342],[10570],{"type":150,"tag":181,"props":10571,"children":10573},{"className":10572},[296,301,342],[10574],{"type":155,"value":558},{"type":150,"tag":181,"props":10576,"children":10577},{"style":1297},[10578,10582],{"type":150,"tag":181,"props":10579,"children":10581},{"className":10580,"style":334},[333],[],{"type":150,"tag":181,"props":10583,"children":10585},{"className":10584},[339,340,341,342],[10586],{"type":150,"tag":181,"props":10587,"children":10589},{"className":10588},[296,342],[10590,10595,10600],{"type":150,"tag":181,"props":10591,"children":10593},{"className":10592},[824,342],[10594],{"type":155,"value":792},{"type":150,"tag":181,"props":10596,"children":10598},{"className":10597},[296,342],[10599],{"type":155,"value":219},{"type":150,"tag":181,"props":10601,"children":10603},{"className":10602},[835,342],[10604],{"type":155,"value":802},{"type":150,"tag":181,"props":10606,"children":10608},{"className":10607},[352],[10609],{"type":155,"value":355},{"type":150,"tag":181,"props":10611,"children":10613},{"className":10612},[318],[10614],{"type":150,"tag":181,"props":10615,"children":10617},{"className":10616,"style":2782},[323],[10618],{"type":150,"tag":181,"props":10619,"children":10620},{},[],{"type":150,"tag":181,"props":10622,"children":10624},{"className":10623},[371],[10625],{"type":155,"value":226},{"type":150,"tag":181,"props":10627,"children":10629},{"className":10628,"style":378},[377],[],{"type":150,"tag":181,"props":10631,"children":10633},{"className":10632},[449],[10634],{"type":155,"value":247},{"type":150,"tag":181,"props":10636,"children":10638},{"className":10637,"style":378},[377],[],{"type":150,"tag":181,"props":10640,"children":10642},{"className":10641,"style":378},[377],[],{"type":150,"tag":181,"props":10644,"children":10646},{"className":10645},[371],[10647],{"type":155,"value":226},{"type":150,"tag":181,"props":10649,"children":10651},{"className":10650,"style":378},[377],[],{"type":150,"tag":181,"props":10653,"children":10655},{"className":10654},[296],[10656,10661],{"type":150,"tag":181,"props":10657,"children":10659},{"className":10658,"style":2151},[296,301],[10660],{"type":155,"value":1737},{"type":150,"tag":181,"props":10662,"children":10664},{"className":10663},[307],[10665],{"type":150,"tag":181,"props":10666,"children":10668},{"className":10667},[312,313],[10669,10728],{"type":150,"tag":181,"props":10670,"children":10672},{"className":10671},[318],[10673,10723],{"type":150,"tag":181,"props":10674,"children":10676},{"className":10675,"style":1276},[323],[10677,10693],{"type":150,"tag":181,"props":10678,"children":10679},{"style":3114},[10680,10684],{"type":150,"tag":181,"props":10681,"children":10683},{"className":10682,"style":334},[333],[],{"type":150,"tag":181,"props":10685,"children":10687},{"className":10686},[339,340,341,342],[10688],{"type":150,"tag":181,"props":10689,"children":10691},{"className":10690},[296,301,342],[10692],{"type":155,"value":558},{"type":150,"tag":181,"props":10694,"children":10695},{"style":1297},[10696,10700],{"type":150,"tag":181,"props":10697,"children":10699},{"className":10698,"style":334},[333],[],{"type":150,"tag":181,"props":10701,"children":10703},{"className":10702},[339,340,341,342],[10704],{"type":150,"tag":181,"props":10705,"children":10707},{"className":10706},[296,342],[10708,10713,10718],{"type":150,"tag":181,"props":10709,"children":10711},{"className":10710},[824,342],[10712],{"type":155,"value":792},{"type":150,"tag":181,"props":10714,"children":10716},{"className":10715,"style":2151},[296,301,342],[10717],{"type":155,"value":7913},{"type":150,"tag":181,"props":10719,"children":10721},{"className":10720},[835,342],[10722],{"type":155,"value":802},{"type":150,"tag":181,"props":10724,"children":10726},{"className":10725},[352],[10727],{"type":155,"value":355},{"type":150,"tag":181,"props":10729,"children":10731},{"className":10730},[318],[10732],{"type":150,"tag":181,"props":10733,"children":10735},{"className":10734,"style":2782},[323],[10736],{"type":150,"tag":181,"props":10737,"children":10738},{},[],{"type":150,"tag":181,"props":10740,"children":10742},{"className":10741},[835],[10743],{"type":155,"value":1148},{"type":150,"tag":181,"props":10745,"children":10747},{"className":10746,"style":658},[377],[],{"type":150,"tag":181,"props":10749,"children":10751},{"className":10750},[663],[10752],{"type":155,"value":563},{"type":150,"tag":181,"props":10754,"children":10756},{"className":10755,"style":658},[377],[],{"type":150,"tag":181,"props":10758,"children":10760},{"className":10759},[285],[10761,10765],{"type":150,"tag":181,"props":10762,"children":10764},{"className":10763,"style":3709},[290],[],{"type":150,"tag":181,"props":10766,"children":10768},{"className":10767},[296],[10769,10774],{"type":150,"tag":181,"props":10770,"children":10772},{"className":10771},[296,686],[10773],{"type":155,"value":573},{"type":150,"tag":181,"props":10775,"children":10777},{"className":10776},[307],[10778],{"type":150,"tag":181,"props":10779,"children":10781},{"className":10780},[312],[10782],{"type":150,"tag":181,"props":10783,"children":10785},{"className":10784},[318],[10786],{"type":150,"tag":181,"props":10787,"children":10789},{"className":10788,"style":3709},[323],[10790],{"type":150,"tag":181,"props":10791,"children":10792},{"style":3730},[10793,10797],{"type":150,"tag":181,"props":10794,"children":10796},{"className":10795,"style":334},[333],[],{"type":150,"tag":181,"props":10798,"children":10800},{"className":10799},[339,340,341,342],[10801],{"type":150,"tag":181,"props":10802,"children":10804},{"className":10803},[296,342],[10805,10810,10815,10872,10877,10934],{"type":150,"tag":181,"props":10806,"children":10808},{"className":10807,"style":2151},[296,301,342],[10809],{"type":155,"value":7913},{"type":150,"tag":181,"props":10811,"children":10813},{"className":10812},[824,342],[10814],{"type":155,"value":792},{"type":150,"tag":181,"props":10816,"children":10818},{"className":10817},[296,342],[10819,10824],{"type":150,"tag":181,"props":10820,"children":10822},{"className":10821},[296,301,342],[10823],{"type":155,"value":578},{"type":150,"tag":181,"props":10825,"children":10827},{"className":10826},[307],[10828],{"type":150,"tag":181,"props":10829,"children":10831},{"className":10830},[312,313],[10832,10861],{"type":150,"tag":181,"props":10833,"children":10835},{"className":10834},[318],[10836,10856],{"type":150,"tag":181,"props":10837,"children":10839},{"className":10838,"style":3862},[323],[10840],{"type":150,"tag":181,"props":10841,"children":10842},{"style":3866},[10843,10847],{"type":150,"tag":181,"props":10844,"children":10846},{"className":10845,"style":3871},[333],[],{"type":150,"tag":181,"props":10848,"children":10850},{"className":10849},[339,3876,3877,342],[10851],{"type":150,"tag":181,"props":10852,"children":10854},{"className":10853,"style":2249},[296,301,342],[10855],{"type":155,"value":1698},{"type":150,"tag":181,"props":10857,"children":10859},{"className":10858},[352],[10860],{"type":155,"value":355},{"type":150,"tag":181,"props":10862,"children":10864},{"className":10863},[318],[10865],{"type":150,"tag":181,"props":10866,"children":10868},{"className":10867,"style":3896},[323],[10869],{"type":150,"tag":181,"props":10870,"children":10871},{},[],{"type":150,"tag":181,"props":10873,"children":10875},{"className":10874},[4998,342],[10876],{"type":155,"value":4962},{"type":150,"tag":181,"props":10878,"children":10880},{"className":10879},[296,342],[10881,10886],{"type":150,"tag":181,"props":10882,"children":10884},{"className":10883},[296,301,342],[10885],{"type":155,"value":578},{"type":150,"tag":181,"props":10887,"children":10889},{"className":10888},[307],[10890],{"type":150,"tag":181,"props":10891,"children":10893},{"className":10892},[312,313],[10894,10923],{"type":150,"tag":181,"props":10895,"children":10897},{"className":10896},[318],[10898,10918],{"type":150,"tag":181,"props":10899,"children":10901},{"className":10900,"style":4640},[323],[10902],{"type":150,"tag":181,"props":10903,"children":10904},{"style":4644},[10905,10909],{"type":150,"tag":181,"props":10906,"children":10908},{"className":10907,"style":3871},[333],[],{"type":150,"tag":181,"props":10910,"children":10912},{"className":10911},[339,3876,3877,342],[10913],{"type":150,"tag":181,"props":10914,"children":10916},{"className":10915,"style":2151},[296,301,342],[10917],{"type":155,"value":1737},{"type":150,"tag":181,"props":10919,"children":10921},{"className":10920},[352],[10922],{"type":155,"value":355},{"type":150,"tag":181,"props":10924,"children":10926},{"className":10925},[318],[10927],{"type":150,"tag":181,"props":10928,"children":10930},{"className":10929,"style":4671},[323],[10931],{"type":150,"tag":181,"props":10932,"children":10933},{},[],{"type":150,"tag":181,"props":10935,"children":10937},{"className":10936},[835,342],[10938],{"type":155,"value":802},{"type":150,"tag":151,"props":10940,"children":10941},{},[10942,10944,11046,11048,11151],{"type":155,"value":10943},"而它整个是由 ",{"type":150,"tag":181,"props":10945,"children":10947},{"className":10946},[184],[10948,10977],{"type":150,"tag":181,"props":10949,"children":10951},{"className":10950},[189],[10952],{"type":150,"tag":192,"props":10953,"children":10954},{"xmlns":194},[10955],{"type":150,"tag":197,"props":10956,"children":10957},{},[10958,10972],{"type":150,"tag":201,"props":10959,"children":10960},{},[10961],{"type":150,"tag":205,"props":10962,"children":10963},{},[10964,10968],{"type":150,"tag":209,"props":10965,"children":10966},{},[10967],{"type":155,"value":213},{"type":150,"tag":209,"props":10969,"children":10970},{},[10971],{"type":155,"value":558},{"type":150,"tag":271,"props":10973,"children":10974},{"encoding":273},[10975],{"type":155,"value":10976},"x_i",{"type":150,"tag":181,"props":10978,"children":10980},{"className":10979,"ariaHidden":223},[280],[10981],{"type":150,"tag":181,"props":10982,"children":10984},{"className":10983},[285],[10985,10989],{"type":150,"tag":181,"props":10986,"children":10988},{"className":10987,"style":1165},[290],[],{"type":150,"tag":181,"props":10990,"children":10992},{"className":10991},[296],[10993,10998],{"type":150,"tag":181,"props":10994,"children":10996},{"className":10995},[296,301],[10997],{"type":155,"value":213},{"type":150,"tag":181,"props":10999,"children":11001},{"className":11000},[307],[11002],{"type":150,"tag":181,"props":11003,"children":11005},{"className":11004},[312,313],[11006,11035],{"type":150,"tag":181,"props":11007,"children":11009},{"className":11008},[318],[11010,11030],{"type":150,"tag":181,"props":11011,"children":11013},{"className":11012,"style":621},[323],[11014],{"type":150,"tag":181,"props":11015,"children":11016},{"style":328},[11017,11021],{"type":150,"tag":181,"props":11018,"children":11020},{"className":11019,"style":334},[333],[],{"type":150,"tag":181,"props":11022,"children":11024},{"className":11023},[339,340,341,342],[11025],{"type":150,"tag":181,"props":11026,"children":11028},{"className":11027},[296,301,342],[11029],{"type":155,"value":558},{"type":150,"tag":181,"props":11031,"children":11033},{"className":11032},[352],[11034],{"type":155,"value":355},{"type":150,"tag":181,"props":11036,"children":11038},{"className":11037},[318],[11039],{"type":150,"tag":181,"props":11040,"children":11042},{"className":11041,"style":363},[323],[11043],{"type":150,"tag":181,"props":11044,"children":11045},{},[],{"type":155,"value":11047}," 一次矩阵乘法得到的。把各组投影矩阵横向拼起来记作 ",{"type":150,"tag":181,"props":11049,"children":11051},{"className":11050},[184],[11052,11081],{"type":150,"tag":181,"props":11053,"children":11055},{"className":11054},[189],[11056],{"type":150,"tag":192,"props":11057,"children":11058},{"xmlns":194},[11059],{"type":150,"tag":197,"props":11060,"children":11061},{},[11062,11076],{"type":150,"tag":201,"props":11063,"children":11064},{},[11065],{"type":150,"tag":205,"props":11066,"children":11067},{},[11068,11072],{"type":150,"tag":209,"props":11069,"children":11070},{},[11071],{"type":155,"value":3265},{"type":150,"tag":209,"props":11073,"children":11074},{},[11075],{"type":155,"value":9859},{"type":150,"tag":271,"props":11077,"children":11078},{"encoding":273},[11079],{"type":155,"value":11080},"W_c",{"type":150,"tag":181,"props":11082,"children":11084},{"className":11083,"ariaHidden":223},[280],[11085],{"type":150,"tag":181,"props":11086,"children":11088},{"className":11087},[285],[11089,11093],{"type":150,"tag":181,"props":11090,"children":11092},{"className":11091,"style":6190},[290],[],{"type":150,"tag":181,"props":11094,"children":11096},{"className":11095},[296],[11097,11102],{"type":150,"tag":181,"props":11098,"children":11100},{"className":11099,"style":3691},[296,301],[11101],{"type":155,"value":3265},{"type":150,"tag":181,"props":11103,"children":11105},{"className":11104},[307],[11106],{"type":150,"tag":181,"props":11107,"children":11109},{"className":11108},[312,313],[11110,11140],{"type":150,"tag":181,"props":11111,"children":11113},{"className":11112},[318],[11114,11135],{"type":150,"tag":181,"props":11115,"children":11117},{"className":11116,"style":4865},[323],[11118],{"type":150,"tag":181,"props":11119,"children":11121},{"style":11120},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[11122,11126],{"type":150,"tag":181,"props":11123,"children":11125},{"className":11124,"style":334},[333],[],{"type":150,"tag":181,"props":11127,"children":11129},{"className":11128},[339,340,341,342],[11130],{"type":150,"tag":181,"props":11131,"children":11133},{"className":11132},[296,301,342],[11134],{"type":155,"value":9859},{"type":150,"tag":181,"props":11136,"children":11138},{"className":11137},[352],[11139],{"type":155,"value":355},{"type":150,"tag":181,"props":11141,"children":11143},{"className":11142},[318],[11144],{"type":150,"tag":181,"props":11145,"children":11147},{"className":11146,"style":363},[323],[11148],{"type":150,"tag":181,"props":11149,"children":11150},{},[],{"type":155,"value":11152},"：",{"type":150,"tag":181,"props":11154,"children":11156},{"className":11155},[1003],[11157],{"type":150,"tag":181,"props":11158,"children":11160},{"className":11159},[184],[11161,11447],{"type":150,"tag":181,"props":11162,"children":11164},{"className":11163},[189],[11165],{"type":150,"tag":192,"props":11166,"children":11167},{"xmlns":194,"display":1015},[11168],{"type":150,"tag":197,"props":11169,"children":11170},{},[11171,11442],{"type":150,"tag":201,"props":11172,"children":11173},{},[11174,11185,11189,11200,11211,11215,11219,11230,11234,11238,11264,11268,11272,11276,11280,11306,11310,11336,11340,11344,11348,11352,11378,11382,11386],{"type":150,"tag":205,"props":11175,"children":11176},{},[11177,11181],{"type":150,"tag":209,"props":11178,"children":11179},{},[11180],{"type":155,"value":9859},{"type":150,"tag":209,"props":11182,"children":11183},{},[11184],{"type":155,"value":558},{"type":150,"tag":221,"props":11186,"children":11187},{},[11188],{"type":155,"value":865},{"type":150,"tag":205,"props":11190,"children":11191},{},[11192,11196],{"type":150,"tag":209,"props":11193,"children":11194},{},[11195],{"type":155,"value":213},{"type":150,"tag":209,"props":11197,"children":11198},{},[11199],{"type":155,"value":558},{"type":150,"tag":205,"props":11201,"children":11202},{},[11203,11207],{"type":150,"tag":209,"props":11204,"children":11205},{},[11206],{"type":155,"value":3265},{"type":150,"tag":209,"props":11208,"children":11209},{},[11210],{"type":155,"value":9859},{"type":150,"tag":221,"props":11212,"children":11213},{"separator":223},[11214],{"type":155,"value":226},{"type":150,"tag":377,"props":11216,"children":11218},{"width":11217},"2em",[],{"type":150,"tag":205,"props":11220,"children":11221},{},[11222,11226],{"type":150,"tag":209,"props":11223,"children":11224},{},[11225],{"type":155,"value":3265},{"type":150,"tag":209,"props":11227,"children":11228},{},[11229],{"type":155,"value":9859},{"type":150,"tag":221,"props":11231,"children":11232},{},[11233],{"type":155,"value":865},{"type":150,"tag":221,"props":11235,"children":11236},{"stretchy":789},[11237],{"type":155,"value":1044},{"type":150,"tag":1046,"props":11239,"children":11240},{},[11241,11245,11249],{"type":150,"tag":209,"props":11242,"children":11243},{},[11244],{"type":155,"value":3265},{"type":150,"tag":209,"props":11246,"children":11247},{},[11248],{"type":155,"value":1698},{"type":150,"tag":201,"props":11250,"children":11251},{},[11252,11256,11260],{"type":150,"tag":221,"props":11253,"children":11254},{"stretchy":789},[11255],{"type":155,"value":792},{"type":150,"tag":215,"props":11257,"children":11258},{},[11259],{"type":155,"value":219},{"type":150,"tag":221,"props":11261,"children":11262},{"stretchy":789},[11263],{"type":155,"value":802},{"type":150,"tag":221,"props":11265,"children":11266},{"separator":223},[11267],{"type":155,"value":226},{"type":150,"tag":221,"props":11269,"children":11270},{},[11271],{"type":155,"value":247},{"type":150,"tag":249,"props":11273,"children":11274},{},[11275],{"type":155,"value":253},{"type":150,"tag":221,"props":11277,"children":11278},{"separator":223},[11279],{"type":155,"value":226},{"type":150,"tag":1046,"props":11281,"children":11282},{},[11283,11287,11291],{"type":150,"tag":209,"props":11284,"children":11285},{},[11286],{"type":155,"value":3265},{"type":150,"tag":209,"props":11288,"children":11289},{},[11290],{"type":155,"value":1698},{"type":150,"tag":201,"props":11292,"children":11293},{},[11294,11298,11302],{"type":150,"tag":221,"props":11295,"children":11296},{"stretchy":789},[11297],{"type":155,"value":792},{"type":150,"tag":209,"props":11299,"children":11300},{},[11301],{"type":155,"value":7913},{"type":150,"tag":221,"props":11303,"children":11304},{"stretchy":789},[11305],{"type":155,"value":802},{"type":150,"tag":221,"props":11307,"children":11308},{"separator":223},[11309],{"type":155,"value":226},{"type":150,"tag":1046,"props":11311,"children":11312},{},[11313,11317,11321],{"type":150,"tag":209,"props":11314,"children":11315},{},[11316],{"type":155,"value":3265},{"type":150,"tag":209,"props":11318,"children":11319},{},[11320],{"type":155,"value":1737},{"type":150,"tag":201,"props":11322,"children":11323},{},[11324,11328,11332],{"type":150,"tag":221,"props":11325,"children":11326},{"stretchy":789},[11327],{"type":155,"value":792},{"type":150,"tag":215,"props":11329,"children":11330},{},[11331],{"type":155,"value":219},{"type":150,"tag":221,"props":11333,"children":11334},{"stretchy":789},[11335],{"type":155,"value":802},{"type":150,"tag":221,"props":11337,"children":11338},{"separator":223},[11339],{"type":155,"value":226},{"type":150,"tag":221,"props":11341,"children":11342},{},[11343],{"type":155,"value":247},{"type":150,"tag":249,"props":11345,"children":11346},{},[11347],{"type":155,"value":253},{"type":150,"tag":221,"props":11349,"children":11350},{"separator":223},[11351],{"type":155,"value":226},{"type":150,"tag":1046,"props":11353,"children":11354},{},[11355,11359,11363],{"type":150,"tag":209,"props":11356,"children":11357},{},[11358],{"type":155,"value":3265},{"type":150,"tag":209,"props":11360,"children":11361},{},[11362],{"type":155,"value":1737},{"type":150,"tag":201,"props":11364,"children":11365},{},[11366,11370,11374],{"type":150,"tag":221,"props":11367,"children":11368},{"stretchy":789},[11369],{"type":155,"value":792},{"type":150,"tag":209,"props":11371,"children":11372},{},[11373],{"type":155,"value":7913},{"type":150,"tag":221,"props":11375,"children":11376},{"stretchy":789},[11377],{"type":155,"value":802},{"type":150,"tag":221,"props":11379,"children":11380},{"stretchy":789},[11381],{"type":155,"value":1148},{"type":150,"tag":221,"props":11383,"children":11384},{},[11385],{"type":155,"value":563},{"type":150,"tag":565,"props":11387,"children":11388},{},[11389,11393],{"type":150,"tag":209,"props":11390,"children":11391},{"mathvariant":570},[11392],{"type":155,"value":573},{"type":150,"tag":201,"props":11394,"children":11395},{},[11396,11400,11404,11408,11412,11423,11427,11438],{"type":150,"tag":209,"props":11397,"children":11398},{},[11399],{"type":155,"value":578},{"type":150,"tag":221,"props":11401,"children":11402},{},[11403],{"type":155,"value":5380},{"type":150,"tag":209,"props":11405,"children":11406},{},[11407],{"type":155,"value":7913},{"type":150,"tag":221,"props":11409,"children":11410},{"stretchy":789},[11411],{"type":155,"value":792},{"type":150,"tag":205,"props":11413,"children":11414},{},[11415,11419],{"type":150,"tag":209,"props":11416,"children":11417},{},[11418],{"type":155,"value":578},{"type":150,"tag":209,"props":11420,"children":11421},{},[11422],{"type":155,"value":1698},{"type":150,"tag":221,"props":11424,"children":11425},{},[11426],{"type":155,"value":4962},{"type":150,"tag":205,"props":11428,"children":11429},{},[11430,11434],{"type":150,"tag":209,"props":11431,"children":11432},{},[11433],{"type":155,"value":578},{"type":150,"tag":209,"props":11435,"children":11436},{},[11437],{"type":155,"value":1737},{"type":150,"tag":221,"props":11439,"children":11440},{"stretchy":789},[11441],{"type":155,"value":802},{"type":150,"tag":271,"props":11443,"children":11444},{"encoding":273},[11445],{"type":155,"value":11446},"c_i = x_i W_c, \\qquad W_c = [W_k^{(1)}, \\cdots, W_k^{(g)}, W_v^{(1)}, \\cdots, W_v^{(g)}] \\in \\mathbb{R}^{d \\times g(d_k + d_v)}",{"type":150,"tag":181,"props":11448,"children":11450},{"className":11449,"ariaHidden":223},[280],[11451,11529,11736,12186],{"type":150,"tag":181,"props":11452,"children":11454},{"className":11453},[285],[11455,11459,11516,11520,11525],{"type":150,"tag":181,"props":11456,"children":11458},{"className":11457,"style":1165},[290],[],{"type":150,"tag":181,"props":11460,"children":11462},{"className":11461},[296],[11463,11468],{"type":150,"tag":181,"props":11464,"children":11466},{"className":11465},[296,301],[11467],{"type":155,"value":9859},{"type":150,"tag":181,"props":11469,"children":11471},{"className":11470},[307],[11472],{"type":150,"tag":181,"props":11473,"children":11475},{"className":11474},[312,313],[11476,11505],{"type":150,"tag":181,"props":11477,"children":11479},{"className":11478},[318],[11480,11500],{"type":150,"tag":181,"props":11481,"children":11483},{"className":11482,"style":621},[323],[11484],{"type":150,"tag":181,"props":11485,"children":11486},{"style":328},[11487,11491],{"type":150,"tag":181,"props":11488,"children":11490},{"className":11489,"style":334},[333],[],{"type":150,"tag":181,"props":11492,"children":11494},{"className":11493},[339,340,341,342],[11495],{"type":150,"tag":181,"props":11496,"children":11498},{"className":11497},[296,301,342],[11499],{"type":155,"value":558},{"type":150,"tag":181,"props":11501,"children":11503},{"className":11502},[352],[11504],{"type":155,"value":355},{"type":150,"tag":181,"props":11506,"children":11508},{"className":11507},[318],[11509],{"type":150,"tag":181,"props":11510,"children":11512},{"className":11511,"style":363},[323],[11513],{"type":150,"tag":181,"props":11514,"children":11515},{},[],{"type":150,"tag":181,"props":11517,"children":11519},{"className":11518,"style":658},[377],[],{"type":150,"tag":181,"props":11521,"children":11523},{"className":11522},[663],[11524],{"type":155,"value":865},{"type":150,"tag":181,"props":11526,"children":11528},{"className":11527,"style":658},[377],[],{"type":150,"tag":181,"props":11530,"children":11532},{"className":11531},[285],[11533,11538,11595,11652,11657,11662,11666,11723,11727,11732],{"type":150,"tag":181,"props":11534,"children":11537},{"className":11535,"style":11536},[290],"height:0.8778em;vertical-align:-0.1944em;",[],{"type":150,"tag":181,"props":11539,"children":11541},{"className":11540},[296],[11542,11547],{"type":150,"tag":181,"props":11543,"children":11545},{"className":11544},[296,301],[11546],{"type":155,"value":213},{"type":150,"tag":181,"props":11548,"children":11550},{"className":11549},[307],[11551],{"type":150,"tag":181,"props":11552,"children":11554},{"className":11553},[312,313],[11555,11584],{"type":150,"tag":181,"props":11556,"children":11558},{"className":11557},[318],[11559,11579],{"type":150,"tag":181,"props":11560,"children":11562},{"className":11561,"style":621},[323],[11563],{"type":150,"tag":181,"props":11564,"children":11565},{"style":328},[11566,11570],{"type":150,"tag":181,"props":11567,"children":11569},{"className":11568,"style":334},[333],[],{"type":150,"tag":181,"props":11571,"children":11573},{"className":11572},[339,340,341,342],[11574],{"type":150,"tag":181,"props":11575,"children":11577},{"className":11576},[296,301,342],[11578],{"type":155,"value":558},{"type":150,"tag":181,"props":11580,"children":11582},{"className":11581},[352],[11583],{"type":155,"value":355},{"type":150,"tag":181,"props":11585,"children":11587},{"className":11586},[318],[11588],{"type":150,"tag":181,"props":11589,"children":11591},{"className":11590,"style":363},[323],[11592],{"type":150,"tag":181,"props":11593,"children":11594},{},[],{"type":150,"tag":181,"props":11596,"children":11598},{"className":11597},[296],[11599,11604],{"type":150,"tag":181,"props":11600,"children":11602},{"className":11601,"style":3691},[296,301],[11603],{"type":155,"value":3265},{"type":150,"tag":181,"props":11605,"children":11607},{"className":11606},[307],[11608],{"type":150,"tag":181,"props":11609,"children":11611},{"className":11610},[312,313],[11612,11641],{"type":150,"tag":181,"props":11613,"children":11615},{"className":11614},[318],[11616,11636],{"type":150,"tag":181,"props":11617,"children":11619},{"className":11618,"style":4865},[323],[11620],{"type":150,"tag":181,"props":11621,"children":11622},{"style":11120},[11623,11627],{"type":150,"tag":181,"props":11624,"children":11626},{"className":11625,"style":334},[333],[],{"type":150,"tag":181,"props":11628,"children":11630},{"className":11629},[339,340,341,342],[11631],{"type":150,"tag":181,"props":11632,"children":11634},{"className":11633},[296,301,342],[11635],{"type":155,"value":9859},{"type":150,"tag":181,"props":11637,"children":11639},{"className":11638},[352],[11640],{"type":155,"value":355},{"type":150,"tag":181,"props":11642,"children":11644},{"className":11643},[318],[11645],{"type":150,"tag":181,"props":11646,"children":11648},{"className":11647,"style":363},[323],[11649],{"type":150,"tag":181,"props":11650,"children":11651},{},[],{"type":150,"tag":181,"props":11653,"children":11655},{"className":11654},[371],[11656],{"type":155,"value":226},{"type":150,"tag":181,"props":11658,"children":11661},{"className":11659,"style":11660},[377],"margin-right:2em;",[],{"type":150,"tag":181,"props":11663,"children":11665},{"className":11664,"style":378},[377],[],{"type":150,"tag":181,"props":11667,"children":11669},{"className":11668},[296],[11670,11675],{"type":150,"tag":181,"props":11671,"children":11673},{"className":11672,"style":3691},[296,301],[11674],{"type":155,"value":3265},{"type":150,"tag":181,"props":11676,"children":11678},{"className":11677},[307],[11679],{"type":150,"tag":181,"props":11680,"children":11682},{"className":11681},[312,313],[11683,11712],{"type":150,"tag":181,"props":11684,"children":11686},{"className":11685},[318],[11687,11707],{"type":150,"tag":181,"props":11688,"children":11690},{"className":11689,"style":4865},[323],[11691],{"type":150,"tag":181,"props":11692,"children":11693},{"style":11120},[11694,11698],{"type":150,"tag":181,"props":11695,"children":11697},{"className":11696,"style":334},[333],[],{"type":150,"tag":181,"props":11699,"children":11701},{"className":11700},[339,340,341,342],[11702],{"type":150,"tag":181,"props":11703,"children":11705},{"className":11704},[296,301,342],[11706],{"type":155,"value":9859},{"type":150,"tag":181,"props":11708,"children":11710},{"className":11709},[352],[11711],{"type":155,"value":355},{"type":150,"tag":181,"props":11713,"children":11715},{"className":11714},[318],[11716],{"type":150,"tag":181,"props":11717,"children":11719},{"className":11718,"style":363},[323],[11720],{"type":150,"tag":181,"props":11721,"children":11722},{},[],{"type":150,"tag":181,"props":11724,"children":11726},{"className":11725,"style":658},[377],[],{"type":150,"tag":181,"props":11728,"children":11730},{"className":11729},[663],[11731],{"type":155,"value":865},{"type":150,"tag":181,"props":11733,"children":11735},{"className":11734,"style":658},[377],[],{"type":150,"tag":181,"props":11737,"children":11739},{"className":11738},[285],[11740,11744,11749,11836,11841,11845,11850,11854,11858,11863,11867,11954,11959,11963,12050,12055,12059,12064,12068,12072,12077,12081,12168,12173,12177,12182],{"type":150,"tag":181,"props":11741,"children":11743},{"className":11742,"style":4022},[290],[],{"type":150,"tag":181,"props":11745,"children":11747},{"className":11746},[824],[11748],{"type":155,"value":1044},{"type":150,"tag":181,"props":11750,"children":11752},{"className":11751},[296],[11753,11758],{"type":150,"tag":181,"props":11754,"children":11756},{"className":11755,"style":3691},[296,301],[11757],{"type":155,"value":3265},{"type":150,"tag":181,"props":11759,"children":11761},{"className":11760},[307],[11762],{"type":150,"tag":181,"props":11763,"children":11765},{"className":11764},[312,313],[11766,11825],{"type":150,"tag":181,"props":11767,"children":11769},{"className":11768},[318],[11770,11820],{"type":150,"tag":181,"props":11771,"children":11773},{"className":11772,"style":1276},[323],[11774,11790],{"type":150,"tag":181,"props":11775,"children":11776},{"style":4108},[11777,11781],{"type":150,"tag":181,"props":11778,"children":11780},{"className":11779,"style":334},[333],[],{"type":150,"tag":181,"props":11782,"children":11784},{"className":11783},[339,340,341,342],[11785],{"type":150,"tag":181,"props":11786,"children":11788},{"className":11787,"style":2249},[296,301,342],[11789],{"type":155,"value":1698},{"type":150,"tag":181,"props":11791,"children":11792},{"style":1297},[11793,11797],{"type":150,"tag":181,"props":11794,"children":11796},{"className":11795,"style":334},[333],[],{"type":150,"tag":181,"props":11798,"children":11800},{"className":11799},[339,340,341,342],[11801],{"type":150,"tag":181,"props":11802,"children":11804},{"className":11803},[296,342],[11805,11810,11815],{"type":150,"tag":181,"props":11806,"children":11808},{"className":11807},[824,342],[11809],{"type":155,"value":792},{"type":150,"tag":181,"props":11811,"children":11813},{"className":11812},[296,342],[11814],{"type":155,"value":219},{"type":150,"tag":181,"props":11816,"children":11818},{"className":11817},[835,342],[11819],{"type":155,"value":802},{"type":150,"tag":181,"props":11821,"children":11823},{"className":11822},[352],[11824],{"type":155,"value":355},{"type":150,"tag":181,"props":11826,"children":11828},{"className":11827},[318],[11829],{"type":150,"tag":181,"props":11830,"children":11832},{"className":11831,"style":4165},[323],[11833],{"type":150,"tag":181,"props":11834,"children":11835},{},[],{"type":150,"tag":181,"props":11837,"children":11839},{"className":11838},[371],[11840],{"type":155,"value":226},{"type":150,"tag":181,"props":11842,"children":11844},{"className":11843,"style":378},[377],[],{"type":150,"tag":181,"props":11846,"children":11848},{"className":11847},[449],[11849],{"type":155,"value":247},{"type":150,"tag":181,"props":11851,"children":11853},{"className":11852,"style":378},[377],[],{"type":150,"tag":181,"props":11855,"children":11857},{"className":11856,"style":378},[377],[],{"type":150,"tag":181,"props":11859,"children":11861},{"className":11860},[371],[11862],{"type":155,"value":226},{"type":150,"tag":181,"props":11864,"children":11866},{"className":11865,"style":378},[377],[],{"type":150,"tag":181,"props":11868,"children":11870},{"className":11869},[296],[11871,11876],{"type":150,"tag":181,"props":11872,"children":11874},{"className":11873,"style":3691},[296,301],[11875],{"type":155,"value":3265},{"type":150,"tag":181,"props":11877,"children":11879},{"className":11878},[307],[11880],{"type":150,"tag":181,"props":11881,"children":11883},{"className":11882},[312,313],[11884,11943],{"type":150,"tag":181,"props":11885,"children":11887},{"className":11886},[318],[11888,11938],{"type":150,"tag":181,"props":11889,"children":11891},{"className":11890,"style":1276},[323],[11892,11908],{"type":150,"tag":181,"props":11893,"children":11894},{"style":4108},[11895,11899],{"type":150,"tag":181,"props":11896,"children":11898},{"className":11897,"style":334},[333],[],{"type":150,"tag":181,"props":11900,"children":11902},{"className":11901},[339,340,341,342],[11903],{"type":150,"tag":181,"props":11904,"children":11906},{"className":11905,"style":2249},[296,301,342],[11907],{"type":155,"value":1698},{"type":150,"tag":181,"props":11909,"children":11910},{"style":1297},[11911,11915],{"type":150,"tag":181,"props":11912,"children":11914},{"className":11913,"style":334},[333],[],{"type":150,"tag":181,"props":11916,"children":11918},{"className":11917},[339,340,341,342],[11919],{"type":150,"tag":181,"props":11920,"children":11922},{"className":11921},[296,342],[11923,11928,11933],{"type":150,"tag":181,"props":11924,"children":11926},{"className":11925},[824,342],[11927],{"type":155,"value":792},{"type":150,"tag":181,"props":11929,"children":11931},{"className":11930,"style":2151},[296,301,342],[11932],{"type":155,"value":7913},{"type":150,"tag":181,"props":11934,"children":11936},{"className":11935},[835,342],[11937],{"type":155,"value":802},{"type":150,"tag":181,"props":11939,"children":11941},{"className":11940},[352],[11942],{"type":155,"value":355},{"type":150,"tag":181,"props":11944,"children":11946},{"className":11945},[318],[11947],{"type":150,"tag":181,"props":11948,"children":11950},{"className":11949,"style":4165},[323],[11951],{"type":150,"tag":181,"props":11952,"children":11953},{},[],{"type":150,"tag":181,"props":11955,"children":11957},{"className":11956},[371],[11958],{"type":155,"value":226},{"type":150,"tag":181,"props":11960,"children":11962},{"className":11961,"style":378},[377],[],{"type":150,"tag":181,"props":11964,"children":11966},{"className":11965},[296],[11967,11972],{"type":150,"tag":181,"props":11968,"children":11970},{"className":11969,"style":3691},[296,301],[11971],{"type":155,"value":3265},{"type":150,"tag":181,"props":11973,"children":11975},{"className":11974},[307],[11976],{"type":150,"tag":181,"props":11977,"children":11979},{"className":11978},[312,313],[11980,12039],{"type":150,"tag":181,"props":11981,"children":11983},{"className":11982},[318],[11984,12034],{"type":150,"tag":181,"props":11985,"children":11987},{"className":11986,"style":3709},[323],[11988,12004],{"type":150,"tag":181,"props":11989,"children":11990},{"style":3713},[11991,11995],{"type":150,"tag":181,"props":11992,"children":11994},{"className":11993,"style":334},[333],[],{"type":150,"tag":181,"props":11996,"children":11998},{"className":11997},[339,340,341,342],[11999],{"type":150,"tag":181,"props":12000,"children":12002},{"className":12001,"style":2151},[296,301,342],[12003],{"type":155,"value":1737},{"type":150,"tag":181,"props":12005,"children":12006},{"style":3730},[12007,12011],{"type":150,"tag":181,"props":12008,"children":12010},{"className":12009,"style":334},[333],[],{"type":150,"tag":181,"props":12012,"children":12014},{"className":12013},[339,340,341,342],[12015],{"type":150,"tag":181,"props":12016,"children":12018},{"className":12017},[296,342],[12019,12024,12029],{"type":150,"tag":181,"props":12020,"children":12022},{"className":12021},[824,342],[12023],{"type":155,"value":792},{"type":150,"tag":181,"props":12025,"children":12027},{"className":12026},[296,342],[12028],{"type":155,"value":219},{"type":150,"tag":181,"props":12030,"children":12032},{"className":12031},[835,342],[12033],{"type":155,"value":802},{"type":150,"tag":181,"props":12035,"children":12037},{"className":12036},[352],[12038],{"type":155,"value":355},{"type":150,"tag":181,"props":12040,"children":12042},{"className":12041},[318],[12043],{"type":150,"tag":181,"props":12044,"children":12046},{"className":12045,"style":4550},[323],[12047],{"type":150,"tag":181,"props":12048,"children":12049},{},[],{"type":150,"tag":181,"props":12051,"children":12053},{"className":12052},[371],[12054],{"type":155,"value":226},{"type":150,"tag":181,"props":12056,"children":12058},{"className":12057,"style":378},[377],[],{"type":150,"tag":181,"props":12060,"children":12062},{"className":12061},[449],[12063],{"type":155,"value":247},{"type":150,"tag":181,"props":12065,"children":12067},{"className":12066,"style":378},[377],[],{"type":150,"tag":181,"props":12069,"children":12071},{"className":12070,"style":378},[377],[],{"type":150,"tag":181,"props":12073,"children":12075},{"className":12074},[371],[12076],{"type":155,"value":226},{"type":150,"tag":181,"props":12078,"children":12080},{"className":12079,"style":378},[377],[],{"type":150,"tag":181,"props":12082,"children":12084},{"className":12083},[296],[12085,12090],{"type":150,"tag":181,"props":12086,"children":12088},{"className":12087,"style":3691},[296,301],[12089],{"type":155,"value":3265},{"type":150,"tag":181,"props":12091,"children":12093},{"className":12092},[307],[12094],{"type":150,"tag":181,"props":12095,"children":12097},{"className":12096},[312,313],[12098,12157],{"type":150,"tag":181,"props":12099,"children":12101},{"className":12100},[318],[12102,12152],{"type":150,"tag":181,"props":12103,"children":12105},{"className":12104,"style":3709},[323],[12106,12122],{"type":150,"tag":181,"props":12107,"children":12108},{"style":3713},[12109,12113],{"type":150,"tag":181,"props":12110,"children":12112},{"className":12111,"style":334},[333],[],{"type":150,"tag":181,"props":12114,"children":12116},{"className":12115},[339,340,341,342],[12117],{"type":150,"tag":181,"props":12118,"children":12120},{"className":12119,"style":2151},[296,301,342],[12121],{"type":155,"value":1737},{"type":150,"tag":181,"props":12123,"children":12124},{"style":3730},[12125,12129],{"type":150,"tag":181,"props":12126,"children":12128},{"className":12127,"style":334},[333],[],{"type":150,"tag":181,"props":12130,"children":12132},{"className":12131},[339,340,341,342],[12133],{"type":150,"tag":181,"props":12134,"children":12136},{"className":12135},[296,342],[12137,12142,12147],{"type":150,"tag":181,"props":12138,"children":12140},{"className":12139},[824,342],[12141],{"type":155,"value":792},{"type":150,"tag":181,"props":12143,"children":12145},{"className":12144,"style":2151},[296,301,342],[12146],{"type":155,"value":7913},{"type":150,"tag":181,"props":12148,"children":12150},{"className":12149},[835,342],[12151],{"type":155,"value":802},{"type":150,"tag":181,"props":12153,"children":12155},{"className":12154},[352],[12156],{"type":155,"value":355},{"type":150,"tag":181,"props":12158,"children":12160},{"className":12159},[318],[12161],{"type":150,"tag":181,"props":12162,"children":12164},{"className":12163,"style":4550},[323],[12165],{"type":150,"tag":181,"props":12166,"children":12167},{},[],{"type":150,"tag":181,"props":12169,"children":12171},{"className":12170},[835],[12172],{"type":155,"value":1148},{"type":150,"tag":181,"props":12174,"children":12176},{"className":12175,"style":658},[377],[],{"type":150,"tag":181,"props":12178,"children":12180},{"className":12179},[663],[12181],{"type":155,"value":563},{"type":150,"tag":181,"props":12183,"children":12185},{"className":12184,"style":658},[377],[],{"type":150,"tag":181,"props":12187,"children":12189},{"className":12188},[285],[12190,12194],{"type":150,"tag":181,"props":12191,"children":12193},{"className":12192,"style":3709},[290],[],{"type":150,"tag":181,"props":12195,"children":12197},{"className":12196},[296],[12198,12203],{"type":150,"tag":181,"props":12199,"children":12201},{"className":12200},[296,686],[12202],{"type":155,"value":573},{"type":150,"tag":181,"props":12204,"children":12206},{"className":12205},[307],[12207],{"type":150,"tag":181,"props":12208,"children":12210},{"className":12209},[312],[12211],{"type":150,"tag":181,"props":12212,"children":12214},{"className":12213},[318],[12215],{"type":150,"tag":181,"props":12216,"children":12218},{"className":12217,"style":3709},[323],[12219],{"type":150,"tag":181,"props":12220,"children":12221},{"style":3730},[12222,12226],{"type":150,"tag":181,"props":12223,"children":12225},{"className":12224,"style":334},[333],[],{"type":150,"tag":181,"props":12227,"children":12229},{"className":12228},[339,340,341,342],[12230],{"type":150,"tag":181,"props":12231,"children":12233},{"className":12232},[296,342],[12234,12239,12244,12249,12254,12311,12316,12373],{"type":150,"tag":181,"props":12235,"children":12237},{"className":12236},[296,301,342],[12238],{"type":155,"value":578},{"type":150,"tag":181,"props":12240,"children":12242},{"className":12241},[4998,342],[12243],{"type":155,"value":5380},{"type":150,"tag":181,"props":12245,"children":12247},{"className":12246,"style":2151},[296,301,342],[12248],{"type":155,"value":7913},{"type":150,"tag":181,"props":12250,"children":12252},{"className":12251},[824,342],[12253],{"type":155,"value":792},{"type":150,"tag":181,"props":12255,"children":12257},{"className":12256},[296,342],[12258,12263],{"type":150,"tag":181,"props":12259,"children":12261},{"className":12260},[296,301,342],[12262],{"type":155,"value":578},{"type":150,"tag":181,"props":12264,"children":12266},{"className":12265},[307],[12267],{"type":150,"tag":181,"props":12268,"children":12270},{"className":12269},[312,313],[12271,12300],{"type":150,"tag":181,"props":12272,"children":12274},{"className":12273},[318],[12275,12295],{"type":150,"tag":181,"props":12276,"children":12278},{"className":12277,"style":3862},[323],[12279],{"type":150,"tag":181,"props":12280,"children":12281},{"style":3866},[12282,12286],{"type":150,"tag":181,"props":12283,"children":12285},{"className":12284,"style":3871},[333],[],{"type":150,"tag":181,"props":12287,"children":12289},{"className":12288},[339,3876,3877,342],[12290],{"type":150,"tag":181,"props":12291,"children":12293},{"className":12292,"style":2249},[296,301,342],[12294],{"type":155,"value":1698},{"type":150,"tag":181,"props":12296,"children":12298},{"className":12297},[352],[12299],{"type":155,"value":355},{"type":150,"tag":181,"props":12301,"children":12303},{"className":12302},[318],[12304],{"type":150,"tag":181,"props":12305,"children":12307},{"className":12306,"style":3896},[323],[12308],{"type":150,"tag":181,"props":12309,"children":12310},{},[],{"type":150,"tag":181,"props":12312,"children":12314},{"className":12313},[4998,342],[12315],{"type":155,"value":4962},{"type":150,"tag":181,"props":12317,"children":12319},{"className":12318},[296,342],[12320,12325],{"type":150,"tag":181,"props":12321,"children":12323},{"className":12322},[296,301,342],[12324],{"type":155,"value":578},{"type":150,"tag":181,"props":12326,"children":12328},{"className":12327},[307],[12329],{"type":150,"tag":181,"props":12330,"children":12332},{"className":12331},[312,313],[12333,12362],{"type":150,"tag":181,"props":12334,"children":12336},{"className":12335},[318],[12337,12357],{"type":150,"tag":181,"props":12338,"children":12340},{"className":12339,"style":4640},[323],[12341],{"type":150,"tag":181,"props":12342,"children":12343},{"style":4644},[12344,12348],{"type":150,"tag":181,"props":12345,"children":12347},{"className":12346,"style":3871},[333],[],{"type":150,"tag":181,"props":12349,"children":12351},{"className":12350},[339,3876,3877,342],[12352],{"type":150,"tag":181,"props":12353,"children":12355},{"className":12354,"style":2151},[296,301,342],[12356],{"type":155,"value":1737},{"type":150,"tag":181,"props":12358,"children":12360},{"className":12359},[352],[12361],{"type":155,"value":355},{"type":150,"tag":181,"props":12363,"children":12365},{"className":12364},[318],[12366],{"type":150,"tag":181,"props":12367,"children":12369},{"className":12368,"style":4671},[323],[12370],{"type":150,"tag":181,"props":12371,"children":12372},{},[],{"type":150,"tag":181,"props":12374,"children":12376},{"className":12375},[835,342],[12377],{"type":155,"value":802},{"type":150,"tag":151,"props":12379,"children":12380},{},[12381,12383,12733,12735,12931,12933,12938,12940,12982,12984,13044,13046,13088],{"type":155,"value":12382},"因为 ",{"type":150,"tag":181,"props":12384,"children":12386},{"className":12385},[184],[12387,12467],{"type":150,"tag":181,"props":12388,"children":12390},{"className":12389},[189],[12391],{"type":150,"tag":192,"props":12392,"children":12393},{"xmlns":194},[12394],{"type":150,"tag":197,"props":12395,"children":12396},{},[12397,12462],{"type":150,"tag":201,"props":12398,"children":12399},{},[12400,12411,12415,12419,12423,12434,12438,12449,12453,12458],{"type":150,"tag":205,"props":12401,"children":12402},{},[12403,12407],{"type":150,"tag":209,"props":12404,"children":12405},{},[12406],{"type":155,"value":578},{"type":150,"tag":209,"props":12408,"children":12409},{},[12410],{"type":155,"value":9859},{"type":150,"tag":221,"props":12412,"children":12413},{},[12414],{"type":155,"value":865},{"type":150,"tag":209,"props":12416,"children":12417},{},[12418],{"type":155,"value":7913},{"type":150,"tag":221,"props":12420,"children":12421},{"stretchy":789},[12422],{"type":155,"value":792},{"type":150,"tag":205,"props":12424,"children":12425},{},[12426,12430],{"type":150,"tag":209,"props":12427,"children":12428},{},[12429],{"type":155,"value":578},{"type":150,"tag":209,"props":12431,"children":12432},{},[12433],{"type":155,"value":1698},{"type":150,"tag":221,"props":12435,"children":12436},{},[12437],{"type":155,"value":4962},{"type":150,"tag":205,"props":12439,"children":12440},{},[12441,12445],{"type":150,"tag":209,"props":12442,"children":12443},{},[12444],{"type":155,"value":578},{"type":150,"tag":209,"props":12446,"children":12447},{},[12448],{"type":155,"value":1737},{"type":150,"tag":221,"props":12450,"children":12451},{"stretchy":789},[12452],{"type":155,"value":802},{"type":150,"tag":221,"props":12454,"children":12455},{},[12456],{"type":155,"value":12457},"\u003C",{"type":150,"tag":209,"props":12459,"children":12460},{},[12461],{"type":155,"value":578},{"type":150,"tag":271,"props":12463,"children":12464},{"encoding":273},[12465],{"type":155,"value":12466},"d_c = g(d_k + d_v) \u003C d",{"type":150,"tag":181,"props":12468,"children":12470},{"className":12469,"ariaHidden":223},[280],[12471,12549,12637,12720],{"type":150,"tag":181,"props":12472,"children":12474},{"className":12473},[285],[12475,12479,12536,12540,12545],{"type":150,"tag":181,"props":12476,"children":12478},{"className":12477,"style":4761},[290],[],{"type":150,"tag":181,"props":12480,"children":12482},{"className":12481},[296],[12483,12488],{"type":150,"tag":181,"props":12484,"children":12486},{"className":12485},[296,301],[12487],{"type":155,"value":578},{"type":150,"tag":181,"props":12489,"children":12491},{"className":12490},[307],[12492],{"type":150,"tag":181,"props":12493,"children":12495},{"className":12494},[312,313],[12496,12525],{"type":150,"tag":181,"props":12497,"children":12499},{"className":12498},[318],[12500,12520],{"type":150,"tag":181,"props":12501,"children":12503},{"className":12502,"style":4865},[323],[12504],{"type":150,"tag":181,"props":12505,"children":12506},{"style":328},[12507,12511],{"type":150,"tag":181,"props":12508,"children":12510},{"className":12509,"style":334},[333],[],{"type":150,"tag":181,"props":12512,"children":12514},{"className":12513},[339,340,341,342],[12515],{"type":150,"tag":181,"props":12516,"children":12518},{"className":12517},[296,301,342],[12519],{"type":155,"value":9859},{"type":150,"tag":181,"props":12521,"children":12523},{"className":12522},[352],[12524],{"type":155,"value":355},{"type":150,"tag":181,"props":12526,"children":12528},{"className":12527},[318],[12529],{"type":150,"tag":181,"props":12530,"children":12532},{"className":12531,"style":363},[323],[12533],{"type":150,"tag":181,"props":12534,"children":12535},{},[],{"type":150,"tag":181,"props":12537,"children":12539},{"className":12538,"style":658},[377],[],{"type":150,"tag":181,"props":12541,"children":12543},{"className":12542},[663],[12544],{"type":155,"value":865},{"type":150,"tag":181,"props":12546,"children":12548},{"className":12547,"style":658},[377],[],{"type":150,"tag":181,"props":12550,"children":12552},{"className":12551},[285],[12553,12557,12562,12567,12624,12628,12633],{"type":150,"tag":181,"props":12554,"children":12556},{"className":12555,"style":819},[290],[],{"type":150,"tag":181,"props":12558,"children":12560},{"className":12559,"style":2151},[296,301],[12561],{"type":155,"value":7913},{"type":150,"tag":181,"props":12563,"children":12565},{"className":12564},[824],[12566],{"type":155,"value":792},{"type":150,"tag":181,"props":12568,"children":12570},{"className":12569},[296],[12571,12576],{"type":150,"tag":181,"props":12572,"children":12574},{"className":12573},[296,301],[12575],{"type":155,"value":578},{"type":150,"tag":181,"props":12577,"children":12579},{"className":12578},[307],[12580],{"type":150,"tag":181,"props":12581,"children":12583},{"className":12582},[312,313],[12584,12613],{"type":150,"tag":181,"props":12585,"children":12587},{"className":12586},[318],[12588,12608],{"type":150,"tag":181,"props":12589,"children":12591},{"className":12590,"style":493},[323],[12592],{"type":150,"tag":181,"props":12593,"children":12594},{"style":328},[12595,12599],{"type":150,"tag":181,"props":12596,"children":12598},{"className":12597,"style":334},[333],[],{"type":150,"tag":181,"props":12600,"children":12602},{"className":12601},[339,340,341,342],[12603],{"type":150,"tag":181,"props":12604,"children":12606},{"className":12605,"style":2249},[296,301,342],[12607],{"type":155,"value":1698},{"type":150,"tag":181,"props":12609,"children":12611},{"className":12610},[352],[12612],{"type":155,"value":355},{"type":150,"tag":181,"props":12614,"children":12616},{"className":12615},[318],[12617],{"type":150,"tag":181,"props":12618,"children":12620},{"className":12619,"style":363},[323],[12621],{"type":150,"tag":181,"props":12622,"children":12623},{},[],{"type":150,"tag":181,"props":12625,"children":12627},{"className":12626,"style":4993},[377],[],{"type":150,"tag":181,"props":12629,"children":12631},{"className":12630},[4998],[12632],{"type":155,"value":4962},{"type":150,"tag":181,"props":12634,"children":12636},{"className":12635,"style":4993},[377],[],{"type":150,"tag":181,"props":12638,"children":12640},{"className":12639},[285],[12641,12645,12702,12707,12711,12716],{"type":150,"tag":181,"props":12642,"children":12644},{"className":12643,"style":819},[290],[],{"type":150,"tag":181,"props":12646,"children":12648},{"className":12647},[296],[12649,12654],{"type":150,"tag":181,"props":12650,"children":12652},{"className":12651},[296,301],[12653],{"type":155,"value":578},{"type":150,"tag":181,"props":12655,"children":12657},{"className":12656},[307],[12658],{"type":150,"tag":181,"props":12659,"children":12661},{"className":12660},[312,313],[12662,12691],{"type":150,"tag":181,"props":12663,"children":12665},{"className":12664},[318],[12666,12686],{"type":150,"tag":181,"props":12667,"children":12669},{"className":12668,"style":4865},[323],[12670],{"type":150,"tag":181,"props":12671,"children":12672},{"style":328},[12673,12677],{"type":150,"tag":181,"props":12674,"children":12676},{"className":12675,"style":334},[333],[],{"type":150,"tag":181,"props":12678,"children":12680},{"className":12679},[339,340,341,342],[12681],{"type":150,"tag":181,"props":12682,"children":12684},{"className":12683,"style":2151},[296,301,342],[12685],{"type":155,"value":1737},{"type":150,"tag":181,"props":12687,"children":12689},{"className":12688},[352],[12690],{"type":155,"value":355},{"type":150,"tag":181,"props":12692,"children":12694},{"className":12693},[318],[12695],{"type":150,"tag":181,"props":12696,"children":12698},{"className":12697,"style":363},[323],[12699],{"type":150,"tag":181,"props":12700,"children":12701},{},[],{"type":150,"tag":181,"props":12703,"children":12705},{"className":12704},[835],[12706],{"type":155,"value":802},{"type":150,"tag":181,"props":12708,"children":12710},{"className":12709,"style":658},[377],[],{"type":150,"tag":181,"props":12712,"children":12714},{"className":12713},[663],[12715],{"type":155,"value":12457},{"type":150,"tag":181,"props":12717,"children":12719},{"className":12718,"style":658},[377],[],{"type":150,"tag":181,"props":12721,"children":12723},{"className":12722},[285],[12724,12728],{"type":150,"tag":181,"props":12725,"children":12727},{"className":12726,"style":761},[290],[],{"type":150,"tag":181,"props":12729,"children":12731},{"className":12730},[296,301],[12732],{"type":155,"value":578},{"type":155,"value":12734},"，这一步就是在把维度压小。所以 ",{"type":150,"tag":181,"props":12736,"children":12738},{"className":12737},[184],[12739,12784],{"type":150,"tag":181,"props":12740,"children":12742},{"className":12741},[189],[12743],{"type":150,"tag":192,"props":12744,"children":12745},{"xmlns":194},[12746],{"type":150,"tag":197,"props":12747,"children":12748},{},[12749,12779],{"type":150,"tag":201,"props":12750,"children":12751},{},[12752,12763,12768],{"type":150,"tag":205,"props":12753,"children":12754},{},[12755,12759],{"type":150,"tag":209,"props":12756,"children":12757},{},[12758],{"type":155,"value":213},{"type":150,"tag":209,"props":12760,"children":12761},{},[12762],{"type":155,"value":558},{"type":150,"tag":221,"props":12764,"children":12765},{},[12766],{"type":155,"value":12767},"→",{"type":150,"tag":205,"props":12769,"children":12770},{},[12771,12775],{"type":150,"tag":209,"props":12772,"children":12773},{},[12774],{"type":155,"value":9859},{"type":150,"tag":209,"props":12776,"children":12777},{},[12778],{"type":155,"value":558},{"type":150,"tag":271,"props":12780,"children":12781},{"encoding":273},[12782],{"type":155,"value":12783},"x_i \\to c_i",{"type":150,"tag":181,"props":12785,"children":12787},{"className":12786,"ariaHidden":223},[280],[12788,12866],{"type":150,"tag":181,"props":12789,"children":12791},{"className":12790},[285],[12792,12796,12853,12857,12862],{"type":150,"tag":181,"props":12793,"children":12795},{"className":12794,"style":1165},[290],[],{"type":150,"tag":181,"props":12797,"children":12799},{"className":12798},[296],[12800,12805],{"type":150,"tag":181,"props":12801,"children":12803},{"className":12802},[296,301],[12804],{"type":155,"value":213},{"type":150,"tag":181,"props":12806,"children":12808},{"className":12807},[307],[12809],{"type":150,"tag":181,"props":12810,"children":12812},{"className":12811},[312,313],[12813,12842],{"type":150,"tag":181,"props":12814,"children":12816},{"className":12815},[318],[12817,12837],{"type":150,"tag":181,"props":12818,"children":12820},{"className":12819,"style":621},[323],[12821],{"type":150,"tag":181,"props":12822,"children":12823},{"style":328},[12824,12828],{"type":150,"tag":181,"props":12825,"children":12827},{"className":12826,"style":334},[333],[],{"type":150,"tag":181,"props":12829,"children":12831},{"className":12830},[339,340,341,342],[12832],{"type":150,"tag":181,"props":12833,"children":12835},{"className":12834},[296,301,342],[12836],{"type":155,"value":558},{"type":150,"tag":181,"props":12838,"children":12840},{"className":12839},[352],[12841],{"type":155,"value":355},{"type":150,"tag":181,"props":12843,"children":12845},{"className":12844},[318],[12846],{"type":150,"tag":181,"props":12847,"children":12849},{"className":12848,"style":363},[323],[12850],{"type":150,"tag":181,"props":12851,"children":12852},{},[],{"type":150,"tag":181,"props":12854,"children":12856},{"className":12855,"style":658},[377],[],{"type":150,"tag":181,"props":12858,"children":12860},{"className":12859},[663],[12861],{"type":155,"value":12767},{"type":150,"tag":181,"props":12863,"children":12865},{"className":12864,"style":658},[377],[],{"type":150,"tag":181,"props":12867,"children":12869},{"className":12868},[285],[12870,12874],{"type":150,"tag":181,"props":12871,"children":12873},{"className":12872,"style":1165},[290],[],{"type":150,"tag":181,"props":12875,"children":12877},{"className":12876},[296],[12878,12883],{"type":150,"tag":181,"props":12879,"children":12881},{"className":12880},[296,301],[12882],{"type":155,"value":9859},{"type":150,"tag":181,"props":12884,"children":12886},{"className":12885},[307],[12887],{"type":150,"tag":181,"props":12888,"children":12890},{"className":12889},[312,313],[12891,12920],{"type":150,"tag":181,"props":12892,"children":12894},{"className":12893},[318],[12895,12915],{"type":150,"tag":181,"props":12896,"children":12898},{"className":12897,"style":621},[323],[12899],{"type":150,"tag":181,"props":12900,"children":12901},{"style":328},[12902,12906],{"type":150,"tag":181,"props":12903,"children":12905},{"className":12904,"style":334},[333],[],{"type":150,"tag":181,"props":12907,"children":12909},{"className":12908},[339,340,341,342],[12910],{"type":150,"tag":181,"props":12911,"children":12913},{"className":12912},[296,301,342],[12914],{"type":155,"value":558},{"type":150,"tag":181,"props":12916,"children":12918},{"className":12917},[352],[12919],{"type":155,"value":355},{"type":150,"tag":181,"props":12921,"children":12923},{"className":12922},[318],[12924],{"type":150,"tag":181,"props":12925,"children":12927},{"className":12926,"style":363},[323],[12928],{"type":150,"tag":181,"props":12929,"children":12930},{},[],{"type":155,"value":12932}," 本来就是低秩投影。也就是说，光看\"低秩投影\"这一步，GQA 和 MLA 没有区别。MLA 的改进落在",{"type":150,"tag":158,"props":12934,"children":12935},{},[12936],{"type":155,"value":12937},"低秩投影之后做了什么",{"type":155,"value":12939},"。GQA 投影之后干的事很简单：把向量对半分成 K 和 V，每一份再均分成 ",{"type":150,"tag":181,"props":12941,"children":12943},{"className":12942},[184],[12944,12965],{"type":150,"tag":181,"props":12945,"children":12947},{"className":12946},[189],[12948],{"type":150,"tag":192,"props":12949,"children":12950},{"xmlns":194},[12951],{"type":150,"tag":197,"props":12952,"children":12953},{},[12954,12961],{"type":150,"tag":201,"props":12955,"children":12956},{},[12957],{"type":150,"tag":209,"props":12958,"children":12959},{},[12960],{"type":155,"value":7913},{"type":150,"tag":271,"props":12962,"children":12963},{"encoding":273},[12964],{"type":155,"value":7913},{"type":150,"tag":181,"props":12966,"children":12968},{"className":12967,"ariaHidden":223},[280],[12969],{"type":150,"tag":181,"props":12970,"children":12972},{"className":12971},[285],[12973,12977],{"type":150,"tag":181,"props":12974,"children":12976},{"className":12975,"style":291},[290],[],{"type":150,"tag":181,"props":12978,"children":12980},{"className":12979,"style":2151},[296,301],[12981],{"type":155,"value":7913},{"type":155,"value":12983}," 份，每份复制 ",{"type":150,"tag":181,"props":12985,"children":12987},{"className":12986},[184],[12988,13017],{"type":150,"tag":181,"props":12989,"children":12991},{"className":12990},[189],[12992],{"type":150,"tag":192,"props":12993,"children":12994},{"xmlns":194},[12995],{"type":150,"tag":197,"props":12996,"children":12997},{},[12998,13013],{"type":150,"tag":201,"props":12999,"children":13000},{},[13001,13005,13009],{"type":150,"tag":209,"props":13002,"children":13003},{},[13004],{"type":155,"value":745},{"type":150,"tag":209,"props":13006,"children":13007},{"mathvariant":1618},[13008],{"type":155,"value":4740},{"type":150,"tag":209,"props":13010,"children":13011},{},[13012],{"type":155,"value":7913},{"type":150,"tag":271,"props":13014,"children":13015},{"encoding":273},[13016],{"type":155,"value":8353},{"type":150,"tag":181,"props":13018,"children":13020},{"className":13019,"ariaHidden":223},[280],[13021],{"type":150,"tag":181,"props":13022,"children":13024},{"className":13023},[285],[13025,13029,13034,13039],{"type":150,"tag":181,"props":13026,"children":13028},{"className":13027,"style":819},[290],[],{"type":150,"tag":181,"props":13030,"children":13032},{"className":13031},[296,301],[13033],{"type":155,"value":745},{"type":150,"tag":181,"props":13035,"children":13037},{"className":13036},[296],[13038],{"type":155,"value":4740},{"type":150,"tag":181,"props":13040,"children":13042},{"className":13041,"style":2151},[296,301],[13043],{"type":155,"value":7913},{"type":155,"value":13045}," 次，凑够 ",{"type":150,"tag":181,"props":13047,"children":13049},{"className":13048},[184],[13050,13071],{"type":150,"tag":181,"props":13051,"children":13053},{"className":13052},[189],[13054],{"type":150,"tag":192,"props":13055,"children":13056},{"xmlns":194},[13057],{"type":150,"tag":197,"props":13058,"children":13059},{},[13060,13067],{"type":150,"tag":201,"props":13061,"children":13062},{},[13063],{"type":150,"tag":209,"props":13064,"children":13065},{},[13066],{"type":155,"value":745},{"type":150,"tag":271,"props":13068,"children":13069},{"encoding":273},[13070],{"type":155,"value":745},{"type":150,"tag":181,"props":13072,"children":13074},{"className":13073,"ariaHidden":223},[280],[13075],{"type":150,"tag":181,"props":13076,"children":13078},{"className":13077},[285],[13079,13083],{"type":150,"tag":181,"props":13080,"children":13082},{"className":13081,"style":761},[290],[],{"type":150,"tag":181,"props":13084,"children":13086},{"className":13085},[296,301],[13087],{"type":155,"value":745},{"type":155,"value":13089}," 个 head。分割和复制都是极简的线性变换。",{"type":150,"tag":151,"props":13091,"children":13092},{},[13093],{"type":155,"value":13094},"MLA 的第一步想法就是把它们换成一般的线性变换，让每个 head 拿到的 K、V 各不相同：",{"type":150,"tag":181,"props":13096,"children":13098},{"className":13097},[1003],[13099],{"type":150,"tag":181,"props":13100,"children":13102},{"className":13101},[184],[13103,13306],{"type":150,"tag":181,"props":13104,"children":13106},{"className":13105},[189],[13107],{"type":150,"tag":192,"props":13108,"children":13109},{"xmlns":194,"display":1015},[13110],{"type":150,"tag":197,"props":13111,"children":13112},{},[13113,13301],{"type":150,"tag":201,"props":13114,"children":13115},{},[13116,13142,13146,13157,13183,13187,13205,13209,13212,13238,13242,13253,13279,13283],{"type":150,"tag":1046,"props":13117,"children":13118},{},[13119,13123,13127],{"type":150,"tag":209,"props":13120,"children":13121},{},[13122],{"type":155,"value":1698},{"type":150,"tag":209,"props":13124,"children":13125},{},[13126],{"type":155,"value":558},{"type":150,"tag":201,"props":13128,"children":13129},{},[13130,13134,13138],{"type":150,"tag":221,"props":13131,"children":13132},{"stretchy":789},[13133],{"type":155,"value":792},{"type":150,"tag":209,"props":13135,"children":13136},{},[13137],{"type":155,"value":797},{"type":150,"tag":221,"props":13139,"children":13140},{"stretchy":789},[13141],{"type":155,"value":802},{"type":150,"tag":221,"props":13143,"children":13144},{},[13145],{"type":155,"value":865},{"type":150,"tag":205,"props":13147,"children":13148},{},[13149,13153],{"type":150,"tag":209,"props":13150,"children":13151},{},[13152],{"type":155,"value":9859},{"type":150,"tag":209,"props":13154,"children":13155},{},[13156],{"type":155,"value":558},{"type":150,"tag":1046,"props":13158,"children":13159},{},[13160,13164,13168],{"type":150,"tag":209,"props":13161,"children":13162},{},[13163],{"type":155,"value":3265},{"type":150,"tag":209,"props":13165,"children":13166},{},[13167],{"type":155,"value":1698},{"type":150,"tag":201,"props":13169,"children":13170},{},[13171,13175,13179],{"type":150,"tag":221,"props":13172,"children":13173},{"stretchy":789},[13174],{"type":155,"value":792},{"type":150,"tag":209,"props":13176,"children":13177},{},[13178],{"type":155,"value":797},{"type":150,"tag":221,"props":13180,"children":13181},{"stretchy":789},[13182],{"type":155,"value":802},{"type":150,"tag":221,"props":13184,"children":13185},{},[13186],{"type":155,"value":563},{"type":150,"tag":565,"props":13188,"children":13189},{},[13190,13194],{"type":150,"tag":209,"props":13191,"children":13192},{"mathvariant":570},[13193],{"type":155,"value":573},{"type":150,"tag":205,"props":13195,"children":13196},{},[13197,13201],{"type":150,"tag":209,"props":13198,"children":13199},{},[13200],{"type":155,"value":578},{"type":150,"tag":209,"props":13202,"children":13203},{},[13204],{"type":155,"value":1698},{"type":150,"tag":221,"props":13206,"children":13207},{"separator":223},[13208],{"type":155,"value":226},{"type":150,"tag":377,"props":13210,"children":13211},{"width":3313},[],{"type":150,"tag":1046,"props":13213,"children":13214},{},[13215,13219,13223],{"type":150,"tag":209,"props":13216,"children":13217},{},[13218],{"type":155,"value":1737},{"type":150,"tag":209,"props":13220,"children":13221},{},[13222],{"type":155,"value":558},{"type":150,"tag":201,"props":13224,"children":13225},{},[13226,13230,13234],{"type":150,"tag":221,"props":13227,"children":13228},{"stretchy":789},[13229],{"type":155,"value":792},{"type":150,"tag":209,"props":13231,"children":13232},{},[13233],{"type":155,"value":797},{"type":150,"tag":221,"props":13235,"children":13236},{"stretchy":789},[13237],{"type":155,"value":802},{"type":150,"tag":221,"props":13239,"children":13240},{},[13241],{"type":155,"value":865},{"type":150,"tag":205,"props":13243,"children":13244},{},[13245,13249],{"type":150,"tag":209,"props":13246,"children":13247},{},[13248],{"type":155,"value":9859},{"type":150,"tag":209,"props":13250,"children":13251},{},[13252],{"type":155,"value":558},{"type":150,"tag":1046,"props":13254,"children":13255},{},[13256,13260,13264],{"type":150,"tag":209,"props":13257,"children":13258},{},[13259],{"type":155,"value":3265},{"type":150,"tag":209,"props":13261,"children":13262},{},[13263],{"type":155,"value":1737},{"type":150,"tag":201,"props":13265,"children":13266},{},[13267,13271,13275],{"type":150,"tag":221,"props":13268,"children":13269},{"stretchy":789},[13270],{"type":155,"value":792},{"type":150,"tag":209,"props":13272,"children":13273},{},[13274],{"type":155,"value":797},{"type":150,"tag":221,"props":13276,"children":13277},{"stretchy":789},[13278],{"type":155,"value":802},{"type":150,"tag":221,"props":13280,"children":13281},{},[13282],{"type":155,"value":563},{"type":150,"tag":565,"props":13284,"children":13285},{},[13286,13290],{"type":150,"tag":209,"props":13287,"children":13288},{"mathvariant":570},[13289],{"type":155,"value":573},{"type":150,"tag":205,"props":13291,"children":13292},{},[13293,13297],{"type":150,"tag":209,"props":13294,"children":13295},{},[13296],{"type":155,"value":578},{"type":150,"tag":209,"props":13298,"children":13299},{},[13300],{"type":155,"value":1737},{"type":150,"tag":271,"props":13302,"children":13303},{"encoding":273},[13304],{"type":155,"value":13305},"k_i^{(s)} = c_i W_k^{(s)} \\in \\mathbb{R}^{d_k}, \\quad v_i^{(s)} = c_i W_v^{(s)} \\in \\mathbb{R}^{d_v}",{"type":150,"tag":181,"props":13307,"children":13309},{"className":13308,"ariaHidden":223},[280],[13310,13418,13583,13801,13966],{"type":150,"tag":181,"props":13311,"children":13313},{"className":13312},[285],[13314,13318,13405,13409,13414],{"type":150,"tag":181,"props":13315,"children":13317},{"className":13316,"style":3516},[290],[],{"type":150,"tag":181,"props":13319,"children":13321},{"className":13320},[296],[13322,13327],{"type":150,"tag":181,"props":13323,"children":13325},{"className":13324,"style":2249},[296,301],[13326],{"type":155,"value":1698},{"type":150,"tag":181,"props":13328,"children":13330},{"className":13329},[307],[13331],{"type":150,"tag":181,"props":13332,"children":13334},{"className":13333},[312,313],[13335,13394],{"type":150,"tag":181,"props":13336,"children":13338},{"className":13337},[318],[13339,13389],{"type":150,"tag":181,"props":13340,"children":13342},{"className":13341,"style":1276},[323],[13343,13359],{"type":150,"tag":181,"props":13344,"children":13345},{"style":2720},[13346,13350],{"type":150,"tag":181,"props":13347,"children":13349},{"className":13348,"style":334},[333],[],{"type":150,"tag":181,"props":13351,"children":13353},{"className":13352},[339,340,341,342],[13354],{"type":150,"tag":181,"props":13355,"children":13357},{"className":13356},[296,301,342],[13358],{"type":155,"value":558},{"type":150,"tag":181,"props":13360,"children":13361},{"style":1297},[13362,13366],{"type":150,"tag":181,"props":13363,"children":13365},{"className":13364,"style":334},[333],[],{"type":150,"tag":181,"props":13367,"children":13369},{"className":13368},[339,340,341,342],[13370],{"type":150,"tag":181,"props":13371,"children":13373},{"className":13372},[296,342],[13374,13379,13384],{"type":150,"tag":181,"props":13375,"children":13377},{"className":13376},[824,342],[13378],{"type":155,"value":792},{"type":150,"tag":181,"props":13380,"children":13382},{"className":13381},[296,301,342],[13383],{"type":155,"value":797},{"type":150,"tag":181,"props":13385,"children":13387},{"className":13386},[835,342],[13388],{"type":155,"value":802},{"type":150,"tag":181,"props":13390,"children":13392},{"className":13391},[352],[13393],{"type":155,"value":355},{"type":150,"tag":181,"props":13395,"children":13397},{"className":13396},[318],[13398],{"type":150,"tag":181,"props":13399,"children":13401},{"className":13400,"style":2782},[323],[13402],{"type":150,"tag":181,"props":13403,"children":13404},{},[],{"type":150,"tag":181,"props":13406,"children":13408},{"className":13407,"style":658},[377],[],{"type":150,"tag":181,"props":13410,"children":13412},{"className":13411},[663],[13413],{"type":155,"value":865},{"type":150,"tag":181,"props":13415,"children":13417},{"className":13416,"style":658},[377],[],{"type":150,"tag":181,"props":13419,"children":13421},{"className":13420},[285],[13422,13426,13483,13570,13574,13579],{"type":150,"tag":181,"props":13423,"children":13425},{"className":13424,"style":4022},[290],[],{"type":150,"tag":181,"props":13427,"children":13429},{"className":13428},[296],[13430,13435],{"type":150,"tag":181,"props":13431,"children":13433},{"className":13432},[296,301],[13434],{"type":155,"value":9859},{"type":150,"tag":181,"props":13436,"children":13438},{"className":13437},[307],[13439],{"type":150,"tag":181,"props":13440,"children":13442},{"className":13441},[312,313],[13443,13472],{"type":150,"tag":181,"props":13444,"children":13446},{"className":13445},[318],[13447,13467],{"type":150,"tag":181,"props":13448,"children":13450},{"className":13449,"style":621},[323],[13451],{"type":150,"tag":181,"props":13452,"children":13453},{"style":328},[13454,13458],{"type":150,"tag":181,"props":13455,"children":13457},{"className":13456,"style":334},[333],[],{"type":150,"tag":181,"props":13459,"children":13461},{"className":13460},[339,340,341,342],[13462],{"type":150,"tag":181,"props":13463,"children":13465},{"className":13464},[296,301,342],[13466],{"type":155,"value":558},{"type":150,"tag":181,"props":13468,"children":13470},{"className":13469},[352],[13471],{"type":155,"value":355},{"type":150,"tag":181,"props":13473,"children":13475},{"className":13474},[318],[13476],{"type":150,"tag":181,"props":13477,"children":13479},{"className":13478,"style":363},[323],[13480],{"type":150,"tag":181,"props":13481,"children":13482},{},[],{"type":150,"tag":181,"props":13484,"children":13486},{"className":13485},[296],[13487,13492],{"type":150,"tag":181,"props":13488,"children":13490},{"className":13489,"style":3691},[296,301],[13491],{"type":155,"value":3265},{"type":150,"tag":181,"props":13493,"children":13495},{"className":13494},[307],[13496],{"type":150,"tag":181,"props":13497,"children":13499},{"className":13498},[312,313],[13500,13559],{"type":150,"tag":181,"props":13501,"children":13503},{"className":13502},[318],[13504,13554],{"type":150,"tag":181,"props":13505,"children":13507},{"className":13506,"style":1276},[323],[13508,13524],{"type":150,"tag":181,"props":13509,"children":13510},{"style":4108},[13511,13515],{"type":150,"tag":181,"props":13512,"children":13514},{"className":13513,"style":334},[333],[],{"type":150,"tag":181,"props":13516,"children":13518},{"className":13517},[339,340,341,342],[13519],{"type":150,"tag":181,"props":13520,"children":13522},{"className":13521,"style":2249},[296,301,342],[13523],{"type":155,"value":1698},{"type":150,"tag":181,"props":13525,"children":13526},{"style":1297},[13527,13531],{"type":150,"tag":181,"props":13528,"children":13530},{"className":13529,"style":334},[333],[],{"type":150,"tag":181,"props":13532,"children":13534},{"className":13533},[339,340,341,342],[13535],{"type":150,"tag":181,"props":13536,"children":13538},{"className":13537},[296,342],[13539,13544,13549],{"type":150,"tag":181,"props":13540,"children":13542},{"className":13541},[824,342],[13543],{"type":155,"value":792},{"type":150,"tag":181,"props":13545,"children":13547},{"className":13546},[296,301,342],[13548],{"type":155,"value":797},{"type":150,"tag":181,"props":13550,"children":13552},{"className":13551},[835,342],[13553],{"type":155,"value":802},{"type":150,"tag":181,"props":13555,"children":13557},{"className":13556},[352],[13558],{"type":155,"value":355},{"type":150,"tag":181,"props":13560,"children":13562},{"className":13561},[318],[13563],{"type":150,"tag":181,"props":13564,"children":13566},{"className":13565,"style":4165},[323],[13567],{"type":150,"tag":181,"props":13568,"children":13569},{},[],{"type":150,"tag":181,"props":13571,"children":13573},{"className":13572,"style":658},[377],[],{"type":150,"tag":181,"props":13575,"children":13577},{"className":13576},[663],[13578],{"type":155,"value":563},{"type":150,"tag":181,"props":13580,"children":13582},{"className":13581,"style":658},[377],[],{"type":150,"tag":181,"props":13584,"children":13586},{"className":13585},[285],[13587,13591,13688,13693,13697,13701,13788,13792,13797],{"type":150,"tag":181,"props":13588,"children":13590},{"className":13589,"style":3516},[290],[],{"type":150,"tag":181,"props":13592,"children":13594},{"className":13593},[296],[13595,13600],{"type":150,"tag":181,"props":13596,"children":13598},{"className":13597},[296,686],[13599],{"type":155,"value":573},{"type":150,"tag":181,"props":13601,"children":13603},{"className":13602},[307],[13604],{"type":150,"tag":181,"props":13605,"children":13607},{"className":13606},[312],[13608],{"type":150,"tag":181,"props":13609,"children":13611},{"className":13610},[318],[13612],{"type":150,"tag":181,"props":13613,"children":13615},{"className":13614,"style":3821},[323],[13616],{"type":150,"tag":181,"props":13617,"children":13618},{"style":3730},[13619,13623],{"type":150,"tag":181,"props":13620,"children":13622},{"className":13621,"style":334},[333],[],{"type":150,"tag":181,"props":13624,"children":13626},{"className":13625},[339,340,341,342],[13627],{"type":150,"tag":181,"props":13628,"children":13630},{"className":13629},[296,342],[13631],{"type":150,"tag":181,"props":13632,"children":13634},{"className":13633},[296,342],[13635,13640],{"type":150,"tag":181,"props":13636,"children":13638},{"className":13637},[296,301,342],[13639],{"type":155,"value":578},{"type":150,"tag":181,"props":13641,"children":13643},{"className":13642},[307],[13644],{"type":150,"tag":181,"props":13645,"children":13647},{"className":13646},[312,313],[13648,13677],{"type":150,"tag":181,"props":13649,"children":13651},{"className":13650},[318],[13652,13672],{"type":150,"tag":181,"props":13653,"children":13655},{"className":13654,"style":3862},[323],[13656],{"type":150,"tag":181,"props":13657,"children":13658},{"style":3866},[13659,13663],{"type":150,"tag":181,"props":13660,"children":13662},{"className":13661,"style":3871},[333],[],{"type":150,"tag":181,"props":13664,"children":13666},{"className":13665},[339,3876,3877,342],[13667],{"type":150,"tag":181,"props":13668,"children":13670},{"className":13669,"style":2249},[296,301,342],[13671],{"type":155,"value":1698},{"type":150,"tag":181,"props":13673,"children":13675},{"className":13674},[352],[13676],{"type":155,"value":355},{"type":150,"tag":181,"props":13678,"children":13680},{"className":13679},[318],[13681],{"type":150,"tag":181,"props":13682,"children":13684},{"className":13683,"style":3896},[323],[13685],{"type":150,"tag":181,"props":13686,"children":13687},{},[],{"type":150,"tag":181,"props":13689,"children":13691},{"className":13690},[371],[13692],{"type":155,"value":226},{"type":150,"tag":181,"props":13694,"children":13696},{"className":13695,"style":3909},[377],[],{"type":150,"tag":181,"props":13698,"children":13700},{"className":13699,"style":378},[377],[],{"type":150,"tag":181,"props":13702,"children":13704},{"className":13703},[296],[13705,13710],{"type":150,"tag":181,"props":13706,"children":13708},{"className":13707,"style":2151},[296,301],[13709],{"type":155,"value":1737},{"type":150,"tag":181,"props":13711,"children":13713},{"className":13712},[307],[13714],{"type":150,"tag":181,"props":13715,"children":13717},{"className":13716},[312,313],[13718,13777],{"type":150,"tag":181,"props":13719,"children":13721},{"className":13720},[318],[13722,13772],{"type":150,"tag":181,"props":13723,"children":13725},{"className":13724,"style":1276},[323],[13726,13742],{"type":150,"tag":181,"props":13727,"children":13728},{"style":3114},[13729,13733],{"type":150,"tag":181,"props":13730,"children":13732},{"className":13731,"style":334},[333],[],{"type":150,"tag":181,"props":13734,"children":13736},{"className":13735},[339,340,341,342],[13737],{"type":150,"tag":181,"props":13738,"children":13740},{"className":13739},[296,301,342],[13741],{"type":155,"value":558},{"type":150,"tag":181,"props":13743,"children":13744},{"style":1297},[13745,13749],{"type":150,"tag":181,"props":13746,"children":13748},{"className":13747,"style":334},[333],[],{"type":150,"tag":181,"props":13750,"children":13752},{"className":13751},[339,340,341,342],[13753],{"type":150,"tag":181,"props":13754,"children":13756},{"className":13755},[296,342],[13757,13762,13767],{"type":150,"tag":181,"props":13758,"children":13760},{"className":13759},[824,342],[13761],{"type":155,"value":792},{"type":150,"tag":181,"props":13763,"children":13765},{"className":13764},[296,301,342],[13766],{"type":155,"value":797},{"type":150,"tag":181,"props":13768,"children":13770},{"className":13769},[835,342],[13771],{"type":155,"value":802},{"type":150,"tag":181,"props":13773,"children":13775},{"className":13774},[352],[13776],{"type":155,"value":355},{"type":150,"tag":181,"props":13778,"children":13780},{"className":13779},[318],[13781],{"type":150,"tag":181,"props":13782,"children":13784},{"className":13783,"style":2782},[323],[13785],{"type":150,"tag":181,"props":13786,"children":13787},{},[],{"type":150,"tag":181,"props":13789,"children":13791},{"className":13790,"style":658},[377],[],{"type":150,"tag":181,"props":13793,"children":13795},{"className":13794},[663],[13796],{"type":155,"value":865},{"type":150,"tag":181,"props":13798,"children":13800},{"className":13799,"style":658},[377],[],{"type":150,"tag":181,"props":13802,"children":13804},{"className":13803},[285],[13805,13809,13866,13953,13957,13962],{"type":150,"tag":181,"props":13806,"children":13808},{"className":13807,"style":4408},[290],[],{"type":150,"tag":181,"props":13810,"children":13812},{"className":13811},[296],[13813,13818],{"type":150,"tag":181,"props":13814,"children":13816},{"className":13815},[296,301],[13817],{"type":155,"value":9859},{"type":150,"tag":181,"props":13819,"children":13821},{"className":13820},[307],[13822],{"type":150,"tag":181,"props":13823,"children":13825},{"className":13824},[312,313],[13826,13855],{"type":150,"tag":181,"props":13827,"children":13829},{"className":13828},[318],[13830,13850],{"type":150,"tag":181,"props":13831,"children":13833},{"className":13832,"style":621},[323],[13834],{"type":150,"tag":181,"props":13835,"children":13836},{"style":328},[13837,13841],{"type":150,"tag":181,"props":13838,"children":13840},{"className":13839,"style":334},[333],[],{"type":150,"tag":181,"props":13842,"children":13844},{"className":13843},[339,340,341,342],[13845],{"type":150,"tag":181,"props":13846,"children":13848},{"className":13847},[296,301,342],[13849],{"type":155,"value":558},{"type":150,"tag":181,"props":13851,"children":13853},{"className":13852},[352],[13854],{"type":155,"value":355},{"type":150,"tag":181,"props":13856,"children":13858},{"className":13857},[318],[13859],{"type":150,"tag":181,"props":13860,"children":13862},{"className":13861,"style":363},[323],[13863],{"type":150,"tag":181,"props":13864,"children":13865},{},[],{"type":150,"tag":181,"props":13867,"children":13869},{"className":13868},[296],[13870,13875],{"type":150,"tag":181,"props":13871,"children":13873},{"className":13872,"style":3691},[296,301],[13874],{"type":155,"value":3265},{"type":150,"tag":181,"props":13876,"children":13878},{"className":13877},[307],[13879],{"type":150,"tag":181,"props":13880,"children":13882},{"className":13881},[312,313],[13883,13942],{"type":150,"tag":181,"props":13884,"children":13886},{"className":13885},[318],[13887,13937],{"type":150,"tag":181,"props":13888,"children":13890},{"className":13889,"style":3709},[323],[13891,13907],{"type":150,"tag":181,"props":13892,"children":13893},{"style":3713},[13894,13898],{"type":150,"tag":181,"props":13895,"children":13897},{"className":13896,"style":334},[333],[],{"type":150,"tag":181,"props":13899,"children":13901},{"className":13900},[339,340,341,342],[13902],{"type":150,"tag":181,"props":13903,"children":13905},{"className":13904,"style":2151},[296,301,342],[13906],{"type":155,"value":1737},{"type":150,"tag":181,"props":13908,"children":13909},{"style":3730},[13910,13914],{"type":150,"tag":181,"props":13911,"children":13913},{"className":13912,"style":334},[333],[],{"type":150,"tag":181,"props":13915,"children":13917},{"className":13916},[339,340,341,342],[13918],{"type":150,"tag":181,"props":13919,"children":13921},{"className":13920},[296,342],[13922,13927,13932],{"type":150,"tag":181,"props":13923,"children":13925},{"className":13924},[824,342],[13926],{"type":155,"value":792},{"type":150,"tag":181,"props":13928,"children":13930},{"className":13929},[296,301,342],[13931],{"type":155,"value":797},{"type":150,"tag":181,"props":13933,"children":13935},{"className":13934},[835,342],[13936],{"type":155,"value":802},{"type":150,"tag":181,"props":13938,"children":13940},{"className":13939},[352],[13941],{"type":155,"value":355},{"type":150,"tag":181,"props":13943,"children":13945},{"className":13944},[318],[13946],{"type":150,"tag":181,"props":13947,"children":13949},{"className":13948,"style":4550},[323],[13950],{"type":150,"tag":181,"props":13951,"children":13952},{},[],{"type":150,"tag":181,"props":13954,"children":13956},{"className":13955,"style":658},[377],[],{"type":150,"tag":181,"props":13958,"children":13960},{"className":13959},[663],[13961],{"type":155,"value":563},{"type":150,"tag":181,"props":13963,"children":13965},{"className":13964,"style":658},[377],[],{"type":150,"tag":181,"props":13967,"children":13969},{"className":13968},[285],[13970,13974],{"type":150,"tag":181,"props":13971,"children":13973},{"className":13972,"style":3821},[290],[],{"type":150,"tag":181,"props":13975,"children":13977},{"className":13976},[296],[13978,13983],{"type":150,"tag":181,"props":13979,"children":13981},{"className":13980},[296,686],[13982],{"type":155,"value":573},{"type":150,"tag":181,"props":13984,"children":13986},{"className":13985},[307],[13987],{"type":150,"tag":181,"props":13988,"children":13990},{"className":13989},[312],[13991],{"type":150,"tag":181,"props":13992,"children":13994},{"className":13993},[318],[13995],{"type":150,"tag":181,"props":13996,"children":13998},{"className":13997,"style":3821},[323],[13999],{"type":150,"tag":181,"props":14000,"children":14001},{"style":3730},[14002,14006],{"type":150,"tag":181,"props":14003,"children":14005},{"className":14004,"style":334},[333],[],{"type":150,"tag":181,"props":14007,"children":14009},{"className":14008},[339,340,341,342],[14010],{"type":150,"tag":181,"props":14011,"children":14013},{"className":14012},[296,342],[14014],{"type":150,"tag":181,"props":14015,"children":14017},{"className":14016},[296,342],[14018,14023],{"type":150,"tag":181,"props":14019,"children":14021},{"className":14020},[296,301,342],[14022],{"type":155,"value":578},{"type":150,"tag":181,"props":14024,"children":14026},{"className":14025},[307],[14027],{"type":150,"tag":181,"props":14028,"children":14030},{"className":14029},[312,313],[14031,14060],{"type":150,"tag":181,"props":14032,"children":14034},{"className":14033},[318],[14035,14055],{"type":150,"tag":181,"props":14036,"children":14038},{"className":14037,"style":4640},[323],[14039],{"type":150,"tag":181,"props":14040,"children":14041},{"style":4644},[14042,14046],{"type":150,"tag":181,"props":14043,"children":14045},{"className":14044,"style":3871},[333],[],{"type":150,"tag":181,"props":14047,"children":14049},{"className":14048},[339,3876,3877,342],[14050],{"type":150,"tag":181,"props":14051,"children":14053},{"className":14052,"style":2151},[296,301,342],[14054],{"type":155,"value":1737},{"type":150,"tag":181,"props":14056,"children":14058},{"className":14057},[352],[14059],{"type":155,"value":355},{"type":150,"tag":181,"props":14061,"children":14063},{"className":14062},[318],[14064],{"type":150,"tag":181,"props":14065,"children":14067},{"className":14066,"style":4671},[323],[14068],{"type":150,"tag":181,"props":14069,"children":14070},{},[],{"type":150,"tag":151,"props":14072,"children":14073},{},[14074,14076,14261],{"type":155,"value":14075},"但这样问题就来了。我们缓存的是投影后的 ",{"type":150,"tag":181,"props":14077,"children":14079},{"className":14078},[184],[14080,14124],{"type":150,"tag":181,"props":14081,"children":14083},{"className":14082},[189],[14084],{"type":150,"tag":192,"props":14085,"children":14086},{"xmlns":194},[14087],{"type":150,"tag":197,"props":14088,"children":14089},{},[14090,14119],{"type":150,"tag":201,"props":14091,"children":14092},{},[14093,14104,14108],{"type":150,"tag":205,"props":14094,"children":14095},{},[14096,14100],{"type":150,"tag":209,"props":14097,"children":14098},{},[14099],{"type":155,"value":1698},{"type":150,"tag":209,"props":14101,"children":14102},{},[14103],{"type":155,"value":558},{"type":150,"tag":221,"props":14105,"children":14106},{"separator":223},[14107],{"type":155,"value":226},{"type":150,"tag":205,"props":14109,"children":14110},{},[14111,14115],{"type":150,"tag":209,"props":14112,"children":14113},{},[14114],{"type":155,"value":1737},{"type":150,"tag":209,"props":14116,"children":14117},{},[14118],{"type":155,"value":558},{"type":150,"tag":271,"props":14120,"children":14121},{"encoding":273},[14122],{"type":155,"value":14123},"k_i, v_i",{"type":150,"tag":181,"props":14125,"children":14127},{"className":14126,"ariaHidden":223},[280],[14128],{"type":150,"tag":181,"props":14129,"children":14131},{"className":14130},[285],[14132,14136,14194,14199,14203],{"type":150,"tag":181,"props":14133,"children":14135},{"className":14134,"style":941},[290],[],{"type":150,"tag":181,"props":14137,"children":14139},{"className":14138},[296],[14140,14145],{"type":150,"tag":181,"props":14141,"children":14143},{"className":14142,"style":2249},[296,301],[14144],{"type":155,"value":1698},{"type":150,"tag":181,"props":14146,"children":14148},{"className":14147},[307],[14149],{"type":150,"tag":181,"props":14150,"children":14152},{"className":14151},[312,313],[14153,14183],{"type":150,"tag":181,"props":14154,"children":14156},{"className":14155},[318],[14157,14178],{"type":150,"tag":181,"props":14158,"children":14160},{"className":14159,"style":621},[323],[14161],{"type":150,"tag":181,"props":14162,"children":14164},{"style":14163},"top:-2.55em;margin-left:-0.0315em;margin-right:0.05em;",[14165,14169],{"type":150,"tag":181,"props":14166,"children":14168},{"className":14167,"style":334},[333],[],{"type":150,"tag":181,"props":14170,"children":14172},{"className":14171},[339,340,341,342],[14173],{"type":150,"tag":181,"props":14174,"children":14176},{"className":14175},[296,301,342],[14177],{"type":155,"value":558},{"type":150,"tag":181,"props":14179,"children":14181},{"className":14180},[352],[14182],{"type":155,"value":355},{"type":150,"tag":181,"props":14184,"children":14186},{"className":14185},[318],[14187],{"type":150,"tag":181,"props":14188,"children":14190},{"className":14189,"style":363},[323],[14191],{"type":150,"tag":181,"props":14192,"children":14193},{},[],{"type":150,"tag":181,"props":14195,"children":14197},{"className":14196},[371],[14198],{"type":155,"value":226},{"type":150,"tag":181,"props":14200,"children":14202},{"className":14201,"style":378},[377],[],{"type":150,"tag":181,"props":14204,"children":14206},{"className":14205},[296],[14207,14212],{"type":150,"tag":181,"props":14208,"children":14210},{"className":14209,"style":2151},[296,301],[14211],{"type":155,"value":1737},{"type":150,"tag":181,"props":14213,"children":14215},{"className":14214},[307],[14216],{"type":150,"tag":181,"props":14217,"children":14219},{"className":14218},[312,313],[14220,14250],{"type":150,"tag":181,"props":14221,"children":14223},{"className":14222},[318],[14224,14245],{"type":150,"tag":181,"props":14225,"children":14227},{"className":14226,"style":621},[323],[14228],{"type":150,"tag":181,"props":14229,"children":14231},{"style":14230},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[14232,14236],{"type":150,"tag":181,"props":14233,"children":14235},{"className":14234,"style":334},[333],[],{"type":150,"tag":181,"props":14237,"children":14239},{"className":14238},[339,340,341,342],[14240],{"type":150,"tag":181,"props":14241,"children":14243},{"className":14242},[296,301,342],[14244],{"type":155,"value":558},{"type":150,"tag":181,"props":14246,"children":14248},{"className":14247},[352],[14249],{"type":155,"value":355},{"type":150,"tag":181,"props":14251,"children":14253},{"className":14252},[318],[14254],{"type":150,"tag":181,"props":14255,"children":14257},{"className":14256,"style":363},[323],[14258],{"type":150,"tag":181,"props":14259,"children":14260},{},[],{"type":155,"value":14262},"，一旦它们各不相同，KV Cache 就恢复成 MHA 那么大，GQA 的初衷全丢了。MLA 的做法是找一个恒等变换绕过去。在推理阶段：",{"type":150,"tag":181,"props":14264,"children":14266},{"className":14265},[1003],[14267],{"type":150,"tag":181,"props":14268,"children":14270},{"className":14269},[184],[14271,14540],{"type":150,"tag":181,"props":14272,"children":14274},{"className":14273},[189],[14275],{"type":150,"tag":192,"props":14276,"children":14277},{"xmlns":194,"display":1015},[14278],{"type":150,"tag":197,"props":14279,"children":14280},{},[14281,14535],{"type":150,"tag":201,"props":14282,"children":14283},{},[14284,14310,14340,14344,14348,14359,14385,14389,14393,14404,14430,14441,14445,14456,14460,14486,14516,14520],{"type":150,"tag":1046,"props":14285,"children":14286},{},[14287,14291,14295],{"type":150,"tag":209,"props":14288,"children":14289},{},[14290],{"type":155,"value":1667},{"type":150,"tag":209,"props":14292,"children":14293},{},[14294],{"type":155,"value":1035},{"type":150,"tag":201,"props":14296,"children":14297},{},[14298,14302,14306],{"type":150,"tag":221,"props":14299,"children":14300},{"stretchy":789},[14301],{"type":155,"value":792},{"type":150,"tag":209,"props":14303,"children":14304},{},[14305],{"type":155,"value":797},{"type":150,"tag":221,"props":14307,"children":14308},{"stretchy":789},[14309],{"type":155,"value":802},{"type":150,"tag":1046,"props":14311,"children":14312},{},[14313,14317,14321],{"type":150,"tag":209,"props":14314,"children":14315},{},[14316],{"type":155,"value":1698},{"type":150,"tag":209,"props":14318,"children":14319},{},[14320],{"type":155,"value":558},{"type":150,"tag":201,"props":14322,"children":14323},{},[14324,14328,14332,14336],{"type":150,"tag":221,"props":14325,"children":14326},{"stretchy":789},[14327],{"type":155,"value":792},{"type":150,"tag":209,"props":14329,"children":14330},{},[14331],{"type":155,"value":797},{"type":150,"tag":221,"props":14333,"children":14334},{"stretchy":789},[14335],{"type":155,"value":802},{"type":150,"tag":209,"props":14337,"children":14338},{"mathvariant":1618},[14339],{"type":155,"value":1874},{"type":150,"tag":221,"props":14341,"children":14342},{},[14343],{"type":155,"value":865},{"type":150,"tag":221,"props":14345,"children":14346},{"stretchy":789},[14347],{"type":155,"value":792},{"type":150,"tag":205,"props":14349,"children":14350},{},[14351,14355],{"type":150,"tag":209,"props":14352,"children":14353},{},[14354],{"type":155,"value":213},{"type":150,"tag":209,"props":14356,"children":14357},{},[14358],{"type":155,"value":1035},{"type":150,"tag":1046,"props":14360,"children":14361},{},[14362,14366,14370],{"type":150,"tag":209,"props":14363,"children":14364},{},[14365],{"type":155,"value":3265},{"type":150,"tag":209,"props":14367,"children":14368},{},[14369],{"type":155,"value":1667},{"type":150,"tag":201,"props":14371,"children":14372},{},[14373,14377,14381],{"type":150,"tag":221,"props":14374,"children":14375},{"stretchy":789},[14376],{"type":155,"value":792},{"type":150,"tag":209,"props":14378,"children":14379},{},[14380],{"type":155,"value":797},{"type":150,"tag":221,"props":14382,"children":14383},{"stretchy":789},[14384],{"type":155,"value":802},{"type":150,"tag":221,"props":14386,"children":14387},{"stretchy":789},[14388],{"type":155,"value":802},{"type":150,"tag":221,"props":14390,"children":14391},{"stretchy":789},[14392],{"type":155,"value":792},{"type":150,"tag":205,"props":14394,"children":14395},{},[14396,14400],{"type":150,"tag":209,"props":14397,"children":14398},{},[14399],{"type":155,"value":9859},{"type":150,"tag":209,"props":14401,"children":14402},{},[14403],{"type":155,"value":558},{"type":150,"tag":1046,"props":14405,"children":14406},{},[14407,14411,14415],{"type":150,"tag":209,"props":14408,"children":14409},{},[14410],{"type":155,"value":3265},{"type":150,"tag":209,"props":14412,"children":14413},{},[14414],{"type":155,"value":1698},{"type":150,"tag":201,"props":14416,"children":14417},{},[14418,14422,14426],{"type":150,"tag":221,"props":14419,"children":14420},{"stretchy":789},[14421],{"type":155,"value":792},{"type":150,"tag":209,"props":14423,"children":14424},{},[14425],{"type":155,"value":797},{"type":150,"tag":221,"props":14427,"children":14428},{"stretchy":789},[14429],{"type":155,"value":802},{"type":150,"tag":565,"props":14431,"children":14432},{},[14433,14437],{"type":150,"tag":221,"props":14434,"children":14435},{"stretchy":789},[14436],{"type":155,"value":802},{"type":150,"tag":209,"props":14438,"children":14439},{"mathvariant":1618},[14440],{"type":155,"value":1874},{"type":150,"tag":221,"props":14442,"children":14443},{},[14444],{"type":155,"value":865},{"type":150,"tag":205,"props":14446,"children":14447},{},[14448,14452],{"type":150,"tag":209,"props":14449,"children":14450},{},[14451],{"type":155,"value":213},{"type":150,"tag":209,"props":14453,"children":14454},{},[14455],{"type":155,"value":1035},{"type":150,"tag":221,"props":14457,"children":14458},{"stretchy":789},[14459],{"type":155,"value":792},{"type":150,"tag":1046,"props":14461,"children":14462},{},[14463,14467,14471],{"type":150,"tag":209,"props":14464,"children":14465},{},[14466],{"type":155,"value":3265},{"type":150,"tag":209,"props":14468,"children":14469},{},[14470],{"type":155,"value":1667},{"type":150,"tag":201,"props":14472,"children":14473},{},[14474,14478,14482],{"type":150,"tag":221,"props":14475,"children":14476},{"stretchy":789},[14477],{"type":155,"value":792},{"type":150,"tag":209,"props":14479,"children":14480},{},[14481],{"type":155,"value":797},{"type":150,"tag":221,"props":14483,"children":14484},{"stretchy":789},[14485],{"type":155,"value":802},{"type":150,"tag":1046,"props":14487,"children":14488},{},[14489,14493,14497],{"type":150,"tag":209,"props":14490,"children":14491},{},[14492],{"type":155,"value":3265},{"type":150,"tag":209,"props":14494,"children":14495},{},[14496],{"type":155,"value":1698},{"type":150,"tag":201,"props":14498,"children":14499},{},[14500,14504,14508,14512],{"type":150,"tag":221,"props":14501,"children":14502},{"stretchy":789},[14503],{"type":155,"value":792},{"type":150,"tag":209,"props":14505,"children":14506},{},[14507],{"type":155,"value":797},{"type":150,"tag":221,"props":14509,"children":14510},{"stretchy":789},[14511],{"type":155,"value":802},{"type":150,"tag":209,"props":14513,"children":14514},{"mathvariant":1618},[14515],{"type":155,"value":1874},{"type":150,"tag":221,"props":14517,"children":14518},{"stretchy":789},[14519],{"type":155,"value":802},{"type":150,"tag":1046,"props":14521,"children":14522},{},[14523,14527,14531],{"type":150,"tag":209,"props":14524,"children":14525},{},[14526],{"type":155,"value":9859},{"type":150,"tag":209,"props":14528,"children":14529},{},[14530],{"type":155,"value":558},{"type":150,"tag":209,"props":14532,"children":14533},{"mathvariant":1618},[14534],{"type":155,"value":1874},{"type":150,"tag":271,"props":14536,"children":14537},{"encoding":273},[14538],{"type":155,"value":14539},"q_t^{(s)} k_i^{(s)\\top} = (x_t W_q^{(s)})(c_i W_k^{(s)})^\\top = x_t (W_q^{(s)} W_k^{(s)\\top}) c_i^\\top",{"type":150,"tag":181,"props":14541,"children":14543},{"className":14542,"ariaHidden":223},[280],[14544,14744,15110],{"type":150,"tag":181,"props":14545,"children":14547},{"className":14546},[285],[14548,14552,14639,14731,14735,14740],{"type":150,"tag":181,"props":14549,"children":14551},{"className":14550,"style":3516},[290],[],{"type":150,"tag":181,"props":14553,"children":14555},{"className":14554},[296],[14556,14561],{"type":150,"tag":181,"props":14557,"children":14559},{"className":14558,"style":2151},[296,301],[14560],{"type":155,"value":1667},{"type":150,"tag":181,"props":14562,"children":14564},{"className":14563},[307],[14565],{"type":150,"tag":181,"props":14566,"children":14568},{"className":14567},[312,313],[14569,14628],{"type":150,"tag":181,"props":14570,"children":14572},{"className":14571},[318],[14573,14623],{"type":150,"tag":181,"props":14574,"children":14576},{"className":14575,"style":1276},[323],[14577,14593],{"type":150,"tag":181,"props":14578,"children":14579},{"style":2172},[14580,14584],{"type":150,"tag":181,"props":14581,"children":14583},{"className":14582,"style":334},[333],[],{"type":150,"tag":181,"props":14585,"children":14587},{"className":14586},[339,340,341,342],[14588],{"type":150,"tag":181,"props":14589,"children":14591},{"className":14590},[296,301,342],[14592],{"type":155,"value":1035},{"type":150,"tag":181,"props":14594,"children":14595},{"style":1297},[14596,14600],{"type":150,"tag":181,"props":14597,"children":14599},{"className":14598,"style":334},[333],[],{"type":150,"tag":181,"props":14601,"children":14603},{"className":14602},[339,340,341,342],[14604],{"type":150,"tag":181,"props":14605,"children":14607},{"className":14606},[296,342],[14608,14613,14618],{"type":150,"tag":181,"props":14609,"children":14611},{"className":14610},[824,342],[14612],{"type":155,"value":792},{"type":150,"tag":181,"props":14614,"children":14616},{"className":14615},[296,301,342],[14617],{"type":155,"value":797},{"type":150,"tag":181,"props":14619,"children":14621},{"className":14620},[835,342],[14622],{"type":155,"value":802},{"type":150,"tag":181,"props":14624,"children":14626},{"className":14625},[352],[14627],{"type":155,"value":355},{"type":150,"tag":181,"props":14629,"children":14631},{"className":14630},[318],[14632],{"type":150,"tag":181,"props":14633,"children":14635},{"className":14634,"style":1338},[323],[14636],{"type":150,"tag":181,"props":14637,"children":14638},{},[],{"type":150,"tag":181,"props":14640,"children":14642},{"className":14641},[296],[14643,14648],{"type":150,"tag":181,"props":14644,"children":14646},{"className":14645,"style":2249},[296,301],[14647],{"type":155,"value":1698},{"type":150,"tag":181,"props":14649,"children":14651},{"className":14650},[307],[14652],{"type":150,"tag":181,"props":14653,"children":14655},{"className":14654},[312,313],[14656,14720],{"type":150,"tag":181,"props":14657,"children":14659},{"className":14658},[318],[14660,14715],{"type":150,"tag":181,"props":14661,"children":14663},{"className":14662,"style":1276},[323],[14664,14680],{"type":150,"tag":181,"props":14665,"children":14666},{"style":2720},[14667,14671],{"type":150,"tag":181,"props":14668,"children":14670},{"className":14669,"style":334},[333],[],{"type":150,"tag":181,"props":14672,"children":14674},{"className":14673},[339,340,341,342],[14675],{"type":150,"tag":181,"props":14676,"children":14678},{"className":14677},[296,301,342],[14679],{"type":155,"value":558},{"type":150,"tag":181,"props":14681,"children":14682},{"style":1297},[14683,14687],{"type":150,"tag":181,"props":14684,"children":14686},{"className":14685,"style":334},[333],[],{"type":150,"tag":181,"props":14688,"children":14690},{"className":14689},[339,340,341,342],[14691],{"type":150,"tag":181,"props":14692,"children":14694},{"className":14693},[296,342],[14695,14700,14705,14710],{"type":150,"tag":181,"props":14696,"children":14698},{"className":14697},[824,342],[14699],{"type":155,"value":792},{"type":150,"tag":181,"props":14701,"children":14703},{"className":14702},[296,301,342],[14704],{"type":155,"value":797},{"type":150,"tag":181,"props":14706,"children":14708},{"className":14707},[835,342],[14709],{"type":155,"value":802},{"type":150,"tag":181,"props":14711,"children":14713},{"className":14712},[296,342],[14714],{"type":155,"value":1874},{"type":150,"tag":181,"props":14716,"children":14718},{"className":14717},[352],[14719],{"type":155,"value":355},{"type":150,"tag":181,"props":14721,"children":14723},{"className":14722},[318],[14724],{"type":150,"tag":181,"props":14725,"children":14727},{"className":14726,"style":2782},[323],[14728],{"type":150,"tag":181,"props":14729,"children":14730},{},[],{"type":150,"tag":181,"props":14732,"children":14734},{"className":14733,"style":658},[377],[],{"type":150,"tag":181,"props":14736,"children":14738},{"className":14737},[663],[14739],{"type":155,"value":865},{"type":150,"tag":181,"props":14741,"children":14743},{"className":14742,"style":658},[377],[],{"type":150,"tag":181,"props":14745,"children":14747},{"className":14746},[285],[14748,14753,14758,14815,14902,14907,14912,14969,15056,15097,15101,15106],{"type":150,"tag":181,"props":14749,"children":14752},{"className":14750,"style":14751},[290],"height:1.4279em;vertical-align:-0.3831em;",[],{"type":150,"tag":181,"props":14754,"children":14756},{"className":14755},[824],[14757],{"type":155,"value":792},{"type":150,"tag":181,"props":14759,"children":14761},{"className":14760},[296],[14762,14767],{"type":150,"tag":181,"props":14763,"children":14765},{"className":14764},[296,301],[14766],{"type":155,"value":213},{"type":150,"tag":181,"props":14768,"children":14770},{"className":14769},[307],[14771],{"type":150,"tag":181,"props":14772,"children":14774},{"className":14773},[312,313],[14775,14804],{"type":150,"tag":181,"props":14776,"children":14778},{"className":14777},[318],[14779,14799],{"type":150,"tag":181,"props":14780,"children":14782},{"className":14781,"style":1191},[323],[14783],{"type":150,"tag":181,"props":14784,"children":14785},{"style":328},[14786,14790],{"type":150,"tag":181,"props":14787,"children":14789},{"className":14788,"style":334},[333],[],{"type":150,"tag":181,"props":14791,"children":14793},{"className":14792},[339,340,341,342],[14794],{"type":150,"tag":181,"props":14795,"children":14797},{"className":14796},[296,301,342],[14798],{"type":155,"value":1035},{"type":150,"tag":181,"props":14800,"children":14802},{"className":14801},[352],[14803],{"type":155,"value":355},{"type":150,"tag":181,"props":14805,"children":14807},{"className":14806},[318],[14808],{"type":150,"tag":181,"props":14809,"children":14811},{"className":14810,"style":363},[323],[14812],{"type":150,"tag":181,"props":14813,"children":14814},{},[],{"type":150,"tag":181,"props":14816,"children":14818},{"className":14817},[296],[14819,14824],{"type":150,"tag":181,"props":14820,"children":14822},{"className":14821,"style":3691},[296,301],[14823],{"type":155,"value":3265},{"type":150,"tag":181,"props":14825,"children":14827},{"className":14826},[307],[14828],{"type":150,"tag":181,"props":14829,"children":14831},{"className":14830},[312,313],[14832,14891],{"type":150,"tag":181,"props":14833,"children":14835},{"className":14834},[318],[14836,14886],{"type":150,"tag":181,"props":14837,"children":14839},{"className":14838,"style":3709},[323],[14840,14856],{"type":150,"tag":181,"props":14841,"children":14842},{"style":3713},[14843,14847],{"type":150,"tag":181,"props":14844,"children":14846},{"className":14845,"style":334},[333],[],{"type":150,"tag":181,"props":14848,"children":14850},{"className":14849},[339,340,341,342],[14851],{"type":150,"tag":181,"props":14852,"children":14854},{"className":14853,"style":2151},[296,301,342],[14855],{"type":155,"value":1667},{"type":150,"tag":181,"props":14857,"children":14858},{"style":3730},[14859,14863],{"type":150,"tag":181,"props":14860,"children":14862},{"className":14861,"style":334},[333],[],{"type":150,"tag":181,"props":14864,"children":14866},{"className":14865},[339,340,341,342],[14867],{"type":150,"tag":181,"props":14868,"children":14870},{"className":14869},[296,342],[14871,14876,14881],{"type":150,"tag":181,"props":14872,"children":14874},{"className":14873},[824,342],[14875],{"type":155,"value":792},{"type":150,"tag":181,"props":14877,"children":14879},{"className":14878},[296,301,342],[14880],{"type":155,"value":797},{"type":150,"tag":181,"props":14882,"children":14884},{"className":14883},[835,342],[14885],{"type":155,"value":802},{"type":150,"tag":181,"props":14887,"children":14889},{"className":14888},[352],[14890],{"type":155,"value":355},{"type":150,"tag":181,"props":14892,"children":14894},{"className":14893},[318],[14895],{"type":150,"tag":181,"props":14896,"children":14898},{"className":14897,"style":3771},[323],[14899],{"type":150,"tag":181,"props":14900,"children":14901},{},[],{"type":150,"tag":181,"props":14903,"children":14905},{"className":14904},[835],[14906],{"type":155,"value":802},{"type":150,"tag":181,"props":14908,"children":14910},{"className":14909},[824],[14911],{"type":155,"value":792},{"type":150,"tag":181,"props":14913,"children":14915},{"className":14914},[296],[14916,14921],{"type":150,"tag":181,"props":14917,"children":14919},{"className":14918},[296,301],[14920],{"type":155,"value":9859},{"type":150,"tag":181,"props":14922,"children":14924},{"className":14923},[307],[14925],{"type":150,"tag":181,"props":14926,"children":14928},{"className":14927},[312,313],[14929,14958],{"type":150,"tag":181,"props":14930,"children":14932},{"className":14931},[318],[14933,14953],{"type":150,"tag":181,"props":14934,"children":14936},{"className":14935,"style":621},[323],[14937],{"type":150,"tag":181,"props":14938,"children":14939},{"style":328},[14940,14944],{"type":150,"tag":181,"props":14941,"children":14943},{"className":14942,"style":334},[333],[],{"type":150,"tag":181,"props":14945,"children":14947},{"className":14946},[339,340,341,342],[14948],{"type":150,"tag":181,"props":14949,"children":14951},{"className":14950},[296,301,342],[14952],{"type":155,"value":558},{"type":150,"tag":181,"props":14954,"children":14956},{"className":14955},[352],[14957],{"type":155,"value":355},{"type":150,"tag":181,"props":14959,"children":14961},{"className":14960},[318],[14962],{"type":150,"tag":181,"props":14963,"children":14965},{"className":14964,"style":363},[323],[14966],{"type":150,"tag":181,"props":14967,"children":14968},{},[],{"type":150,"tag":181,"props":14970,"children":14972},{"className":14971},[296],[14973,14978],{"type":150,"tag":181,"props":14974,"children":14976},{"className":14975,"style":3691},[296,301],[14977],{"type":155,"value":3265},{"type":150,"tag":181,"props":14979,"children":14981},{"className":14980},[307],[14982],{"type":150,"tag":181,"props":14983,"children":14985},{"className":14984},[312,313],[14986,15045],{"type":150,"tag":181,"props":14987,"children":14989},{"className":14988},[318],[14990,15040],{"type":150,"tag":181,"props":14991,"children":14993},{"className":14992,"style":1276},[323],[14994,15010],{"type":150,"tag":181,"props":14995,"children":14996},{"style":4108},[14997,15001],{"type":150,"tag":181,"props":14998,"children":15000},{"className":14999,"style":334},[333],[],{"type":150,"tag":181,"props":15002,"children":15004},{"className":15003},[339,340,341,342],[15005],{"type":150,"tag":181,"props":15006,"children":15008},{"className":15007,"style":2249},[296,301,342],[15009],{"type":155,"value":1698},{"type":150,"tag":181,"props":15011,"children":15012},{"style":1297},[15013,15017],{"type":150,"tag":181,"props":15014,"children":15016},{"className":15015,"style":334},[333],[],{"type":150,"tag":181,"props":15018,"children":15020},{"className":15019},[339,340,341,342],[15021],{"type":150,"tag":181,"props":15022,"children":15024},{"className":15023},[296,342],[15025,15030,15035],{"type":150,"tag":181,"props":15026,"children":15028},{"className":15027},[824,342],[15029],{"type":155,"value":792},{"type":150,"tag":181,"props":15031,"children":15033},{"className":15032},[296,301,342],[15034],{"type":155,"value":797},{"type":150,"tag":181,"props":15036,"children":15038},{"className":15037},[835,342],[15039],{"type":155,"value":802},{"type":150,"tag":181,"props":15041,"children":15043},{"className":15042},[352],[15044],{"type":155,"value":355},{"type":150,"tag":181,"props":15046,"children":15048},{"className":15047},[318],[15049],{"type":150,"tag":181,"props":15050,"children":15052},{"className":15051,"style":4165},[323],[15053],{"type":150,"tag":181,"props":15054,"children":15055},{},[],{"type":150,"tag":181,"props":15057,"children":15059},{"className":15058},[835],[15060,15065],{"type":150,"tag":181,"props":15061,"children":15063},{"className":15062},[835],[15064],{"type":155,"value":802},{"type":150,"tag":181,"props":15066,"children":15068},{"className":15067},[307],[15069],{"type":150,"tag":181,"props":15070,"children":15072},{"className":15071},[312],[15073],{"type":150,"tag":181,"props":15074,"children":15076},{"className":15075},[318],[15077],{"type":150,"tag":181,"props":15078,"children":15080},{"className":15079,"style":3821},[323],[15081],{"type":150,"tag":181,"props":15082,"children":15083},{"style":3730},[15084,15088],{"type":150,"tag":181,"props":15085,"children":15087},{"className":15086,"style":334},[333],[],{"type":150,"tag":181,"props":15089,"children":15091},{"className":15090},[339,340,341,342],[15092],{"type":150,"tag":181,"props":15093,"children":15095},{"className":15094},[296,342],[15096],{"type":155,"value":1874},{"type":150,"tag":181,"props":15098,"children":15100},{"className":15099,"style":658},[377],[],{"type":150,"tag":181,"props":15102,"children":15104},{"className":15103},[663],[15105],{"type":155,"value":865},{"type":150,"tag":181,"props":15107,"children":15109},{"className":15108,"style":658},[377],[],{"type":150,"tag":181,"props":15111,"children":15113},{"className":15112},[285],[15114,15118,15175,15180,15267,15359,15364],{"type":150,"tag":181,"props":15115,"children":15117},{"className":15116,"style":14751},[290],[],{"type":150,"tag":181,"props":15119,"children":15121},{"className":15120},[296],[15122,15127],{"type":150,"tag":181,"props":15123,"children":15125},{"className":15124},[296,301],[15126],{"type":155,"value":213},{"type":150,"tag":181,"props":15128,"children":15130},{"className":15129},[307],[15131],{"type":150,"tag":181,"props":15132,"children":15134},{"className":15133},[312,313],[15135,15164],{"type":150,"tag":181,"props":15136,"children":15138},{"className":15137},[318],[15139,15159],{"type":150,"tag":181,"props":15140,"children":15142},{"className":15141,"style":1191},[323],[15143],{"type":150,"tag":181,"props":15144,"children":15145},{"style":328},[15146,15150],{"type":150,"tag":181,"props":15147,"children":15149},{"className":15148,"style":334},[333],[],{"type":150,"tag":181,"props":15151,"children":15153},{"className":15152},[339,340,341,342],[15154],{"type":150,"tag":181,"props":15155,"children":15157},{"className":15156},[296,301,342],[15158],{"type":155,"value":1035},{"type":150,"tag":181,"props":15160,"children":15162},{"className":15161},[352],[15163],{"type":155,"value":355},{"type":150,"tag":181,"props":15165,"children":15167},{"className":15166},[318],[15168],{"type":150,"tag":181,"props":15169,"children":15171},{"className":15170,"style":363},[323],[15172],{"type":150,"tag":181,"props":15173,"children":15174},{},[],{"type":150,"tag":181,"props":15176,"children":15178},{"className":15177},[824],[15179],{"type":155,"value":792},{"type":150,"tag":181,"props":15181,"children":15183},{"className":15182},[296],[15184,15189],{"type":150,"tag":181,"props":15185,"children":15187},{"className":15186,"style":3691},[296,301],[15188],{"type":155,"value":3265},{"type":150,"tag":181,"props":15190,"children":15192},{"className":15191},[307],[15193],{"type":150,"tag":181,"props":15194,"children":15196},{"className":15195},[312,313],[15197,15256],{"type":150,"tag":181,"props":15198,"children":15200},{"className":15199},[318],[15201,15251],{"type":150,"tag":181,"props":15202,"children":15204},{"className":15203,"style":3709},[323],[15205,15221],{"type":150,"tag":181,"props":15206,"children":15207},{"style":3713},[15208,15212],{"type":150,"tag":181,"props":15209,"children":15211},{"className":15210,"style":334},[333],[],{"type":150,"tag":181,"props":15213,"children":15215},{"className":15214},[339,340,341,342],[15216],{"type":150,"tag":181,"props":15217,"children":15219},{"className":15218,"style":2151},[296,301,342],[15220],{"type":155,"value":1667},{"type":150,"tag":181,"props":15222,"children":15223},{"style":3730},[15224,15228],{"type":150,"tag":181,"props":15225,"children":15227},{"className":15226,"style":334},[333],[],{"type":150,"tag":181,"props":15229,"children":15231},{"className":15230},[339,340,341,342],[15232],{"type":150,"tag":181,"props":15233,"children":15235},{"className":15234},[296,342],[15236,15241,15246],{"type":150,"tag":181,"props":15237,"children":15239},{"className":15238},[824,342],[15240],{"type":155,"value":792},{"type":150,"tag":181,"props":15242,"children":15244},{"className":15243},[296,301,342],[15245],{"type":155,"value":797},{"type":150,"tag":181,"props":15247,"children":15249},{"className":15248},[835,342],[15250],{"type":155,"value":802},{"type":150,"tag":181,"props":15252,"children":15254},{"className":15253},[352],[15255],{"type":155,"value":355},{"type":150,"tag":181,"props":15257,"children":15259},{"className":15258},[318],[15260],{"type":150,"tag":181,"props":15261,"children":15263},{"className":15262,"style":3771},[323],[15264],{"type":150,"tag":181,"props":15265,"children":15266},{},[],{"type":150,"tag":181,"props":15268,"children":15270},{"className":15269},[296],[15271,15276],{"type":150,"tag":181,"props":15272,"children":15274},{"className":15273,"style":3691},[296,301],[15275],{"type":155,"value":3265},{"type":150,"tag":181,"props":15277,"children":15279},{"className":15278},[307],[15280],{"type":150,"tag":181,"props":15281,"children":15283},{"className":15282},[312,313],[15284,15348],{"type":150,"tag":181,"props":15285,"children":15287},{"className":15286},[318],[15288,15343],{"type":150,"tag":181,"props":15289,"children":15291},{"className":15290,"style":1276},[323],[15292,15308],{"type":150,"tag":181,"props":15293,"children":15294},{"style":4108},[15295,15299],{"type":150,"tag":181,"props":15296,"children":15298},{"className":15297,"style":334},[333],[],{"type":150,"tag":181,"props":15300,"children":15302},{"className":15301},[339,340,341,342],[15303],{"type":150,"tag":181,"props":15304,"children":15306},{"className":15305,"style":2249},[296,301,342],[15307],{"type":155,"value":1698},{"type":150,"tag":181,"props":15309,"children":15310},{"style":1297},[15311,15315],{"type":150,"tag":181,"props":15312,"children":15314},{"className":15313,"style":334},[333],[],{"type":150,"tag":181,"props":15316,"children":15318},{"className":15317},[339,340,341,342],[15319],{"type":150,"tag":181,"props":15320,"children":15322},{"className":15321},[296,342],[15323,15328,15333,15338],{"type":150,"tag":181,"props":15324,"children":15326},{"className":15325},[824,342],[15327],{"type":155,"value":792},{"type":150,"tag":181,"props":15329,"children":15331},{"className":15330},[296,301,342],[15332],{"type":155,"value":797},{"type":150,"tag":181,"props":15334,"children":15336},{"className":15335},[835,342],[15337],{"type":155,"value":802},{"type":150,"tag":181,"props":15339,"children":15341},{"className":15340},[296,342],[15342],{"type":155,"value":1874},{"type":150,"tag":181,"props":15344,"children":15346},{"className":15345},[352],[15347],{"type":155,"value":355},{"type":150,"tag":181,"props":15349,"children":15351},{"className":15350},[318],[15352],{"type":150,"tag":181,"props":15353,"children":15355},{"className":15354,"style":4165},[323],[15356],{"type":150,"tag":181,"props":15357,"children":15358},{},[],{"type":150,"tag":181,"props":15360,"children":15362},{"className":15361},[835],[15363],{"type":155,"value":802},{"type":150,"tag":181,"props":15365,"children":15367},{"className":15366},[296],[15368,15373],{"type":150,"tag":181,"props":15369,"children":15371},{"className":15370},[296,301],[15372],{"type":155,"value":9859},{"type":150,"tag":181,"props":15374,"children":15376},{"className":15375},[307],[15377],{"type":150,"tag":181,"props":15378,"children":15380},{"className":15379},[312,313],[15381,15427],{"type":150,"tag":181,"props":15382,"children":15384},{"className":15383},[318],[15385,15422],{"type":150,"tag":181,"props":15386,"children":15388},{"className":15387,"style":3821},[323],[15389,15406],{"type":150,"tag":181,"props":15390,"children":15392},{"style":15391},"top:-2.453em;margin-left:0em;margin-right:0.05em;",[15393,15397],{"type":150,"tag":181,"props":15394,"children":15396},{"className":15395,"style":334},[333],[],{"type":150,"tag":181,"props":15398,"children":15400},{"className":15399},[339,340,341,342],[15401],{"type":150,"tag":181,"props":15402,"children":15404},{"className":15403},[296,301,342],[15405],{"type":155,"value":558},{"type":150,"tag":181,"props":15407,"children":15408},{"style":3730},[15409,15413],{"type":150,"tag":181,"props":15410,"children":15412},{"className":15411,"style":334},[333],[],{"type":150,"tag":181,"props":15414,"children":15416},{"className":15415},[339,340,341,342],[15417],{"type":150,"tag":181,"props":15418,"children":15420},{"className":15419},[296,342],[15421],{"type":155,"value":1874},{"type":150,"tag":181,"props":15423,"children":15425},{"className":15424},[352],[15426],{"type":155,"value":355},{"type":150,"tag":181,"props":15428,"children":15430},{"className":15429},[318],[15431],{"type":150,"tag":181,"props":15432,"children":15434},{"className":15433,"style":4550},[323],[15435],{"type":150,"tag":181,"props":15436,"children":15437},{},[],{"type":150,"tag":151,"props":15439,"children":15440},{},[15441,15443,15714,15716,15817,15819,15921,15923,16278,16280,16428,16430,16532,16534,16635,16637,16738,16740,16841,16843,16903],{"type":155,"value":15442},"这意味着可以把 ",{"type":150,"tag":181,"props":15444,"children":15446},{"className":15445},[184],[15447,15521],{"type":150,"tag":181,"props":15448,"children":15450},{"className":15449},[189],[15451],{"type":150,"tag":192,"props":15452,"children":15453},{"xmlns":194},[15454],{"type":150,"tag":197,"props":15455,"children":15456},{},[15457,15516],{"type":150,"tag":201,"props":15458,"children":15459},{},[15460,15486],{"type":150,"tag":1046,"props":15461,"children":15462},{},[15463,15467,15471],{"type":150,"tag":209,"props":15464,"children":15465},{},[15466],{"type":155,"value":3265},{"type":150,"tag":209,"props":15468,"children":15469},{},[15470],{"type":155,"value":1667},{"type":150,"tag":201,"props":15472,"children":15473},{},[15474,15478,15482],{"type":150,"tag":221,"props":15475,"children":15476},{"stretchy":789},[15477],{"type":155,"value":792},{"type":150,"tag":209,"props":15479,"children":15480},{},[15481],{"type":155,"value":797},{"type":150,"tag":221,"props":15483,"children":15484},{"stretchy":789},[15485],{"type":155,"value":802},{"type":150,"tag":1046,"props":15487,"children":15488},{},[15489,15493,15497],{"type":150,"tag":209,"props":15490,"children":15491},{},[15492],{"type":155,"value":3265},{"type":150,"tag":209,"props":15494,"children":15495},{},[15496],{"type":155,"value":1698},{"type":150,"tag":201,"props":15498,"children":15499},{},[15500,15504,15508,15512],{"type":150,"tag":221,"props":15501,"children":15502},{"stretchy":789},[15503],{"type":155,"value":792},{"type":150,"tag":209,"props":15505,"children":15506},{},[15507],{"type":155,"value":797},{"type":150,"tag":221,"props":15509,"children":15510},{"stretchy":789},[15511],{"type":155,"value":802},{"type":150,"tag":209,"props":15513,"children":15514},{"mathvariant":1618},[15515],{"type":155,"value":1874},{"type":150,"tag":271,"props":15517,"children":15518},{"encoding":273},[15519],{"type":155,"value":15520},"W_q^{(s)} W_k^{(s)\\top}",{"type":150,"tag":181,"props":15522,"children":15524},{"className":15523,"ariaHidden":223},[280],[15525],{"type":150,"tag":181,"props":15526,"children":15528},{"className":15527},[285],[15529,15533,15622],{"type":150,"tag":181,"props":15530,"children":15532},{"className":15531,"style":4022},[290],[],{"type":150,"tag":181,"props":15534,"children":15536},{"className":15535},[296],[15537,15542],{"type":150,"tag":181,"props":15538,"children":15540},{"className":15539,"style":3691},[296,301],[15541],{"type":155,"value":3265},{"type":150,"tag":181,"props":15543,"children":15545},{"className":15544},[307],[15546],{"type":150,"tag":181,"props":15547,"children":15549},{"className":15548},[312,313],[15550,15610],{"type":150,"tag":181,"props":15551,"children":15553},{"className":15552},[318],[15554,15605],{"type":150,"tag":181,"props":15555,"children":15557},{"className":15556,"style":1276},[323],[15558,15575],{"type":150,"tag":181,"props":15559,"children":15561},{"style":15560},"top:-2.5834em;margin-left:-0.1389em;margin-right:0.05em;",[15562,15566],{"type":150,"tag":181,"props":15563,"children":15565},{"className":15564,"style":334},[333],[],{"type":150,"tag":181,"props":15567,"children":15569},{"className":15568},[339,340,341,342],[15570],{"type":150,"tag":181,"props":15571,"children":15573},{"className":15572,"style":2151},[296,301,342],[15574],{"type":155,"value":1667},{"type":150,"tag":181,"props":15576,"children":15577},{"style":1297},[15578,15582],{"type":150,"tag":181,"props":15579,"children":15581},{"className":15580,"style":334},[333],[],{"type":150,"tag":181,"props":15583,"children":15585},{"className":15584},[339,340,341,342],[15586],{"type":150,"tag":181,"props":15587,"children":15589},{"className":15588},[296,342],[15590,15595,15600],{"type":150,"tag":181,"props":15591,"children":15593},{"className":15592},[824,342],[15594],{"type":155,"value":792},{"type":150,"tag":181,"props":15596,"children":15598},{"className":15597},[296,301,342],[15599],{"type":155,"value":797},{"type":150,"tag":181,"props":15601,"children":15603},{"className":15602},[835,342],[15604],{"type":155,"value":802},{"type":150,"tag":181,"props":15606,"children":15608},{"className":15607},[352],[15609],{"type":155,"value":355},{"type":150,"tag":181,"props":15611,"children":15613},{"className":15612},[318],[15614],{"type":150,"tag":181,"props":15615,"children":15618},{"className":15616,"style":15617},[323],"height:0.2527em;",[15619],{"type":150,"tag":181,"props":15620,"children":15621},{},[],{"type":150,"tag":181,"props":15623,"children":15625},{"className":15624},[296],[15626,15631],{"type":150,"tag":181,"props":15627,"children":15629},{"className":15628,"style":3691},[296,301],[15630],{"type":155,"value":3265},{"type":150,"tag":181,"props":15632,"children":15634},{"className":15633},[307],[15635],{"type":150,"tag":181,"props":15636,"children":15638},{"className":15637},[312,313],[15639,15703],{"type":150,"tag":181,"props":15640,"children":15642},{"className":15641},[318],[15643,15698],{"type":150,"tag":181,"props":15644,"children":15646},{"className":15645,"style":1276},[323],[15647,15663],{"type":150,"tag":181,"props":15648,"children":15649},{"style":4108},[15650,15654],{"type":150,"tag":181,"props":15651,"children":15653},{"className":15652,"style":334},[333],[],{"type":150,"tag":181,"props":15655,"children":15657},{"className":15656},[339,340,341,342],[15658],{"type":150,"tag":181,"props":15659,"children":15661},{"className":15660,"style":2249},[296,301,342],[15662],{"type":155,"value":1698},{"type":150,"tag":181,"props":15664,"children":15665},{"style":1297},[15666,15670],{"type":150,"tag":181,"props":15667,"children":15669},{"className":15668,"style":334},[333],[],{"type":150,"tag":181,"props":15671,"children":15673},{"className":15672},[339,340,341,342],[15674],{"type":150,"tag":181,"props":15675,"children":15677},{"className":15676},[296,342],[15678,15683,15688,15693],{"type":150,"tag":181,"props":15679,"children":15681},{"className":15680},[824,342],[15682],{"type":155,"value":792},{"type":150,"tag":181,"props":15684,"children":15686},{"className":15685},[296,301,342],[15687],{"type":155,"value":797},{"type":150,"tag":181,"props":15689,"children":15691},{"className":15690},[835,342],[15692],{"type":155,"value":802},{"type":150,"tag":181,"props":15694,"children":15696},{"className":15695},[296,342],[15697],{"type":155,"value":1874},{"type":150,"tag":181,"props":15699,"children":15701},{"className":15700},[352],[15702],{"type":155,"value":355},{"type":150,"tag":181,"props":15704,"children":15706},{"className":15705},[318],[15707],{"type":150,"tag":181,"props":15708,"children":15710},{"className":15709,"style":4165},[323],[15711],{"type":150,"tag":181,"props":15712,"children":15713},{},[],{"type":155,"value":15715}," 合并起来当作 Q 的投影矩阵，那么 ",{"type":150,"tag":181,"props":15717,"children":15719},{"className":15718},[184],[15720,15748],{"type":150,"tag":181,"props":15721,"children":15723},{"className":15722},[189],[15724],{"type":150,"tag":192,"props":15725,"children":15726},{"xmlns":194},[15727],{"type":150,"tag":197,"props":15728,"children":15729},{},[15730,15744],{"type":150,"tag":201,"props":15731,"children":15732},{},[15733],{"type":150,"tag":205,"props":15734,"children":15735},{},[15736,15740],{"type":150,"tag":209,"props":15737,"children":15738},{},[15739],{"type":155,"value":9859},{"type":150,"tag":209,"props":15741,"children":15742},{},[15743],{"type":155,"value":558},{"type":150,"tag":271,"props":15745,"children":15746},{"encoding":273},[15747],{"type":155,"value":9868},{"type":150,"tag":181,"props":15749,"children":15751},{"className":15750,"ariaHidden":223},[280],[15752],{"type":150,"tag":181,"props":15753,"children":15755},{"className":15754},[285],[15756,15760],{"type":150,"tag":181,"props":15757,"children":15759},{"className":15758,"style":1165},[290],[],{"type":150,"tag":181,"props":15761,"children":15763},{"className":15762},[296],[15764,15769],{"type":150,"tag":181,"props":15765,"children":15767},{"className":15766},[296,301],[15768],{"type":155,"value":9859},{"type":150,"tag":181,"props":15770,"children":15772},{"className":15771},[307],[15773],{"type":150,"tag":181,"props":15774,"children":15776},{"className":15775},[312,313],[15777,15806],{"type":150,"tag":181,"props":15778,"children":15780},{"className":15779},[318],[15781,15801],{"type":150,"tag":181,"props":15782,"children":15784},{"className":15783,"style":621},[323],[15785],{"type":150,"tag":181,"props":15786,"children":15787},{"style":328},[15788,15792],{"type":150,"tag":181,"props":15789,"children":15791},{"className":15790,"style":334},[333],[],{"type":150,"tag":181,"props":15793,"children":15795},{"className":15794},[339,340,341,342],[15796],{"type":150,"tag":181,"props":15797,"children":15799},{"className":15798},[296,301,342],[15800],{"type":155,"value":558},{"type":150,"tag":181,"props":15802,"children":15804},{"className":15803},[352],[15805],{"type":155,"value":355},{"type":150,"tag":181,"props":15807,"children":15809},{"className":15808},[318],[15810],{"type":150,"tag":181,"props":15811,"children":15813},{"className":15812,"style":363},[323],[15814],{"type":150,"tag":181,"props":15815,"children":15816},{},[],{"type":155,"value":15818}," 就取代了 ",{"type":150,"tag":181,"props":15820,"children":15822},{"className":15821},[184],[15823,15852],{"type":150,"tag":181,"props":15824,"children":15826},{"className":15825},[189],[15827],{"type":150,"tag":192,"props":15828,"children":15829},{"xmlns":194},[15830],{"type":150,"tag":197,"props":15831,"children":15832},{},[15833,15847],{"type":150,"tag":201,"props":15834,"children":15835},{},[15836],{"type":150,"tag":205,"props":15837,"children":15838},{},[15839,15843],{"type":150,"tag":209,"props":15840,"children":15841},{},[15842],{"type":155,"value":1698},{"type":150,"tag":209,"props":15844,"children":15845},{},[15846],{"type":155,"value":558},{"type":150,"tag":271,"props":15848,"children":15849},{"encoding":273},[15850],{"type":155,"value":15851},"k_i",{"type":150,"tag":181,"props":15853,"children":15855},{"className":15854,"ariaHidden":223},[280],[15856],{"type":150,"tag":181,"props":15857,"children":15859},{"className":15858},[285],[15860,15864],{"type":150,"tag":181,"props":15861,"children":15863},{"className":15862,"style":4761},[290],[],{"type":150,"tag":181,"props":15865,"children":15867},{"className":15866},[296],[15868,15873],{"type":150,"tag":181,"props":15869,"children":15871},{"className":15870,"style":2249},[296,301],[15872],{"type":155,"value":1698},{"type":150,"tag":181,"props":15874,"children":15876},{"className":15875},[307],[15877],{"type":150,"tag":181,"props":15878,"children":15880},{"className":15879},[312,313],[15881,15910],{"type":150,"tag":181,"props":15882,"children":15884},{"className":15883},[318],[15885,15905],{"type":150,"tag":181,"props":15886,"children":15888},{"className":15887,"style":621},[323],[15889],{"type":150,"tag":181,"props":15890,"children":15891},{"style":14163},[15892,15896],{"type":150,"tag":181,"props":15893,"children":15895},{"className":15894,"style":334},[333],[],{"type":150,"tag":181,"props":15897,"children":15899},{"className":15898},[339,340,341,342],[15900],{"type":150,"tag":181,"props":15901,"children":15903},{"className":15902},[296,301,342],[15904],{"type":155,"value":558},{"type":150,"tag":181,"props":15906,"children":15908},{"className":15907},[352],[15909],{"type":155,"value":355},{"type":150,"tag":181,"props":15911,"children":15913},{"className":15912},[318],[15914],{"type":150,"tag":181,"props":15915,"children":15917},{"className":15916,"style":363},[323],[15918],{"type":150,"tag":181,"props":15919,"children":15920},{},[],{"type":155,"value":15922},"。同理 ",{"type":150,"tag":181,"props":15924,"children":15926},{"className":15925},[184],[15927,16012],{"type":150,"tag":181,"props":15928,"children":15930},{"className":15929},[189],[15931],{"type":150,"tag":192,"props":15932,"children":15933},{"xmlns":194},[15934],{"type":150,"tag":197,"props":15935,"children":15936},{},[15937,16007],{"type":150,"tag":201,"props":15938,"children":15939},{},[15940,15966,15970,15981],{"type":150,"tag":1046,"props":15941,"children":15942},{},[15943,15947,15951],{"type":150,"tag":209,"props":15944,"children":15945},{},[15946],{"type":155,"value":1737},{"type":150,"tag":209,"props":15948,"children":15949},{},[15950],{"type":155,"value":558},{"type":150,"tag":201,"props":15952,"children":15953},{},[15954,15958,15962],{"type":150,"tag":221,"props":15955,"children":15956},{"stretchy":789},[15957],{"type":155,"value":792},{"type":150,"tag":209,"props":15959,"children":15960},{},[15961],{"type":155,"value":797},{"type":150,"tag":221,"props":15963,"children":15964},{"stretchy":789},[15965],{"type":155,"value":802},{"type":150,"tag":221,"props":15967,"children":15968},{},[15969],{"type":155,"value":865},{"type":150,"tag":205,"props":15971,"children":15972},{},[15973,15977],{"type":150,"tag":209,"props":15974,"children":15975},{},[15976],{"type":155,"value":9859},{"type":150,"tag":209,"props":15978,"children":15979},{},[15980],{"type":155,"value":558},{"type":150,"tag":1046,"props":15982,"children":15983},{},[15984,15988,15992],{"type":150,"tag":209,"props":15985,"children":15986},{},[15987],{"type":155,"value":3265},{"type":150,"tag":209,"props":15989,"children":15990},{},[15991],{"type":155,"value":1737},{"type":150,"tag":201,"props":15993,"children":15994},{},[15995,15999,16003],{"type":150,"tag":221,"props":15996,"children":15997},{"stretchy":789},[15998],{"type":155,"value":792},{"type":150,"tag":209,"props":16000,"children":16001},{},[16002],{"type":155,"value":797},{"type":150,"tag":221,"props":16004,"children":16005},{"stretchy":789},[16006],{"type":155,"value":802},{"type":150,"tag":271,"props":16008,"children":16009},{"encoding":273},[16010],{"type":155,"value":16011},"v_i^{(s)} = c_i W_v^{(s)}",{"type":150,"tag":181,"props":16013,"children":16015},{"className":16014,"ariaHidden":223},[280],[16016,16124],{"type":150,"tag":181,"props":16017,"children":16019},{"className":16018},[285],[16020,16024,16111,16115,16120],{"type":150,"tag":181,"props":16021,"children":16023},{"className":16022,"style":3516},[290],[],{"type":150,"tag":181,"props":16025,"children":16027},{"className":16026},[296],[16028,16033],{"type":150,"tag":181,"props":16029,"children":16031},{"className":16030,"style":2151},[296,301],[16032],{"type":155,"value":1737},{"type":150,"tag":181,"props":16034,"children":16036},{"className":16035},[307],[16037],{"type":150,"tag":181,"props":16038,"children":16040},{"className":16039},[312,313],[16041,16100],{"type":150,"tag":181,"props":16042,"children":16044},{"className":16043},[318],[16045,16095],{"type":150,"tag":181,"props":16046,"children":16048},{"className":16047,"style":1276},[323],[16049,16065],{"type":150,"tag":181,"props":16050,"children":16051},{"style":3114},[16052,16056],{"type":150,"tag":181,"props":16053,"children":16055},{"className":16054,"style":334},[333],[],{"type":150,"tag":181,"props":16057,"children":16059},{"className":16058},[339,340,341,342],[16060],{"type":150,"tag":181,"props":16061,"children":16063},{"className":16062},[296,301,342],[16064],{"type":155,"value":558},{"type":150,"tag":181,"props":16066,"children":16067},{"style":1297},[16068,16072],{"type":150,"tag":181,"props":16069,"children":16071},{"className":16070,"style":334},[333],[],{"type":150,"tag":181,"props":16073,"children":16075},{"className":16074},[339,340,341,342],[16076],{"type":150,"tag":181,"props":16077,"children":16079},{"className":16078},[296,342],[16080,16085,16090],{"type":150,"tag":181,"props":16081,"children":16083},{"className":16082},[824,342],[16084],{"type":155,"value":792},{"type":150,"tag":181,"props":16086,"children":16088},{"className":16087},[296,301,342],[16089],{"type":155,"value":797},{"type":150,"tag":181,"props":16091,"children":16093},{"className":16092},[835,342],[16094],{"type":155,"value":802},{"type":150,"tag":181,"props":16096,"children":16098},{"className":16097},[352],[16099],{"type":155,"value":355},{"type":150,"tag":181,"props":16101,"children":16103},{"className":16102},[318],[16104],{"type":150,"tag":181,"props":16105,"children":16107},{"className":16106,"style":2782},[323],[16108],{"type":150,"tag":181,"props":16109,"children":16110},{},[],{"type":150,"tag":181,"props":16112,"children":16114},{"className":16113,"style":658},[377],[],{"type":150,"tag":181,"props":16116,"children":16118},{"className":16117},[663],[16119],{"type":155,"value":865},{"type":150,"tag":181,"props":16121,"children":16123},{"className":16122,"style":658},[377],[],{"type":150,"tag":181,"props":16125,"children":16127},{"className":16126},[285],[16128,16133,16190],{"type":150,"tag":181,"props":16129,"children":16132},{"className":16130,"style":16131},[290],"height:1.1948em;vertical-align:-0.15em;",[],{"type":150,"tag":181,"props":16134,"children":16136},{"className":16135},[296],[16137,16142],{"type":150,"tag":181,"props":16138,"children":16140},{"className":16139},[296,301],[16141],{"type":155,"value":9859},{"type":150,"tag":181,"props":16143,"children":16145},{"className":16144},[307],[16146],{"type":150,"tag":181,"props":16147,"children":16149},{"className":16148},[312,313],[16150,16179],{"type":150,"tag":181,"props":16151,"children":16153},{"className":16152},[318],[16154,16174],{"type":150,"tag":181,"props":16155,"children":16157},{"className":16156,"style":621},[323],[16158],{"type":150,"tag":181,"props":16159,"children":16160},{"style":328},[16161,16165],{"type":150,"tag":181,"props":16162,"children":16164},{"className":16163,"style":334},[333],[],{"type":150,"tag":181,"props":16166,"children":16168},{"className":16167},[339,340,341,342],[16169],{"type":150,"tag":181,"props":16170,"children":16172},{"className":16171},[296,301,342],[16173],{"type":155,"value":558},{"type":150,"tag":181,"props":16175,"children":16177},{"className":16176},[352],[16178],{"type":155,"value":355},{"type":150,"tag":181,"props":16180,"children":16182},{"className":16181},[318],[16183],{"type":150,"tag":181,"props":16184,"children":16186},{"className":16185,"style":363},[323],[16187],{"type":150,"tag":181,"props":16188,"children":16189},{},[],{"type":150,"tag":181,"props":16191,"children":16193},{"className":16192},[296],[16194,16199],{"type":150,"tag":181,"props":16195,"children":16197},{"className":16196,"style":3691},[296,301],[16198],{"type":155,"value":3265},{"type":150,"tag":181,"props":16200,"children":16202},{"className":16201},[307],[16203],{"type":150,"tag":181,"props":16204,"children":16206},{"className":16205},[312,313],[16207,16266],{"type":150,"tag":181,"props":16208,"children":16210},{"className":16209},[318],[16211,16261],{"type":150,"tag":181,"props":16212,"children":16214},{"className":16213,"style":1276},[323],[16215,16231],{"type":150,"tag":181,"props":16216,"children":16217},{"style":15560},[16218,16222],{"type":150,"tag":181,"props":16219,"children":16221},{"className":16220,"style":334},[333],[],{"type":150,"tag":181,"props":16223,"children":16225},{"className":16224},[339,340,341,342],[16226],{"type":150,"tag":181,"props":16227,"children":16229},{"className":16228,"style":2151},[296,301,342],[16230],{"type":155,"value":1737},{"type":150,"tag":181,"props":16232,"children":16233},{"style":1297},[16234,16238],{"type":150,"tag":181,"props":16235,"children":16237},{"className":16236,"style":334},[333],[],{"type":150,"tag":181,"props":16239,"children":16241},{"className":16240},[339,340,341,342],[16242],{"type":150,"tag":181,"props":16243,"children":16245},{"className":16244},[296,342],[16246,16251,16256],{"type":150,"tag":181,"props":16247,"children":16249},{"className":16248},[824,342],[16250],{"type":155,"value":792},{"type":150,"tag":181,"props":16252,"children":16254},{"className":16253},[296,301,342],[16255],{"type":155,"value":797},{"type":150,"tag":181,"props":16257,"children":16259},{"className":16258},[835,342],[16260],{"type":155,"value":802},{"type":150,"tag":181,"props":16262,"children":16264},{"className":16263},[352],[16265],{"type":155,"value":355},{"type":150,"tag":181,"props":16267,"children":16269},{"className":16268},[318],[16270],{"type":150,"tag":181,"props":16271,"children":16274},{"className":16272,"style":16273},[323],"height:0.1166em;",[16275],{"type":150,"tag":181,"props":16276,"children":16277},{},[],{"type":155,"value":16279}," 里的 ",{"type":150,"tag":181,"props":16281,"children":16283},{"className":16282},[184],[16284,16328],{"type":150,"tag":181,"props":16285,"children":16287},{"className":16286},[189],[16288],{"type":150,"tag":192,"props":16289,"children":16290},{"xmlns":194},[16291],{"type":150,"tag":197,"props":16292,"children":16293},{},[16294,16323],{"type":150,"tag":201,"props":16295,"children":16296},{},[16297],{"type":150,"tag":1046,"props":16298,"children":16299},{},[16300,16304,16308],{"type":150,"tag":209,"props":16301,"children":16302},{},[16303],{"type":155,"value":3265},{"type":150,"tag":209,"props":16305,"children":16306},{},[16307],{"type":155,"value":1737},{"type":150,"tag":201,"props":16309,"children":16310},{},[16311,16315,16319],{"type":150,"tag":221,"props":16312,"children":16313},{"stretchy":789},[16314],{"type":155,"value":792},{"type":150,"tag":209,"props":16316,"children":16317},{},[16318],{"type":155,"value":797},{"type":150,"tag":221,"props":16320,"children":16321},{"stretchy":789},[16322],{"type":155,"value":802},{"type":150,"tag":271,"props":16324,"children":16325},{"encoding":273},[16326],{"type":155,"value":16327},"W_v^{(s)}",{"type":150,"tag":181,"props":16329,"children":16331},{"className":16330,"ariaHidden":223},[280],[16332],{"type":150,"tag":181,"props":16333,"children":16335},{"className":16334},[285],[16336,16341],{"type":150,"tag":181,"props":16337,"children":16340},{"className":16338,"style":16339},[290],"height:1.1614em;vertical-align:-0.1166em;",[],{"type":150,"tag":181,"props":16342,"children":16344},{"className":16343},[296],[16345,16350],{"type":150,"tag":181,"props":16346,"children":16348},{"className":16347,"style":3691},[296,301],[16349],{"type":155,"value":3265},{"type":150,"tag":181,"props":16351,"children":16353},{"className":16352},[307],[16354],{"type":150,"tag":181,"props":16355,"children":16357},{"className":16356},[312,313],[16358,16417],{"type":150,"tag":181,"props":16359,"children":16361},{"className":16360},[318],[16362,16412],{"type":150,"tag":181,"props":16363,"children":16365},{"className":16364,"style":1276},[323],[16366,16382],{"type":150,"tag":181,"props":16367,"children":16368},{"style":15560},[16369,16373],{"type":150,"tag":181,"props":16370,"children":16372},{"className":16371,"style":334},[333],[],{"type":150,"tag":181,"props":16374,"children":16376},{"className":16375},[339,340,341,342],[16377],{"type":150,"tag":181,"props":16378,"children":16380},{"className":16379,"style":2151},[296,301,342],[16381],{"type":155,"value":1737},{"type":150,"tag":181,"props":16383,"children":16384},{"style":1297},[16385,16389],{"type":150,"tag":181,"props":16386,"children":16388},{"className":16387,"style":334},[333],[],{"type":150,"tag":181,"props":16390,"children":16392},{"className":16391},[339,340,341,342],[16393],{"type":150,"tag":181,"props":16394,"children":16396},{"className":16395},[296,342],[16397,16402,16407],{"type":150,"tag":181,"props":16398,"children":16400},{"className":16399},[824,342],[16401],{"type":155,"value":792},{"type":150,"tag":181,"props":16403,"children":16405},{"className":16404},[296,301,342],[16406],{"type":155,"value":797},{"type":150,"tag":181,"props":16408,"children":16410},{"className":16409},[835,342],[16411],{"type":155,"value":802},{"type":150,"tag":181,"props":16413,"children":16415},{"className":16414},[352],[16416],{"type":155,"value":355},{"type":150,"tag":181,"props":16418,"children":16420},{"className":16419},[318],[16421],{"type":150,"tag":181,"props":16422,"children":16424},{"className":16423,"style":16273},[323],[16425],{"type":150,"tag":181,"props":16426,"children":16427},{},[],{"type":155,"value":16429}," 也能吸收进后面的输出投影，",{"type":150,"tag":181,"props":16431,"children":16433},{"className":16432},[184],[16434,16463],{"type":150,"tag":181,"props":16435,"children":16437},{"className":16436},[189],[16438],{"type":150,"tag":192,"props":16439,"children":16440},{"xmlns":194},[16441],{"type":150,"tag":197,"props":16442,"children":16443},{},[16444,16458],{"type":150,"tag":201,"props":16445,"children":16446},{},[16447],{"type":150,"tag":205,"props":16448,"children":16449},{},[16450,16454],{"type":150,"tag":209,"props":16451,"children":16452},{},[16453],{"type":155,"value":1737},{"type":150,"tag":209,"props":16455,"children":16456},{},[16457],{"type":155,"value":558},{"type":150,"tag":271,"props":16459,"children":16460},{"encoding":273},[16461],{"type":155,"value":16462},"v_i",{"type":150,"tag":181,"props":16464,"children":16466},{"className":16465,"ariaHidden":223},[280],[16467],{"type":150,"tag":181,"props":16468,"children":16470},{"className":16469},[285],[16471,16475],{"type":150,"tag":181,"props":16472,"children":16474},{"className":16473,"style":1165},[290],[],{"type":150,"tag":181,"props":16476,"children":16478},{"className":16477},[296],[16479,16484],{"type":150,"tag":181,"props":16480,"children":16482},{"className":16481,"style":2151},[296,301],[16483],{"type":155,"value":1737},{"type":150,"tag":181,"props":16485,"children":16487},{"className":16486},[307],[16488],{"type":150,"tag":181,"props":16489,"children":16491},{"className":16490},[312,313],[16492,16521],{"type":150,"tag":181,"props":16493,"children":16495},{"className":16494},[318],[16496,16516],{"type":150,"tag":181,"props":16497,"children":16499},{"className":16498,"style":621},[323],[16500],{"type":150,"tag":181,"props":16501,"children":16502},{"style":14230},[16503,16507],{"type":150,"tag":181,"props":16504,"children":16506},{"className":16505,"style":334},[333],[],{"type":150,"tag":181,"props":16508,"children":16510},{"className":16509},[339,340,341,342],[16511],{"type":150,"tag":181,"props":16512,"children":16514},{"className":16513},[296,301,342],[16515],{"type":155,"value":558},{"type":150,"tag":181,"props":16517,"children":16519},{"className":16518},[352],[16520],{"type":155,"value":355},{"type":150,"tag":181,"props":16522,"children":16524},{"className":16523},[318],[16525],{"type":150,"tag":181,"props":16526,"children":16528},{"className":16527,"style":363},[323],[16529],{"type":150,"tag":181,"props":16530,"children":16531},{},[],{"type":155,"value":16533}," 也可以用 ",{"type":150,"tag":181,"props":16535,"children":16537},{"className":16536},[184],[16538,16566],{"type":150,"tag":181,"props":16539,"children":16541},{"className":16540},[189],[16542],{"type":150,"tag":192,"props":16543,"children":16544},{"xmlns":194},[16545],{"type":150,"tag":197,"props":16546,"children":16547},{},[16548,16562],{"type":150,"tag":201,"props":16549,"children":16550},{},[16551],{"type":150,"tag":205,"props":16552,"children":16553},{},[16554,16558],{"type":150,"tag":209,"props":16555,"children":16556},{},[16557],{"type":155,"value":9859},{"type":150,"tag":209,"props":16559,"children":16560},{},[16561],{"type":155,"value":558},{"type":150,"tag":271,"props":16563,"children":16564},{"encoding":273},[16565],{"type":155,"value":9868},{"type":150,"tag":181,"props":16567,"children":16569},{"className":16568,"ariaHidden":223},[280],[16570],{"type":150,"tag":181,"props":16571,"children":16573},{"className":16572},[285],[16574,16578],{"type":150,"tag":181,"props":16575,"children":16577},{"className":16576,"style":1165},[290],[],{"type":150,"tag":181,"props":16579,"children":16581},{"className":16580},[296],[16582,16587],{"type":150,"tag":181,"props":16583,"children":16585},{"className":16584},[296,301],[16586],{"type":155,"value":9859},{"type":150,"tag":181,"props":16588,"children":16590},{"className":16589},[307],[16591],{"type":150,"tag":181,"props":16592,"children":16594},{"className":16593},[312,313],[16595,16624],{"type":150,"tag":181,"props":16596,"children":16598},{"className":16597},[318],[16599,16619],{"type":150,"tag":181,"props":16600,"children":16602},{"className":16601,"style":621},[323],[16603],{"type":150,"tag":181,"props":16604,"children":16605},{"style":328},[16606,16610],{"type":150,"tag":181,"props":16607,"children":16609},{"className":16608,"style":334},[333],[],{"type":150,"tag":181,"props":16611,"children":16613},{"className":16612},[339,340,341,342],[16614],{"type":150,"tag":181,"props":16615,"children":16617},{"className":16616},[296,301,342],[16618],{"type":155,"value":558},{"type":150,"tag":181,"props":16620,"children":16622},{"className":16621},[352],[16623],{"type":155,"value":355},{"type":150,"tag":181,"props":16625,"children":16627},{"className":16626},[318],[16628],{"type":150,"tag":181,"props":16629,"children":16631},{"className":16630,"style":363},[323],[16632],{"type":150,"tag":181,"props":16633,"children":16634},{},[],{"type":155,"value":16636}," 代替。于是推理时 KV Cache 只需要存 ",{"type":150,"tag":181,"props":16638,"children":16640},{"className":16639},[184],[16641,16669],{"type":150,"tag":181,"props":16642,"children":16644},{"className":16643},[189],[16645],{"type":150,"tag":192,"props":16646,"children":16647},{"xmlns":194},[16648],{"type":150,"tag":197,"props":16649,"children":16650},{},[16651,16665],{"type":150,"tag":201,"props":16652,"children":16653},{},[16654],{"type":150,"tag":205,"props":16655,"children":16656},{},[16657,16661],{"type":150,"tag":209,"props":16658,"children":16659},{},[16660],{"type":155,"value":9859},{"type":150,"tag":209,"props":16662,"children":16663},{},[16664],{"type":155,"value":558},{"type":150,"tag":271,"props":16666,"children":16667},{"encoding":273},[16668],{"type":155,"value":9868},{"type":150,"tag":181,"props":16670,"children":16672},{"className":16671,"ariaHidden":223},[280],[16673],{"type":150,"tag":181,"props":16674,"children":16676},{"className":16675},[285],[16677,16681],{"type":150,"tag":181,"props":16678,"children":16680},{"className":16679,"style":1165},[290],[],{"type":150,"tag":181,"props":16682,"children":16684},{"className":16683},[296],[16685,16690],{"type":150,"tag":181,"props":16686,"children":16688},{"className":16687},[296,301],[16689],{"type":155,"value":9859},{"type":150,"tag":181,"props":16691,"children":16693},{"className":16692},[307],[16694],{"type":150,"tag":181,"props":16695,"children":16697},{"className":16696},[312,313],[16698,16727],{"type":150,"tag":181,"props":16699,"children":16701},{"className":16700},[318],[16702,16722],{"type":150,"tag":181,"props":16703,"children":16705},{"className":16704,"style":621},[323],[16706],{"type":150,"tag":181,"props":16707,"children":16708},{"style":328},[16709,16713],{"type":150,"tag":181,"props":16710,"children":16712},{"className":16711,"style":334},[333],[],{"type":150,"tag":181,"props":16714,"children":16716},{"className":16715},[339,340,341,342],[16717],{"type":150,"tag":181,"props":16718,"children":16720},{"className":16719},[296,301,342],[16721],{"type":155,"value":558},{"type":150,"tag":181,"props":16723,"children":16725},{"className":16724},[352],[16726],{"type":155,"value":355},{"type":150,"tag":181,"props":16728,"children":16730},{"className":16729},[318],[16731],{"type":150,"tag":181,"props":16732,"children":16734},{"className":16733,"style":363},[323],[16735],{"type":150,"tag":181,"props":16736,"children":16737},{},[],{"type":155,"value":16739},"。而 ",{"type":150,"tag":181,"props":16741,"children":16743},{"className":16742},[184],[16744,16772],{"type":150,"tag":181,"props":16745,"children":16747},{"className":16746},[189],[16748],{"type":150,"tag":192,"props":16749,"children":16750},{"xmlns":194},[16751],{"type":150,"tag":197,"props":16752,"children":16753},{},[16754,16768],{"type":150,"tag":201,"props":16755,"children":16756},{},[16757],{"type":150,"tag":205,"props":16758,"children":16759},{},[16760,16764],{"type":150,"tag":209,"props":16761,"children":16762},{},[16763],{"type":155,"value":9859},{"type":150,"tag":209,"props":16765,"children":16766},{},[16767],{"type":155,"value":558},{"type":150,"tag":271,"props":16769,"children":16770},{"encoding":273},[16771],{"type":155,"value":9868},{"type":150,"tag":181,"props":16773,"children":16775},{"className":16774,"ariaHidden":223},[280],[16776],{"type":150,"tag":181,"props":16777,"children":16779},{"className":16778},[285],[16780,16784],{"type":150,"tag":181,"props":16781,"children":16783},{"className":16782,"style":1165},[290],[],{"type":150,"tag":181,"props":16785,"children":16787},{"className":16786},[296],[16788,16793],{"type":150,"tag":181,"props":16789,"children":16791},{"className":16790},[296,301],[16792],{"type":155,"value":9859},{"type":150,"tag":181,"props":16794,"children":16796},{"className":16795},[307],[16797],{"type":150,"tag":181,"props":16798,"children":16800},{"className":16799},[312,313],[16801,16830],{"type":150,"tag":181,"props":16802,"children":16804},{"className":16803},[318],[16805,16825],{"type":150,"tag":181,"props":16806,"children":16808},{"className":16807,"style":621},[323],[16809],{"type":150,"tag":181,"props":16810,"children":16811},{"style":328},[16812,16816],{"type":150,"tag":181,"props":16813,"children":16815},{"className":16814,"style":334},[333],[],{"type":150,"tag":181,"props":16817,"children":16819},{"className":16818},[339,340,341,342],[16820],{"type":150,"tag":181,"props":16821,"children":16823},{"className":16822},[296,301,342],[16824],{"type":155,"value":558},{"type":150,"tag":181,"props":16826,"children":16828},{"className":16827},[352],[16829],{"type":155,"value":355},{"type":150,"tag":181,"props":16831,"children":16833},{"className":16832},[318],[16834],{"type":150,"tag":181,"props":16835,"children":16837},{"className":16836,"style":363},[323],[16838],{"type":150,"tag":181,"props":16839,"children":16840},{},[],{"type":155,"value":16842}," 与 ",{"type":150,"tag":181,"props":16844,"children":16846},{"className":16845},[184],[16847,16876],{"type":150,"tag":181,"props":16848,"children":16850},{"className":16849},[189],[16851],{"type":150,"tag":192,"props":16852,"children":16853},{"xmlns":194},[16854],{"type":150,"tag":197,"props":16855,"children":16856},{},[16857,16872],{"type":150,"tag":201,"props":16858,"children":16859},{},[16860,16864,16868],{"type":150,"tag":221,"props":16861,"children":16862},{"stretchy":789},[16863],{"type":155,"value":792},{"type":150,"tag":209,"props":16865,"children":16866},{},[16867],{"type":155,"value":797},{"type":150,"tag":221,"props":16869,"children":16870},{"stretchy":789},[16871],{"type":155,"value":802},{"type":150,"tag":271,"props":16873,"children":16874},{"encoding":273},[16875],{"type":155,"value":807},{"type":150,"tag":181,"props":16877,"children":16879},{"className":16878,"ariaHidden":223},[280],[16880],{"type":150,"tag":181,"props":16881,"children":16883},{"className":16882},[285],[16884,16888,16893,16898],{"type":150,"tag":181,"props":16885,"children":16887},{"className":16886,"style":819},[290],[],{"type":150,"tag":181,"props":16889,"children":16891},{"className":16890},[824],[16892],{"type":155,"value":792},{"type":150,"tag":181,"props":16894,"children":16896},{"className":16895},[296,301],[16897],{"type":155,"value":797},{"type":150,"tag":181,"props":16899,"children":16901},{"className":16900},[835],[16902],{"type":155,"value":802},{"type":155,"value":16904}," 无关，是所有 head 共享的。",{"type":150,"tag":151,"props":16906,"children":16907},{},[16908,16913,16915,17017,17019,17241],{"type":150,"tag":158,"props":16909,"children":16910},{},[16911],{"type":155,"value":16912},"MLA 在推理阶段恒等变换成了一个 MQA。",{"type":155,"value":16914}," 到这里 MLA 完成了什么就很清楚了：它用不同的投影矩阵增强了 GQA 的能力，同时在推理时保持同样大小的 KV Cache。反过来想，如果只需要跟 GQA 相当的能力，那 ",{"type":150,"tag":181,"props":16916,"children":16918},{"className":16917},[184],[16919,16948],{"type":150,"tag":181,"props":16920,"children":16922},{"className":16921},[189],[16923],{"type":150,"tag":192,"props":16924,"children":16925},{"xmlns":194},[16926],{"type":150,"tag":197,"props":16927,"children":16928},{},[16929,16943],{"type":150,"tag":201,"props":16930,"children":16931},{},[16932],{"type":150,"tag":205,"props":16933,"children":16934},{},[16935,16939],{"type":150,"tag":209,"props":16936,"children":16937},{},[16938],{"type":155,"value":578},{"type":150,"tag":209,"props":16940,"children":16941},{},[16942],{"type":155,"value":9859},{"type":150,"tag":271,"props":16944,"children":16945},{"encoding":273},[16946],{"type":155,"value":16947},"d_c",{"type":150,"tag":181,"props":16949,"children":16951},{"className":16950,"ariaHidden":223},[280],[16952],{"type":150,"tag":181,"props":16953,"children":16955},{"className":16954},[285],[16956,16960],{"type":150,"tag":181,"props":16957,"children":16959},{"className":16958,"style":4761},[290],[],{"type":150,"tag":181,"props":16961,"children":16963},{"className":16962},[296],[16964,16969],{"type":150,"tag":181,"props":16965,"children":16967},{"className":16966},[296,301],[16968],{"type":155,"value":578},{"type":150,"tag":181,"props":16970,"children":16972},{"className":16971},[307],[16973],{"type":150,"tag":181,"props":16974,"children":16976},{"className":16975},[312,313],[16977,17006],{"type":150,"tag":181,"props":16978,"children":16980},{"className":16979},[318],[16981,17001],{"type":150,"tag":181,"props":16982,"children":16984},{"className":16983,"style":4865},[323],[16985],{"type":150,"tag":181,"props":16986,"children":16987},{"style":328},[16988,16992],{"type":150,"tag":181,"props":16989,"children":16991},{"className":16990,"style":334},[333],[],{"type":150,"tag":181,"props":16993,"children":16995},{"className":16994},[339,340,341,342],[16996],{"type":150,"tag":181,"props":16997,"children":16999},{"className":16998},[296,301,342],[17000],{"type":155,"value":9859},{"type":150,"tag":181,"props":17002,"children":17004},{"className":17003},[352],[17005],{"type":155,"value":355},{"type":150,"tag":181,"props":17007,"children":17009},{"className":17008},[318],[17010],{"type":150,"tag":181,"props":17011,"children":17013},{"className":17012,"style":363},[323],[17014],{"type":150,"tag":181,"props":17015,"children":17016},{},[],{"type":155,"value":17018}," 就不必取 ",{"type":150,"tag":181,"props":17020,"children":17022},{"className":17021},[184],[17023,17079],{"type":150,"tag":181,"props":17024,"children":17026},{"className":17025},[189],[17027],{"type":150,"tag":192,"props":17028,"children":17029},{"xmlns":194},[17030],{"type":150,"tag":197,"props":17031,"children":17032},{},[17033,17074],{"type":150,"tag":201,"props":17034,"children":17035},{},[17036,17040,17044,17055,17059,17070],{"type":150,"tag":209,"props":17037,"children":17038},{},[17039],{"type":155,"value":7913},{"type":150,"tag":221,"props":17041,"children":17042},{"stretchy":789},[17043],{"type":155,"value":792},{"type":150,"tag":205,"props":17045,"children":17046},{},[17047,17051],{"type":150,"tag":209,"props":17048,"children":17049},{},[17050],{"type":155,"value":578},{"type":150,"tag":209,"props":17052,"children":17053},{},[17054],{"type":155,"value":1698},{"type":150,"tag":221,"props":17056,"children":17057},{},[17058],{"type":155,"value":4962},{"type":150,"tag":205,"props":17060,"children":17061},{},[17062,17066],{"type":150,"tag":209,"props":17063,"children":17064},{},[17065],{"type":155,"value":578},{"type":150,"tag":209,"props":17067,"children":17068},{},[17069],{"type":155,"value":1737},{"type":150,"tag":221,"props":17071,"children":17072},{"stretchy":789},[17073],{"type":155,"value":802},{"type":150,"tag":271,"props":17075,"children":17076},{"encoding":273},[17077],{"type":155,"value":17078},"g(d_k + d_v)",{"type":150,"tag":181,"props":17080,"children":17082},{"className":17081,"ariaHidden":223},[280],[17083,17171],{"type":150,"tag":181,"props":17084,"children":17086},{"className":17085},[285],[17087,17091,17096,17101,17158,17162,17167],{"type":150,"tag":181,"props":17088,"children":17090},{"className":17089,"style":819},[290],[],{"type":150,"tag":181,"props":17092,"children":17094},{"className":17093,"style":2151},[296,301],[17095],{"type":155,"value":7913},{"type":150,"tag":181,"props":17097,"children":17099},{"className":17098},[824],[17100],{"type":155,"value":792},{"type":150,"tag":181,"props":17102,"children":17104},{"className":17103},[296],[17105,17110],{"type":150,"tag":181,"props":17106,"children":17108},{"className":17107},[296,301],[17109],{"type":155,"value":578},{"type":150,"tag":181,"props":17111,"children":17113},{"className":17112},[307],[17114],{"type":150,"tag":181,"props":17115,"children":17117},{"className":17116},[312,313],[17118,17147],{"type":150,"tag":181,"props":17119,"children":17121},{"className":17120},[318],[17122,17142],{"type":150,"tag":181,"props":17123,"children":17125},{"className":17124,"style":493},[323],[17126],{"type":150,"tag":181,"props":17127,"children":17128},{"style":328},[17129,17133],{"type":150,"tag":181,"props":17130,"children":17132},{"className":17131,"style":334},[333],[],{"type":150,"tag":181,"props":17134,"children":17136},{"className":17135},[339,340,341,342],[17137],{"type":150,"tag":181,"props":17138,"children":17140},{"className":17139,"style":2249},[296,301,342],[17141],{"type":155,"value":1698},{"type":150,"tag":181,"props":17143,"children":17145},{"className":17144},[352],[17146],{"type":155,"value":355},{"type":150,"tag":181,"props":17148,"children":17150},{"className":17149},[318],[17151],{"type":150,"tag":181,"props":17152,"children":17154},{"className":17153,"style":363},[323],[17155],{"type":150,"tag":181,"props":17156,"children":17157},{},[],{"type":150,"tag":181,"props":17159,"children":17161},{"className":17160,"style":4993},[377],[],{"type":150,"tag":181,"props":17163,"children":17165},{"className":17164},[4998],[17166],{"type":155,"value":4962},{"type":150,"tag":181,"props":17168,"children":17170},{"className":17169,"style":4993},[377],[],{"type":150,"tag":181,"props":17172,"children":17174},{"className":17173},[285],[17175,17179,17236],{"type":150,"tag":181,"props":17176,"children":17178},{"className":17177,"style":819},[290],[],{"type":150,"tag":181,"props":17180,"children":17182},{"className":17181},[296],[17183,17188],{"type":150,"tag":181,"props":17184,"children":17186},{"className":17185},[296,301],[17187],{"type":155,"value":578},{"type":150,"tag":181,"props":17189,"children":17191},{"className":17190},[307],[17192],{"type":150,"tag":181,"props":17193,"children":17195},{"className":17194},[312,313],[17196,17225],{"type":150,"tag":181,"props":17197,"children":17199},{"className":17198},[318],[17200,17220],{"type":150,"tag":181,"props":17201,"children":17203},{"className":17202,"style":4865},[323],[17204],{"type":150,"tag":181,"props":17205,"children":17206},{"style":328},[17207,17211],{"type":150,"tag":181,"props":17208,"children":17210},{"className":17209,"style":334},[333],[],{"type":150,"tag":181,"props":17212,"children":17214},{"className":17213},[339,340,341,342],[17215],{"type":150,"tag":181,"props":17216,"children":17218},{"className":17217,"style":2151},[296,301,342],[17219],{"type":155,"value":1737},{"type":150,"tag":181,"props":17221,"children":17223},{"className":17222},[352],[17224],{"type":155,"value":355},{"type":150,"tag":181,"props":17226,"children":17228},{"className":17227},[318],[17229],{"type":150,"tag":181,"props":17230,"children":17232},{"className":17231,"style":363},[323],[17233],{"type":150,"tag":181,"props":17234,"children":17235},{},[],{"type":150,"tag":181,"props":17237,"children":17239},{"className":17238},[835],[17240],{"type":155,"value":802},{"type":155,"value":17242},"，取更小的值就行。DeepSeek-V2 取了 512。",{"type":150,"tag":151,"props":17244,"children":17245},{},[17246],{"type":155,"value":17247},"这就是 MLA 的核心。低秩压缩为什么有效？因为不同 head 需要的信息本来就有大量重叠。下投影矩阵负责提取这些共同信息，上投影矩阵让各个 head 用各自的方式去使用它。这甚至可能带来一点正则化效果。它强制模型把冗余的共同信息压进同一个低维空间，而不是每个 head 各存一份。",{"type":150,"tag":151,"props":17249,"children":17250},{},[17251],{"type":155,"value":17252},"我一开始以为 MLA 只是\"把 KV 存小一点\"。后来才反应过来，它换掉的是一整套共享方式。存小只是结果，不是原因。",{"type":150,"tag":4677,"props":17254,"children":17256},{"title":17255},"数值精度",[17257,17530],{"type":150,"tag":151,"props":17258,"children":17259},{},[17260,17528],{"type":150,"tag":181,"props":17261,"children":17263},{"className":17262},[184],[17264,17337],{"type":150,"tag":181,"props":17265,"children":17267},{"className":17266},[189],[17268],{"type":150,"tag":192,"props":17269,"children":17270},{"xmlns":194},[17271],{"type":150,"tag":197,"props":17272,"children":17273},{},[17274,17333],{"type":150,"tag":201,"props":17275,"children":17276},{},[17277,17303],{"type":150,"tag":1046,"props":17278,"children":17279},{},[17280,17284,17288],{"type":150,"tag":209,"props":17281,"children":17282},{},[17283],{"type":155,"value":3265},{"type":150,"tag":209,"props":17285,"children":17286},{},[17287],{"type":155,"value":1667},{"type":150,"tag":201,"props":17289,"children":17290},{},[17291,17295,17299],{"type":150,"tag":221,"props":17292,"children":17293},{"stretchy":789},[17294],{"type":155,"value":792},{"type":150,"tag":209,"props":17296,"children":17297},{},[17298],{"type":155,"value":797},{"type":150,"tag":221,"props":17300,"children":17301},{"stretchy":789},[17302],{"type":155,"value":802},{"type":150,"tag":1046,"props":17304,"children":17305},{},[17306,17310,17314],{"type":150,"tag":209,"props":17307,"children":17308},{},[17309],{"type":155,"value":3265},{"type":150,"tag":209,"props":17311,"children":17312},{},[17313],{"type":155,"value":1698},{"type":150,"tag":201,"props":17315,"children":17316},{},[17317,17321,17325,17329],{"type":150,"tag":221,"props":17318,"children":17319},{"stretchy":789},[17320],{"type":155,"value":792},{"type":150,"tag":209,"props":17322,"children":17323},{},[17324],{"type":155,"value":797},{"type":150,"tag":221,"props":17326,"children":17327},{"stretchy":789},[17328],{"type":155,"value":802},{"type":150,"tag":209,"props":17330,"children":17331},{"mathvariant":1618},[17332],{"type":155,"value":1874},{"type":150,"tag":271,"props":17334,"children":17335},{"encoding":273},[17336],{"type":155,"value":15520},{"type":150,"tag":181,"props":17338,"children":17340},{"className":17339,"ariaHidden":223},[280],[17341],{"type":150,"tag":181,"props":17342,"children":17344},{"className":17343},[285],[17345,17349,17436],{"type":150,"tag":181,"props":17346,"children":17348},{"className":17347,"style":4022},[290],[],{"type":150,"tag":181,"props":17350,"children":17352},{"className":17351},[296],[17353,17358],{"type":150,"tag":181,"props":17354,"children":17356},{"className":17355,"style":3691},[296,301],[17357],{"type":155,"value":3265},{"type":150,"tag":181,"props":17359,"children":17361},{"className":17360},[307],[17362],{"type":150,"tag":181,"props":17363,"children":17365},{"className":17364},[312,313],[17366,17425],{"type":150,"tag":181,"props":17367,"children":17369},{"className":17368},[318],[17370,17420],{"type":150,"tag":181,"props":17371,"children":17373},{"className":17372,"style":1276},[323],[17374,17390],{"type":150,"tag":181,"props":17375,"children":17376},{"style":15560},[17377,17381],{"type":150,"tag":181,"props":17378,"children":17380},{"className":17379,"style":334},[333],[],{"type":150,"tag":181,"props":17382,"children":17384},{"className":17383},[339,340,341,342],[17385],{"type":150,"tag":181,"props":17386,"children":17388},{"className":17387,"style":2151},[296,301,342],[17389],{"type":155,"value":1667},{"type":150,"tag":181,"props":17391,"children":17392},{"style":1297},[17393,17397],{"type":150,"tag":181,"props":17394,"children":17396},{"className":17395,"style":334},[333],[],{"type":150,"tag":181,"props":17398,"children":17400},{"className":17399},[339,340,341,342],[17401],{"type":150,"tag":181,"props":17402,"children":17404},{"className":17403},[296,342],[17405,17410,17415],{"type":150,"tag":181,"props":17406,"children":17408},{"className":17407},[824,342],[17409],{"type":155,"value":792},{"type":150,"tag":181,"props":17411,"children":17413},{"className":17412},[296,301,342],[17414],{"type":155,"value":797},{"type":150,"tag":181,"props":17416,"children":17418},{"className":17417},[835,342],[17419],{"type":155,"value":802},{"type":150,"tag":181,"props":17421,"children":17423},{"className":17422},[352],[17424],{"type":155,"value":355},{"type":150,"tag":181,"props":17426,"children":17428},{"className":17427},[318],[17429],{"type":150,"tag":181,"props":17430,"children":17432},{"className":17431,"style":15617},[323],[17433],{"type":150,"tag":181,"props":17434,"children":17435},{},[],{"type":150,"tag":181,"props":17437,"children":17439},{"className":17438},[296],[17440,17445],{"type":150,"tag":181,"props":17441,"children":17443},{"className":17442,"style":3691},[296,301],[17444],{"type":155,"value":3265},{"type":150,"tag":181,"props":17446,"children":17448},{"className":17447},[307],[17449],{"type":150,"tag":181,"props":17450,"children":17452},{"className":17451},[312,313],[17453,17517],{"type":150,"tag":181,"props":17454,"children":17456},{"className":17455},[318],[17457,17512],{"type":150,"tag":181,"props":17458,"children":17460},{"className":17459,"style":1276},[323],[17461,17477],{"type":150,"tag":181,"props":17462,"children":17463},{"style":4108},[17464,17468],{"type":150,"tag":181,"props":17465,"children":17467},{"className":17466,"style":334},[333],[],{"type":150,"tag":181,"props":17469,"children":17471},{"className":17470},[339,340,341,342],[17472],{"type":150,"tag":181,"props":17473,"children":17475},{"className":17474,"style":2249},[296,301,342],[17476],{"type":155,"value":1698},{"type":150,"tag":181,"props":17478,"children":17479},{"style":1297},[17480,17484],{"type":150,"tag":181,"props":17481,"children":17483},{"className":17482,"style":334},[333],[],{"type":150,"tag":181,"props":17485,"children":17487},{"className":17486},[339,340,341,342],[17488],{"type":150,"tag":181,"props":17489,"children":17491},{"className":17490},[296,342],[17492,17497,17502,17507],{"type":150,"tag":181,"props":17493,"children":17495},{"className":17494},[824,342],[17496],{"type":155,"value":792},{"type":150,"tag":181,"props":17498,"children":17500},{"className":17499},[296,301,342],[17501],{"type":155,"value":797},{"type":150,"tag":181,"props":17503,"children":17505},{"className":17504},[835,342],[17506],{"type":155,"value":802},{"type":150,"tag":181,"props":17508,"children":17510},{"className":17509},[296,342],[17511],{"type":155,"value":1874},{"type":150,"tag":181,"props":17513,"children":17515},{"className":17514},[352],[17516],{"type":155,"value":355},{"type":150,"tag":181,"props":17518,"children":17520},{"className":17519},[318],[17521],{"type":150,"tag":181,"props":17522,"children":17524},{"className":17523,"style":4165},[323],[17525],{"type":150,"tag":181,"props":17526,"children":17527},{},[],{"type":155,"value":17529}," 合并成一个矩阵，理论上只在无限精度下成立。用 BF16 时变换后的精度损失往往挺明显，多层累积后可能放大到可观的程度。",{"type":150,"tag":151,"props":17531,"children":17532},{},[17533,17535,17890,17892,18247],{"type":155,"value":17534},"实际实现一般不按 ",{"type":150,"tag":181,"props":17536,"children":17538},{"className":17537},[184],[17539,17632],{"type":150,"tag":181,"props":17540,"children":17542},{"className":17541},[189],[17543],{"type":150,"tag":192,"props":17544,"children":17545},{"xmlns":194},[17546],{"type":150,"tag":197,"props":17547,"children":17548},{},[17549,17627],{"type":150,"tag":201,"props":17550,"children":17551},{},[17552,17563,17567,17593,17623],{"type":150,"tag":205,"props":17553,"children":17554},{},[17555,17559],{"type":150,"tag":209,"props":17556,"children":17557},{},[17558],{"type":155,"value":213},{"type":150,"tag":209,"props":17560,"children":17561},{},[17562],{"type":155,"value":1035},{"type":150,"tag":221,"props":17564,"children":17565},{"stretchy":789},[17566],{"type":155,"value":792},{"type":150,"tag":1046,"props":17568,"children":17569},{},[17570,17574,17578],{"type":150,"tag":209,"props":17571,"children":17572},{},[17573],{"type":155,"value":3265},{"type":150,"tag":209,"props":17575,"children":17576},{},[17577],{"type":155,"value":1667},{"type":150,"tag":201,"props":17579,"children":17580},{},[17581,17585,17589],{"type":150,"tag":221,"props":17582,"children":17583},{"stretchy":789},[17584],{"type":155,"value":792},{"type":150,"tag":209,"props":17586,"children":17587},{},[17588],{"type":155,"value":797},{"type":150,"tag":221,"props":17590,"children":17591},{"stretchy":789},[17592],{"type":155,"value":802},{"type":150,"tag":1046,"props":17594,"children":17595},{},[17596,17600,17604],{"type":150,"tag":209,"props":17597,"children":17598},{},[17599],{"type":155,"value":3265},{"type":150,"tag":209,"props":17601,"children":17602},{},[17603],{"type":155,"value":1698},{"type":150,"tag":201,"props":17605,"children":17606},{},[17607,17611,17615,17619],{"type":150,"tag":221,"props":17608,"children":17609},{"stretchy":789},[17610],{"type":155,"value":792},{"type":150,"tag":209,"props":17612,"children":17613},{},[17614],{"type":155,"value":797},{"type":150,"tag":221,"props":17616,"children":17617},{"stretchy":789},[17618],{"type":155,"value":802},{"type":150,"tag":209,"props":17620,"children":17621},{"mathvariant":1618},[17622],{"type":155,"value":1874},{"type":150,"tag":221,"props":17624,"children":17625},{"stretchy":789},[17626],{"type":155,"value":802},{"type":150,"tag":271,"props":17628,"children":17629},{"encoding":273},[17630],{"type":155,"value":17631},"x_t (W_q^{(s)} W_k^{(s)\\top})",{"type":150,"tag":181,"props":17633,"children":17635},{"className":17634,"ariaHidden":223},[280],[17636],{"type":150,"tag":181,"props":17637,"children":17639},{"className":17638},[285],[17640,17644,17701,17706,17793,17885],{"type":150,"tag":181,"props":17641,"children":17643},{"className":17642,"style":4022},[290],[],{"type":150,"tag":181,"props":17645,"children":17647},{"className":17646},[296],[17648,17653],{"type":150,"tag":181,"props":17649,"children":17651},{"className":17650},[296,301],[17652],{"type":155,"value":213},{"type":150,"tag":181,"props":17654,"children":17656},{"className":17655},[307],[17657],{"type":150,"tag":181,"props":17658,"children":17660},{"className":17659},[312,313],[17661,17690],{"type":150,"tag":181,"props":17662,"children":17664},{"className":17663},[318],[17665,17685],{"type":150,"tag":181,"props":17666,"children":17668},{"className":17667,"style":1191},[323],[17669],{"type":150,"tag":181,"props":17670,"children":17671},{"style":328},[17672,17676],{"type":150,"tag":181,"props":17673,"children":17675},{"className":17674,"style":334},[333],[],{"type":150,"tag":181,"props":17677,"children":17679},{"className":17678},[339,340,341,342],[17680],{"type":150,"tag":181,"props":17681,"children":17683},{"className":17682},[296,301,342],[17684],{"type":155,"value":1035},{"type":150,"tag":181,"props":17686,"children":17688},{"className":17687},[352],[17689],{"type":155,"value":355},{"type":150,"tag":181,"props":17691,"children":17693},{"className":17692},[318],[17694],{"type":150,"tag":181,"props":17695,"children":17697},{"className":17696,"style":363},[323],[17698],{"type":150,"tag":181,"props":17699,"children":17700},{},[],{"type":150,"tag":181,"props":17702,"children":17704},{"className":17703},[824],[17705],{"type":155,"value":792},{"type":150,"tag":181,"props":17707,"children":17709},{"className":17708},[296],[17710,17715],{"type":150,"tag":181,"props":17711,"children":17713},{"className":17712,"style":3691},[296,301],[17714],{"type":155,"value":3265},{"type":150,"tag":181,"props":17716,"children":17718},{"className":17717},[307],[17719],{"type":150,"tag":181,"props":17720,"children":17722},{"className":17721},[312,313],[17723,17782],{"type":150,"tag":181,"props":17724,"children":17726},{"className":17725},[318],[17727,17777],{"type":150,"tag":181,"props":17728,"children":17730},{"className":17729,"style":1276},[323],[17731,17747],{"type":150,"tag":181,"props":17732,"children":17733},{"style":15560},[17734,17738],{"type":150,"tag":181,"props":17735,"children":17737},{"className":17736,"style":334},[333],[],{"type":150,"tag":181,"props":17739,"children":17741},{"className":17740},[339,340,341,342],[17742],{"type":150,"tag":181,"props":17743,"children":17745},{"className":17744,"style":2151},[296,301,342],[17746],{"type":155,"value":1667},{"type":150,"tag":181,"props":17748,"children":17749},{"style":1297},[17750,17754],{"type":150,"tag":181,"props":17751,"children":17753},{"className":17752,"style":334},[333],[],{"type":150,"tag":181,"props":17755,"children":17757},{"className":17756},[339,340,341,342],[17758],{"type":150,"tag":181,"props":17759,"children":17761},{"className":17760},[296,342],[17762,17767,17772],{"type":150,"tag":181,"props":17763,"children":17765},{"className":17764},[824,342],[17766],{"type":155,"value":792},{"type":150,"tag":181,"props":17768,"children":17770},{"className":17769},[296,301,342],[17771],{"type":155,"value":797},{"type":150,"tag":181,"props":17773,"children":17775},{"className":17774},[835,342],[17776],{"type":155,"value":802},{"type":150,"tag":181,"props":17778,"children":17780},{"className":17779},[352],[17781],{"type":155,"value":355},{"type":150,"tag":181,"props":17783,"children":17785},{"className":17784},[318],[17786],{"type":150,"tag":181,"props":17787,"children":17789},{"className":17788,"style":15617},[323],[17790],{"type":150,"tag":181,"props":17791,"children":17792},{},[],{"type":150,"tag":181,"props":17794,"children":17796},{"className":17795},[296],[17797,17802],{"type":150,"tag":181,"props":17798,"children":17800},{"className":17799,"style":3691},[296,301],[17801],{"type":155,"value":3265},{"type":150,"tag":181,"props":17803,"children":17805},{"className":17804},[307],[17806],{"type":150,"tag":181,"props":17807,"children":17809},{"className":17808},[312,313],[17810,17874],{"type":150,"tag":181,"props":17811,"children":17813},{"className":17812},[318],[17814,17869],{"type":150,"tag":181,"props":17815,"children":17817},{"className":17816,"style":1276},[323],[17818,17834],{"type":150,"tag":181,"props":17819,"children":17820},{"style":4108},[17821,17825],{"type":150,"tag":181,"props":17822,"children":17824},{"className":17823,"style":334},[333],[],{"type":150,"tag":181,"props":17826,"children":17828},{"className":17827},[339,340,341,342],[17829],{"type":150,"tag":181,"props":17830,"children":17832},{"className":17831,"style":2249},[296,301,342],[17833],{"type":155,"value":1698},{"type":150,"tag":181,"props":17835,"children":17836},{"style":1297},[17837,17841],{"type":150,"tag":181,"props":17838,"children":17840},{"className":17839,"style":334},[333],[],{"type":150,"tag":181,"props":17842,"children":17844},{"className":17843},[339,340,341,342],[17845],{"type":150,"tag":181,"props":17846,"children":17848},{"className":17847},[296,342],[17849,17854,17859,17864],{"type":150,"tag":181,"props":17850,"children":17852},{"className":17851},[824,342],[17853],{"type":155,"value":792},{"type":150,"tag":181,"props":17855,"children":17857},{"className":17856},[296,301,342],[17858],{"type":155,"value":797},{"type":150,"tag":181,"props":17860,"children":17862},{"className":17861},[835,342],[17863],{"type":155,"value":802},{"type":150,"tag":181,"props":17865,"children":17867},{"className":17866},[296,342],[17868],{"type":155,"value":1874},{"type":150,"tag":181,"props":17870,"children":17872},{"className":17871},[352],[17873],{"type":155,"value":355},{"type":150,"tag":181,"props":17875,"children":17877},{"className":17876},[318],[17878],{"type":150,"tag":181,"props":17879,"children":17881},{"className":17880,"style":4165},[323],[17882],{"type":150,"tag":181,"props":17883,"children":17884},{},[],{"type":150,"tag":181,"props":17886,"children":17888},{"className":17887},[835],[17889],{"type":155,"value":802},{"type":155,"value":17891}," 算 Q，而是按 ",{"type":150,"tag":181,"props":17893,"children":17895},{"className":17894},[184],[17896,17989],{"type":150,"tag":181,"props":17897,"children":17899},{"className":17898},[189],[17900],{"type":150,"tag":192,"props":17901,"children":17902},{"xmlns":194},[17903],{"type":150,"tag":197,"props":17904,"children":17905},{},[17906,17984],{"type":150,"tag":201,"props":17907,"children":17908},{},[17909,17913,17924,17950,17954],{"type":150,"tag":221,"props":17910,"children":17911},{"stretchy":789},[17912],{"type":155,"value":792},{"type":150,"tag":205,"props":17914,"children":17915},{},[17916,17920],{"type":150,"tag":209,"props":17917,"children":17918},{},[17919],{"type":155,"value":213},{"type":150,"tag":209,"props":17921,"children":17922},{},[17923],{"type":155,"value":1035},{"type":150,"tag":1046,"props":17925,"children":17926},{},[17927,17931,17935],{"type":150,"tag":209,"props":17928,"children":17929},{},[17930],{"type":155,"value":3265},{"type":150,"tag":209,"props":17932,"children":17933},{},[17934],{"type":155,"value":1667},{"type":150,"tag":201,"props":17936,"children":17937},{},[17938,17942,17946],{"type":150,"tag":221,"props":17939,"children":17940},{"stretchy":789},[17941],{"type":155,"value":792},{"type":150,"tag":209,"props":17943,"children":17944},{},[17945],{"type":155,"value":797},{"type":150,"tag":221,"props":17947,"children":17948},{"stretchy":789},[17949],{"type":155,"value":802},{"type":150,"tag":221,"props":17951,"children":17952},{"stretchy":789},[17953],{"type":155,"value":802},{"type":150,"tag":1046,"props":17955,"children":17956},{},[17957,17961,17965],{"type":150,"tag":209,"props":17958,"children":17959},{},[17960],{"type":155,"value":3265},{"type":150,"tag":209,"props":17962,"children":17963},{},[17964],{"type":155,"value":1698},{"type":150,"tag":201,"props":17966,"children":17967},{},[17968,17972,17976,17980],{"type":150,"tag":221,"props":17969,"children":17970},{"stretchy":789},[17971],{"type":155,"value":792},{"type":150,"tag":209,"props":17973,"children":17974},{},[17975],{"type":155,"value":797},{"type":150,"tag":221,"props":17977,"children":17978},{"stretchy":789},[17979],{"type":155,"value":802},{"type":150,"tag":209,"props":17981,"children":17982},{"mathvariant":1618},[17983],{"type":155,"value":1874},{"type":150,"tag":271,"props":17985,"children":17986},{"encoding":273},[17987],{"type":155,"value":17988},"(x_t W_q^{(s)}) W_k^{(s)\\top}",{"type":150,"tag":181,"props":17990,"children":17992},{"className":17991,"ariaHidden":223},[280],[17993],{"type":150,"tag":181,"props":17994,"children":17996},{"className":17995},[285],[17997,18001,18006,18063,18150,18155],{"type":150,"tag":181,"props":17998,"children":18000},{"className":17999,"style":4022},[290],[],{"type":150,"tag":181,"props":18002,"children":18004},{"className":18003},[824],[18005],{"type":155,"value":792},{"type":150,"tag":181,"props":18007,"children":18009},{"className":18008},[296],[18010,18015],{"type":150,"tag":181,"props":18011,"children":18013},{"className":18012},[296,301],[18014],{"type":155,"value":213},{"type":150,"tag":181,"props":18016,"children":18018},{"className":18017},[307],[18019],{"type":150,"tag":181,"props":18020,"children":18022},{"className":18021},[312,313],[18023,18052],{"type":150,"tag":181,"props":18024,"children":18026},{"className":18025},[318],[18027,18047],{"type":150,"tag":181,"props":18028,"children":18030},{"className":18029,"style":1191},[323],[18031],{"type":150,"tag":181,"props":18032,"children":18033},{"style":328},[18034,18038],{"type":150,"tag":181,"props":18035,"children":18037},{"className":18036,"style":334},[333],[],{"type":150,"tag":181,"props":18039,"children":18041},{"className":18040},[339,340,341,342],[18042],{"type":150,"tag":181,"props":18043,"children":18045},{"className":18044},[296,301,342],[18046],{"type":155,"value":1035},{"type":150,"tag":181,"props":18048,"children":18050},{"className":18049},[352],[18051],{"type":155,"value":355},{"type":150,"tag":181,"props":18053,"children":18055},{"className":18054},[318],[18056],{"type":150,"tag":181,"props":18057,"children":18059},{"className":18058,"style":363},[323],[18060],{"type":150,"tag":181,"props":18061,"children":18062},{},[],{"type":150,"tag":181,"props":18064,"children":18066},{"className":18065},[296],[18067,18072],{"type":150,"tag":181,"props":18068,"children":18070},{"className":18069,"style":3691},[296,301],[18071],{"type":155,"value":3265},{"type":150,"tag":181,"props":18073,"children":18075},{"className":18074},[307],[18076],{"type":150,"tag":181,"props":18077,"children":18079},{"className":18078},[312,313],[18080,18139],{"type":150,"tag":181,"props":18081,"children":18083},{"className":18082},[318],[18084,18134],{"type":150,"tag":181,"props":18085,"children":18087},{"className":18086,"style":1276},[323],[18088,18104],{"type":150,"tag":181,"props":18089,"children":18090},{"style":15560},[18091,18095],{"type":150,"tag":181,"props":18092,"children":18094},{"className":18093,"style":334},[333],[],{"type":150,"tag":181,"props":18096,"children":18098},{"className":18097},[339,340,341,342],[18099],{"type":150,"tag":181,"props":18100,"children":18102},{"className":18101,"style":2151},[296,301,342],[18103],{"type":155,"value":1667},{"type":150,"tag":181,"props":18105,"children":18106},{"style":1297},[18107,18111],{"type":150,"tag":181,"props":18108,"children":18110},{"className":18109,"style":334},[333],[],{"type":150,"tag":181,"props":18112,"children":18114},{"className":18113},[339,340,341,342],[18115],{"type":150,"tag":181,"props":18116,"children":18118},{"className":18117},[296,342],[18119,18124,18129],{"type":150,"tag":181,"props":18120,"children":18122},{"className":18121},[824,342],[18123],{"type":155,"value":792},{"type":150,"tag":181,"props":18125,"children":18127},{"className":18126},[296,301,342],[18128],{"type":155,"value":797},{"type":150,"tag":181,"props":18130,"children":18132},{"className":18131},[835,342],[18133],{"type":155,"value":802},{"type":150,"tag":181,"props":18135,"children":18137},{"className":18136},[352],[18138],{"type":155,"value":355},{"type":150,"tag":181,"props":18140,"children":18142},{"className":18141},[318],[18143],{"type":150,"tag":181,"props":18144,"children":18146},{"className":18145,"style":15617},[323],[18147],{"type":150,"tag":181,"props":18148,"children":18149},{},[],{"type":150,"tag":181,"props":18151,"children":18153},{"className":18152},[835],[18154],{"type":155,"value":802},{"type":150,"tag":181,"props":18156,"children":18158},{"className":18157},[296],[18159,18164],{"type":150,"tag":181,"props":18160,"children":18162},{"className":18161,"style":3691},[296,301],[18163],{"type":155,"value":3265},{"type":150,"tag":181,"props":18165,"children":18167},{"className":18166},[307],[18168],{"type":150,"tag":181,"props":18169,"children":18171},{"className":18170},[312,313],[18172,18236],{"type":150,"tag":181,"props":18173,"children":18175},{"className":18174},[318],[18176,18231],{"type":150,"tag":181,"props":18177,"children":18179},{"className":18178,"style":1276},[323],[18180,18196],{"type":150,"tag":181,"props":18181,"children":18182},{"style":4108},[18183,18187],{"type":150,"tag":181,"props":18184,"children":18186},{"className":18185,"style":334},[333],[],{"type":150,"tag":181,"props":18188,"children":18190},{"className":18189},[339,340,341,342],[18191],{"type":150,"tag":181,"props":18192,"children":18194},{"className":18193,"style":2249},[296,301,342],[18195],{"type":155,"value":1698},{"type":150,"tag":181,"props":18197,"children":18198},{"style":1297},[18199,18203],{"type":150,"tag":181,"props":18200,"children":18202},{"className":18201,"style":334},[333],[],{"type":150,"tag":181,"props":18204,"children":18206},{"className":18205},[339,340,341,342],[18207],{"type":150,"tag":181,"props":18208,"children":18210},{"className":18209},[296,342],[18211,18216,18221,18226],{"type":150,"tag":181,"props":18212,"children":18214},{"className":18213},[824,342],[18215],{"type":155,"value":792},{"type":150,"tag":181,"props":18217,"children":18219},{"className":18218},[296,301,342],[18220],{"type":155,"value":797},{"type":150,"tag":181,"props":18222,"children":18224},{"className":18223},[835,342],[18225],{"type":155,"value":802},{"type":150,"tag":181,"props":18227,"children":18229},{"className":18228},[296,342],[18230],{"type":155,"value":1874},{"type":150,"tag":181,"props":18232,"children":18234},{"className":18233},[352],[18235],{"type":155,"value":355},{"type":150,"tag":181,"props":18237,"children":18239},{"className":18238},[318],[18240],{"type":150,"tag":181,"props":18241,"children":18243},{"className":18242,"style":4165},[323],[18244],{"type":150,"tag":181,"props":18245,"children":18246},{},[],{"type":155,"value":18248}," 算。虽然是串行的，但在低秩假设下计算量更少，理论精度损失也更小。",{"type":150,"tag":151,"props":18250,"children":18251},{},[18252,18254,18259,18261,18529],{"type":155,"value":18253},"不过别急。到这里 MLA 有个绕不开的缺陷：",{"type":150,"tag":158,"props":18255,"children":18256},{},[18257],{"type":155,"value":18258},"不兼容 RoPE",{"type":155,"value":18260},"。前面那步恒等变换能成立，关键在于 ",{"type":150,"tag":181,"props":18262,"children":18264},{"className":18263},[184],[18265,18338],{"type":150,"tag":181,"props":18266,"children":18268},{"className":18267},[189],[18269],{"type":150,"tag":192,"props":18270,"children":18271},{"xmlns":194},[18272],{"type":150,"tag":197,"props":18273,"children":18274},{},[18275,18334],{"type":150,"tag":201,"props":18276,"children":18277},{},[18278,18304],{"type":150,"tag":1046,"props":18279,"children":18280},{},[18281,18285,18289],{"type":150,"tag":209,"props":18282,"children":18283},{},[18284],{"type":155,"value":3265},{"type":150,"tag":209,"props":18286,"children":18287},{},[18288],{"type":155,"value":1667},{"type":150,"tag":201,"props":18290,"children":18291},{},[18292,18296,18300],{"type":150,"tag":221,"props":18293,"children":18294},{"stretchy":789},[18295],{"type":155,"value":792},{"type":150,"tag":209,"props":18297,"children":18298},{},[18299],{"type":155,"value":797},{"type":150,"tag":221,"props":18301,"children":18302},{"stretchy":789},[18303],{"type":155,"value":802},{"type":150,"tag":1046,"props":18305,"children":18306},{},[18307,18311,18315],{"type":150,"tag":209,"props":18308,"children":18309},{},[18310],{"type":155,"value":3265},{"type":150,"tag":209,"props":18312,"children":18313},{},[18314],{"type":155,"value":1698},{"type":150,"tag":201,"props":18316,"children":18317},{},[18318,18322,18326,18330],{"type":150,"tag":221,"props":18319,"children":18320},{"stretchy":789},[18321],{"type":155,"value":792},{"type":150,"tag":209,"props":18323,"children":18324},{},[18325],{"type":155,"value":797},{"type":150,"tag":221,"props":18327,"children":18328},{"stretchy":789},[18329],{"type":155,"value":802},{"type":150,"tag":209,"props":18331,"children":18332},{"mathvariant":1618},[18333],{"type":155,"value":1874},{"type":150,"tag":271,"props":18335,"children":18336},{"encoding":273},[18337],{"type":155,"value":15520},{"type":150,"tag":181,"props":18339,"children":18341},{"className":18340,"ariaHidden":223},[280],[18342],{"type":150,"tag":181,"props":18343,"children":18345},{"className":18344},[285],[18346,18350,18437],{"type":150,"tag":181,"props":18347,"children":18349},{"className":18348,"style":4022},[290],[],{"type":150,"tag":181,"props":18351,"children":18353},{"className":18352},[296],[18354,18359],{"type":150,"tag":181,"props":18355,"children":18357},{"className":18356,"style":3691},[296,301],[18358],{"type":155,"value":3265},{"type":150,"tag":181,"props":18360,"children":18362},{"className":18361},[307],[18363],{"type":150,"tag":181,"props":18364,"children":18366},{"className":18365},[312,313],[18367,18426],{"type":150,"tag":181,"props":18368,"children":18370},{"className":18369},[318],[18371,18421],{"type":150,"tag":181,"props":18372,"children":18374},{"className":18373,"style":1276},[323],[18375,18391],{"type":150,"tag":181,"props":18376,"children":18377},{"style":15560},[18378,18382],{"type":150,"tag":181,"props":18379,"children":18381},{"className":18380,"style":334},[333],[],{"type":150,"tag":181,"props":18383,"children":18385},{"className":18384},[339,340,341,342],[18386],{"type":150,"tag":181,"props":18387,"children":18389},{"className":18388,"style":2151},[296,301,342],[18390],{"type":155,"value":1667},{"type":150,"tag":181,"props":18392,"children":18393},{"style":1297},[18394,18398],{"type":150,"tag":181,"props":18395,"children":18397},{"className":18396,"style":334},[333],[],{"type":150,"tag":181,"props":18399,"children":18401},{"className":18400},[339,340,341,342],[18402],{"type":150,"tag":181,"props":18403,"children":18405},{"className":18404},[296,342],[18406,18411,18416],{"type":150,"tag":181,"props":18407,"children":18409},{"className":18408},[824,342],[18410],{"type":155,"value":792},{"type":150,"tag":181,"props":18412,"children":18414},{"className":18413},[296,301,342],[18415],{"type":155,"value":797},{"type":150,"tag":181,"props":18417,"children":18419},{"className":18418},[835,342],[18420],{"type":155,"value":802},{"type":150,"tag":181,"props":18422,"children":18424},{"className":18423},[352],[18425],{"type":155,"value":355},{"type":150,"tag":181,"props":18427,"children":18429},{"className":18428},[318],[18430],{"type":150,"tag":181,"props":18431,"children":18433},{"className":18432,"style":15617},[323],[18434],{"type":150,"tag":181,"props":18435,"children":18436},{},[],{"type":150,"tag":181,"props":18438,"children":18440},{"className":18439},[296],[18441,18446],{"type":150,"tag":181,"props":18442,"children":18444},{"className":18443,"style":3691},[296,301],[18445],{"type":155,"value":3265},{"type":150,"tag":181,"props":18447,"children":18449},{"className":18448},[307],[18450],{"type":150,"tag":181,"props":18451,"children":18453},{"className":18452},[312,313],[18454,18518],{"type":150,"tag":181,"props":18455,"children":18457},{"className":18456},[318],[18458,18513],{"type":150,"tag":181,"props":18459,"children":18461},{"className":18460,"style":1276},[323],[18462,18478],{"type":150,"tag":181,"props":18463,"children":18464},{"style":4108},[18465,18469],{"type":150,"tag":181,"props":18466,"children":18468},{"className":18467,"style":334},[333],[],{"type":150,"tag":181,"props":18470,"children":18472},{"className":18471},[339,340,341,342],[18473],{"type":150,"tag":181,"props":18474,"children":18476},{"className":18475,"style":2249},[296,301,342],[18477],{"type":155,"value":1698},{"type":150,"tag":181,"props":18479,"children":18480},{"style":1297},[18481,18485],{"type":150,"tag":181,"props":18482,"children":18484},{"className":18483,"style":334},[333],[],{"type":150,"tag":181,"props":18486,"children":18488},{"className":18487},[339,340,341,342],[18489],{"type":150,"tag":181,"props":18490,"children":18492},{"className":18491},[296,342],[18493,18498,18503,18508],{"type":150,"tag":181,"props":18494,"children":18496},{"className":18495},[824,342],[18497],{"type":155,"value":792},{"type":150,"tag":181,"props":18499,"children":18501},{"className":18500},[296,301,342],[18502],{"type":155,"value":797},{"type":150,"tag":181,"props":18504,"children":18506},{"className":18505},[835,342],[18507],{"type":155,"value":802},{"type":150,"tag":181,"props":18509,"children":18511},{"className":18510},[296,342],[18512],{"type":155,"value":1874},{"type":150,"tag":181,"props":18514,"children":18516},{"className":18515},[352],[18517],{"type":155,"value":355},{"type":150,"tag":181,"props":18519,"children":18521},{"className":18520},[318],[18522],{"type":150,"tag":181,"props":18523,"children":18525},{"className":18524,"style":4165},[323],[18526],{"type":150,"tag":181,"props":18527,"children":18528},{},[],{"type":155,"value":18530}," 是一个跟位置无关的固定矩阵。加上 RoPE 之后就不行了：",{"type":150,"tag":181,"props":18532,"children":18534},{"className":18533},[1003],[18535],{"type":150,"tag":181,"props":18536,"children":18538},{"className":18537},[184],[18539,18853],{"type":150,"tag":181,"props":18540,"children":18542},{"className":18541},[189],[18543],{"type":150,"tag":192,"props":18544,"children":18545},{"xmlns":194,"display":1015},[18546],{"type":150,"tag":197,"props":18547,"children":18548},{},[18549,18848],{"type":150,"tag":201,"props":18550,"children":18551},{},[18552,18578,18608,18612,18616,18627,18653,18664,18668,18672,18683,18709,18720,18731,18735,18746,18750,18776,18799,18829,18833],{"type":150,"tag":1046,"props":18553,"children":18554},{},[18555,18559,18563],{"type":150,"tag":209,"props":18556,"children":18557},{},[18558],{"type":155,"value":1667},{"type":150,"tag":209,"props":18560,"children":18561},{},[18562],{"type":155,"value":1035},{"type":150,"tag":201,"props":18564,"children":18565},{},[18566,18570,18574],{"type":150,"tag":221,"props":18567,"children":18568},{"stretchy":789},[18569],{"type":155,"value":792},{"type":150,"tag":209,"props":18571,"children":18572},{},[18573],{"type":155,"value":797},{"type":150,"tag":221,"props":18575,"children":18576},{"stretchy":789},[18577],{"type":155,"value":802},{"type":150,"tag":1046,"props":18579,"children":18580},{},[18581,18585,18589],{"type":150,"tag":209,"props":18582,"children":18583},{},[18584],{"type":155,"value":1698},{"type":150,"tag":209,"props":18586,"children":18587},{},[18588],{"type":155,"value":558},{"type":150,"tag":201,"props":18590,"children":18591},{},[18592,18596,18600,18604],{"type":150,"tag":221,"props":18593,"children":18594},{"stretchy":789},[18595],{"type":155,"value":792},{"type":150,"tag":209,"props":18597,"children":18598},{},[18599],{"type":155,"value":797},{"type":150,"tag":221,"props":18601,"children":18602},{"stretchy":789},[18603],{"type":155,"value":802},{"type":150,"tag":209,"props":18605,"children":18606},{"mathvariant":1618},[18607],{"type":155,"value":1874},{"type":150,"tag":221,"props":18609,"children":18610},{},[18611],{"type":155,"value":865},{"type":150,"tag":221,"props":18613,"children":18614},{"stretchy":789},[18615],{"type":155,"value":792},{"type":150,"tag":205,"props":18617,"children":18618},{},[18619,18623],{"type":150,"tag":209,"props":18620,"children":18621},{},[18622],{"type":155,"value":213},{"type":150,"tag":209,"props":18624,"children":18625},{},[18626],{"type":155,"value":1035},{"type":150,"tag":1046,"props":18628,"children":18629},{},[18630,18634,18638],{"type":150,"tag":209,"props":18631,"children":18632},{},[18633],{"type":155,"value":3265},{"type":150,"tag":209,"props":18635,"children":18636},{},[18637],{"type":155,"value":1667},{"type":150,"tag":201,"props":18639,"children":18640},{},[18641,18645,18649],{"type":150,"tag":221,"props":18642,"children":18643},{"stretchy":789},[18644],{"type":155,"value":792},{"type":150,"tag":209,"props":18646,"children":18647},{},[18648],{"type":155,"value":797},{"type":150,"tag":221,"props":18650,"children":18651},{"stretchy":789},[18652],{"type":155,"value":802},{"type":150,"tag":205,"props":18654,"children":18655},{},[18656,18660],{"type":150,"tag":209,"props":18657,"children":18658},{},[18659],{"type":155,"value":573},{"type":150,"tag":209,"props":18661,"children":18662},{},[18663],{"type":155,"value":1035},{"type":150,"tag":221,"props":18665,"children":18666},{"stretchy":789},[18667],{"type":155,"value":802},{"type":150,"tag":221,"props":18669,"children":18670},{"stretchy":789},[18671],{"type":155,"value":792},{"type":150,"tag":205,"props":18673,"children":18674},{},[18675,18679],{"type":150,"tag":209,"props":18676,"children":18677},{},[18678],{"type":155,"value":9859},{"type":150,"tag":209,"props":18680,"children":18681},{},[18682],{"type":155,"value":558},{"type":150,"tag":1046,"props":18684,"children":18685},{},[18686,18690,18694],{"type":150,"tag":209,"props":18687,"children":18688},{},[18689],{"type":155,"value":3265},{"type":150,"tag":209,"props":18691,"children":18692},{},[18693],{"type":155,"value":1698},{"type":150,"tag":201,"props":18695,"children":18696},{},[18697,18701,18705],{"type":150,"tag":221,"props":18698,"children":18699},{"stretchy":789},[18700],{"type":155,"value":792},{"type":150,"tag":209,"props":18702,"children":18703},{},[18704],{"type":155,"value":797},{"type":150,"tag":221,"props":18706,"children":18707},{"stretchy":789},[18708],{"type":155,"value":802},{"type":150,"tag":205,"props":18710,"children":18711},{},[18712,18716],{"type":150,"tag":209,"props":18713,"children":18714},{},[18715],{"type":155,"value":573},{"type":150,"tag":209,"props":18717,"children":18718},{},[18719],{"type":155,"value":558},{"type":150,"tag":565,"props":18721,"children":18722},{},[18723,18727],{"type":150,"tag":221,"props":18724,"children":18725},{"stretchy":789},[18726],{"type":155,"value":802},{"type":150,"tag":209,"props":18728,"children":18729},{"mathvariant":1618},[18730],{"type":155,"value":1874},{"type":150,"tag":221,"props":18732,"children":18733},{},[18734],{"type":155,"value":865},{"type":150,"tag":205,"props":18736,"children":18737},{},[18738,18742],{"type":150,"tag":209,"props":18739,"children":18740},{},[18741],{"type":155,"value":213},{"type":150,"tag":209,"props":18743,"children":18744},{},[18745],{"type":155,"value":1035},{"type":150,"tag":221,"props":18747,"children":18748},{"stretchy":789},[18749],{"type":155,"value":792},{"type":150,"tag":1046,"props":18751,"children":18752},{},[18753,18757,18761],{"type":150,"tag":209,"props":18754,"children":18755},{},[18756],{"type":155,"value":3265},{"type":150,"tag":209,"props":18758,"children":18759},{},[18760],{"type":155,"value":1667},{"type":150,"tag":201,"props":18762,"children":18763},{},[18764,18768,18772],{"type":150,"tag":221,"props":18765,"children":18766},{"stretchy":789},[18767],{"type":155,"value":792},{"type":150,"tag":209,"props":18769,"children":18770},{},[18771],{"type":155,"value":797},{"type":150,"tag":221,"props":18773,"children":18774},{"stretchy":789},[18775],{"type":155,"value":802},{"type":150,"tag":205,"props":18777,"children":18778},{},[18779,18783],{"type":150,"tag":209,"props":18780,"children":18781},{},[18782],{"type":155,"value":573},{"type":150,"tag":201,"props":18784,"children":18785},{},[18786,18790,18795],{"type":150,"tag":209,"props":18787,"children":18788},{},[18789],{"type":155,"value":1035},{"type":150,"tag":221,"props":18791,"children":18792},{},[18793],{"type":155,"value":18794},"−",{"type":150,"tag":209,"props":18796,"children":18797},{},[18798],{"type":155,"value":558},{"type":150,"tag":1046,"props":18800,"children":18801},{},[18802,18806,18810],{"type":150,"tag":209,"props":18803,"children":18804},{},[18805],{"type":155,"value":3265},{"type":150,"tag":209,"props":18807,"children":18808},{},[18809],{"type":155,"value":1698},{"type":150,"tag":201,"props":18811,"children":18812},{},[18813,18817,18821,18825],{"type":150,"tag":221,"props":18814,"children":18815},{"stretchy":789},[18816],{"type":155,"value":792},{"type":150,"tag":209,"props":18818,"children":18819},{},[18820],{"type":155,"value":797},{"type":150,"tag":221,"props":18822,"children":18823},{"stretchy":789},[18824],{"type":155,"value":802},{"type":150,"tag":209,"props":18826,"children":18827},{"mathvariant":1618},[18828],{"type":155,"value":1874},{"type":150,"tag":221,"props":18830,"children":18831},{"stretchy":789},[18832],{"type":155,"value":802},{"type":150,"tag":1046,"props":18834,"children":18835},{},[18836,18840,18844],{"type":150,"tag":209,"props":18837,"children":18838},{},[18839],{"type":155,"value":9859},{"type":150,"tag":209,"props":18841,"children":18842},{},[18843],{"type":155,"value":558},{"type":150,"tag":209,"props":18845,"children":18846},{"mathvariant":1618},[18847],{"type":155,"value":1874},{"type":150,"tag":271,"props":18849,"children":18850},{"encoding":273},[18851],{"type":155,"value":18852},"q_t^{(s)} k_i^{(s)\\top} = (x_t W_q^{(s)} R_t)(c_i W_k^{(s)} R_i)^\\top = x_t (W_q^{(s)} R_{t-i} W_k^{(s)\\top}) c_i^\\top",{"type":150,"tag":181,"props":18854,"children":18856},{"className":18855,"ariaHidden":223},[280],[18857,19057,19538],{"type":150,"tag":181,"props":18858,"children":18860},{"className":18859},[285],[18861,18865,18952,19044,19048,19053],{"type":150,"tag":181,"props":18862,"children":18864},{"className":18863,"style":3516},[290],[],{"type":150,"tag":181,"props":18866,"children":18868},{"className":18867},[296],[18869,18874],{"type":150,"tag":181,"props":18870,"children":18872},{"className":18871,"style":2151},[296,301],[18873],{"type":155,"value":1667},{"type":150,"tag":181,"props":18875,"children":18877},{"className":18876},[307],[18878],{"type":150,"tag":181,"props":18879,"children":18881},{"className":18880},[312,313],[18882,18941],{"type":150,"tag":181,"props":18883,"children":18885},{"className":18884},[318],[18886,18936],{"type":150,"tag":181,"props":18887,"children":18889},{"className":18888,"style":1276},[323],[18890,18906],{"type":150,"tag":181,"props":18891,"children":18892},{"style":2172},[18893,18897],{"type":150,"tag":181,"props":18894,"children":18896},{"className":18895,"style":334},[333],[],{"type":150,"tag":181,"props":18898,"children":18900},{"className":18899},[339,340,341,342],[18901],{"type":150,"tag":181,"props":18902,"children":18904},{"className":18903},[296,301,342],[18905],{"type":155,"value":1035},{"type":150,"tag":181,"props":18907,"children":18908},{"style":1297},[18909,18913],{"type":150,"tag":181,"props":18910,"children":18912},{"className":18911,"style":334},[333],[],{"type":150,"tag":181,"props":18914,"children":18916},{"className":18915},[339,340,341,342],[18917],{"type":150,"tag":181,"props":18918,"children":18920},{"className":18919},[296,342],[18921,18926,18931],{"type":150,"tag":181,"props":18922,"children":18924},{"className":18923},[824,342],[18925],{"type":155,"value":792},{"type":150,"tag":181,"props":18927,"children":18929},{"className":18928},[296,301,342],[18930],{"type":155,"value":797},{"type":150,"tag":181,"props":18932,"children":18934},{"className":18933},[835,342],[18935],{"type":155,"value":802},{"type":150,"tag":181,"props":18937,"children":18939},{"className":18938},[352],[18940],{"type":155,"value":355},{"type":150,"tag":181,"props":18942,"children":18944},{"className":18943},[318],[18945],{"type":150,"tag":181,"props":18946,"children":18948},{"className":18947,"style":1338},[323],[18949],{"type":150,"tag":181,"props":18950,"children":18951},{},[],{"type":150,"tag":181,"props":18953,"children":18955},{"className":18954},[296],[18956,18961],{"type":150,"tag":181,"props":18957,"children":18959},{"className":18958,"style":2249},[296,301],[18960],{"type":155,"value":1698},{"type":150,"tag":181,"props":18962,"children":18964},{"className":18963},[307],[18965],{"type":150,"tag":181,"props":18966,"children":18968},{"className":18967},[312,313],[18969,19033],{"type":150,"tag":181,"props":18970,"children":18972},{"className":18971},[318],[18973,19028],{"type":150,"tag":181,"props":18974,"children":18976},{"className":18975,"style":1276},[323],[18977,18993],{"type":150,"tag":181,"props":18978,"children":18979},{"style":2720},[18980,18984],{"type":150,"tag":181,"props":18981,"children":18983},{"className":18982,"style":334},[333],[],{"type":150,"tag":181,"props":18985,"children":18987},{"className":18986},[339,340,341,342],[18988],{"type":150,"tag":181,"props":18989,"children":18991},{"className":18990},[296,301,342],[18992],{"type":155,"value":558},{"type":150,"tag":181,"props":18994,"children":18995},{"style":1297},[18996,19000],{"type":150,"tag":181,"props":18997,"children":18999},{"className":18998,"style":334},[333],[],{"type":150,"tag":181,"props":19001,"children":19003},{"className":19002},[339,340,341,342],[19004],{"type":150,"tag":181,"props":19005,"children":19007},{"className":19006},[296,342],[19008,19013,19018,19023],{"type":150,"tag":181,"props":19009,"children":19011},{"className":19010},[824,342],[19012],{"type":155,"value":792},{"type":150,"tag":181,"props":19014,"children":19016},{"className":19015},[296,301,342],[19017],{"type":155,"value":797},{"type":150,"tag":181,"props":19019,"children":19021},{"className":19020},[835,342],[19022],{"type":155,"value":802},{"type":150,"tag":181,"props":19024,"children":19026},{"className":19025},[296,342],[19027],{"type":155,"value":1874},{"type":150,"tag":181,"props":19029,"children":19031},{"className":19030},[352],[19032],{"type":155,"value":355},{"type":150,"tag":181,"props":19034,"children":19036},{"className":19035},[318],[19037],{"type":150,"tag":181,"props":19038,"children":19040},{"className":19039,"style":2782},[323],[19041],{"type":150,"tag":181,"props":19042,"children":19043},{},[],{"type":150,"tag":181,"props":19045,"children":19047},{"className":19046,"style":658},[377],[],{"type":150,"tag":181,"props":19049,"children":19051},{"className":19050},[663],[19052],{"type":155,"value":865},{"type":150,"tag":181,"props":19054,"children":19056},{"className":19055,"style":658},[377],[],{"type":150,"tag":181,"props":19058,"children":19060},{"className":19059},[285],[19061,19065,19070,19127,19214,19273,19278,19283,19340,19427,19484,19525,19529,19534],{"type":150,"tag":181,"props":19062,"children":19064},{"className":19063,"style":14751},[290],[],{"type":150,"tag":181,"props":19066,"children":19068},{"className":19067},[824],[19069],{"type":155,"value":792},{"type":150,"tag":181,"props":19071,"children":19073},{"className":19072},[296],[19074,19079],{"type":150,"tag":181,"props":19075,"children":19077},{"className":19076},[296,301],[19078],{"type":155,"value":213},{"type":150,"tag":181,"props":19080,"children":19082},{"className":19081},[307],[19083],{"type":150,"tag":181,"props":19084,"children":19086},{"className":19085},[312,313],[19087,19116],{"type":150,"tag":181,"props":19088,"children":19090},{"className":19089},[318],[19091,19111],{"type":150,"tag":181,"props":19092,"children":19094},{"className":19093,"style":1191},[323],[19095],{"type":150,"tag":181,"props":19096,"children":19097},{"style":328},[19098,19102],{"type":150,"tag":181,"props":19099,"children":19101},{"className":19100,"style":334},[333],[],{"type":150,"tag":181,"props":19103,"children":19105},{"className":19104},[339,340,341,342],[19106],{"type":150,"tag":181,"props":19107,"children":19109},{"className":19108},[296,301,342],[19110],{"type":155,"value":1035},{"type":150,"tag":181,"props":19112,"children":19114},{"className":19113},[352],[19115],{"type":155,"value":355},{"type":150,"tag":181,"props":19117,"children":19119},{"className":19118},[318],[19120],{"type":150,"tag":181,"props":19121,"children":19123},{"className":19122,"style":363},[323],[19124],{"type":150,"tag":181,"props":19125,"children":19126},{},[],{"type":150,"tag":181,"props":19128,"children":19130},{"className":19129},[296],[19131,19136],{"type":150,"tag":181,"props":19132,"children":19134},{"className":19133,"style":3691},[296,301],[19135],{"type":155,"value":3265},{"type":150,"tag":181,"props":19137,"children":19139},{"className":19138},[307],[19140],{"type":150,"tag":181,"props":19141,"children":19143},{"className":19142},[312,313],[19144,19203],{"type":150,"tag":181,"props":19145,"children":19147},{"className":19146},[318],[19148,19198],{"type":150,"tag":181,"props":19149,"children":19151},{"className":19150,"style":3709},[323],[19152,19168],{"type":150,"tag":181,"props":19153,"children":19154},{"style":3713},[19155,19159],{"type":150,"tag":181,"props":19156,"children":19158},{"className":19157,"style":334},[333],[],{"type":150,"tag":181,"props":19160,"children":19162},{"className":19161},[339,340,341,342],[19163],{"type":150,"tag":181,"props":19164,"children":19166},{"className":19165,"style":2151},[296,301,342],[19167],{"type":155,"value":1667},{"type":150,"tag":181,"props":19169,"children":19170},{"style":3730},[19171,19175],{"type":150,"tag":181,"props":19172,"children":19174},{"className":19173,"style":334},[333],[],{"type":150,"tag":181,"props":19176,"children":19178},{"className":19177},[339,340,341,342],[19179],{"type":150,"tag":181,"props":19180,"children":19182},{"className":19181},[296,342],[19183,19188,19193],{"type":150,"tag":181,"props":19184,"children":19186},{"className":19185},[824,342],[19187],{"type":155,"value":792},{"type":150,"tag":181,"props":19189,"children":19191},{"className":19190},[296,301,342],[19192],{"type":155,"value":797},{"type":150,"tag":181,"props":19194,"children":19196},{"className":19195},[835,342],[19197],{"type":155,"value":802},{"type":150,"tag":181,"props":19199,"children":19201},{"className":19200},[352],[19202],{"type":155,"value":355},{"type":150,"tag":181,"props":19204,"children":19206},{"className":19205},[318],[19207],{"type":150,"tag":181,"props":19208,"children":19210},{"className":19209,"style":3771},[323],[19211],{"type":150,"tag":181,"props":19212,"children":19213},{},[],{"type":150,"tag":181,"props":19215,"children":19217},{"className":19216},[296],[19218,19224],{"type":150,"tag":181,"props":19219,"children":19222},{"className":19220,"style":19221},[296,301],"margin-right:0.0077em;",[19223],{"type":155,"value":573},{"type":150,"tag":181,"props":19225,"children":19227},{"className":19226},[307],[19228],{"type":150,"tag":181,"props":19229,"children":19231},{"className":19230},[312,313],[19232,19262],{"type":150,"tag":181,"props":19233,"children":19235},{"className":19234},[318],[19236,19257],{"type":150,"tag":181,"props":19237,"children":19239},{"className":19238,"style":1191},[323],[19240],{"type":150,"tag":181,"props":19241,"children":19243},{"style":19242},"top:-2.55em;margin-left:-0.0077em;margin-right:0.05em;",[19244,19248],{"type":150,"tag":181,"props":19245,"children":19247},{"className":19246,"style":334},[333],[],{"type":150,"tag":181,"props":19249,"children":19251},{"className":19250},[339,340,341,342],[19252],{"type":150,"tag":181,"props":19253,"children":19255},{"className":19254},[296,301,342],[19256],{"type":155,"value":1035},{"type":150,"tag":181,"props":19258,"children":19260},{"className":19259},[352],[19261],{"type":155,"value":355},{"type":150,"tag":181,"props":19263,"children":19265},{"className":19264},[318],[19266],{"type":150,"tag":181,"props":19267,"children":19269},{"className":19268,"style":363},[323],[19270],{"type":150,"tag":181,"props":19271,"children":19272},{},[],{"type":150,"tag":181,"props":19274,"children":19276},{"className":19275},[835],[19277],{"type":155,"value":802},{"type":150,"tag":181,"props":19279,"children":19281},{"className":19280},[824],[19282],{"type":155,"value":792},{"type":150,"tag":181,"props":19284,"children":19286},{"className":19285},[296],[19287,19292],{"type":150,"tag":181,"props":19288,"children":19290},{"className":19289},[296,301],[19291],{"type":155,"value":9859},{"type":150,"tag":181,"props":19293,"children":19295},{"className":19294},[307],[19296],{"type":150,"tag":181,"props":19297,"children":19299},{"className":19298},[312,313],[19300,19329],{"type":150,"tag":181,"props":19301,"children":19303},{"className":19302},[318],[19304,19324],{"type":150,"tag":181,"props":19305,"children":19307},{"className":19306,"style":621},[323],[19308],{"type":150,"tag":181,"props":19309,"children":19310},{"style":328},[19311,19315],{"type":150,"tag":181,"props":19312,"children":19314},{"className":19313,"style":334},[333],[],{"type":150,"tag":181,"props":19316,"children":19318},{"className":19317},[339,340,341,342],[19319],{"type":150,"tag":181,"props":19320,"children":19322},{"className":19321},[296,301,342],[19323],{"type":155,"value":558},{"type":150,"tag":181,"props":19325,"children":19327},{"className":19326},[352],[19328],{"type":155,"value":355},{"type":150,"tag":181,"props":19330,"children":19332},{"className":19331},[318],[19333],{"type":150,"tag":181,"props":19334,"children":19336},{"className":19335,"style":363},[323],[19337],{"type":150,"tag":181,"props":19338,"children":19339},{},[],{"type":150,"tag":181,"props":19341,"children":19343},{"className":19342},[296],[19344,19349],{"type":150,"tag":181,"props":19345,"children":19347},{"className":19346,"style":3691},[296,301],[19348],{"type":155,"value":3265},{"type":150,"tag":181,"props":19350,"children":19352},{"className":19351},[307],[19353],{"type":150,"tag":181,"props":19354,"children":19356},{"className":19355},[312,313],[19357,19416],{"type":150,"tag":181,"props":19358,"children":19360},{"className":19359},[318],[19361,19411],{"type":150,"tag":181,"props":19362,"children":19364},{"className":19363,"style":1276},[323],[19365,19381],{"type":150,"tag":181,"props":19366,"children":19367},{"style":4108},[19368,19372],{"type":150,"tag":181,"props":19369,"children":19371},{"className":19370,"style":334},[333],[],{"type":150,"tag":181,"props":19373,"children":19375},{"className":19374},[339,340,341,342],[19376],{"type":150,"tag":181,"props":19377,"children":19379},{"className":19378,"style":2249},[296,301,342],[19380],{"type":155,"value":1698},{"type":150,"tag":181,"props":19382,"children":19383},{"style":1297},[19384,19388],{"type":150,"tag":181,"props":19385,"children":19387},{"className":19386,"style":334},[333],[],{"type":150,"tag":181,"props":19389,"children":19391},{"className":19390},[339,340,341,342],[19392],{"type":150,"tag":181,"props":19393,"children":19395},{"className":19394},[296,342],[19396,19401,19406],{"type":150,"tag":181,"props":19397,"children":19399},{"className":19398},[824,342],[19400],{"type":155,"value":792},{"type":150,"tag":181,"props":19402,"children":19404},{"className":19403},[296,301,342],[19405],{"type":155,"value":797},{"type":150,"tag":181,"props":19407,"children":19409},{"className":19408},[835,342],[19410],{"type":155,"value":802},{"type":150,"tag":181,"props":19412,"children":19414},{"className":19413},[352],[19415],{"type":155,"value":355},{"type":150,"tag":181,"props":19417,"children":19419},{"className":19418},[318],[19420],{"type":150,"tag":181,"props":19421,"children":19423},{"className":19422,"style":4165},[323],[19424],{"type":150,"tag":181,"props":19425,"children":19426},{},[],{"type":150,"tag":181,"props":19428,"children":19430},{"className":19429},[296],[19431,19436],{"type":150,"tag":181,"props":19432,"children":19434},{"className":19433,"style":19221},[296,301],[19435],{"type":155,"value":573},{"type":150,"tag":181,"props":19437,"children":19439},{"className":19438},[307],[19440],{"type":150,"tag":181,"props":19441,"children":19443},{"className":19442},[312,313],[19444,19473],{"type":150,"tag":181,"props":19445,"children":19447},{"className":19446},[318],[19448,19468],{"type":150,"tag":181,"props":19449,"children":19451},{"className":19450,"style":621},[323],[19452],{"type":150,"tag":181,"props":19453,"children":19454},{"style":19242},[19455,19459],{"type":150,"tag":181,"props":19456,"children":19458},{"className":19457,"style":334},[333],[],{"type":150,"tag":181,"props":19460,"children":19462},{"className":19461},[339,340,341,342],[19463],{"type":150,"tag":181,"props":19464,"children":19466},{"className":19465},[296,301,342],[19467],{"type":155,"value":558},{"type":150,"tag":181,"props":19469,"children":19471},{"className":19470},[352],[19472],{"type":155,"value":355},{"type":150,"tag":181,"props":19474,"children":19476},{"className":19475},[318],[19477],{"type":150,"tag":181,"props":19478,"children":19480},{"className":19479,"style":363},[323],[19481],{"type":150,"tag":181,"props":19482,"children":19483},{},[],{"type":150,"tag":181,"props":19485,"children":19487},{"className":19486},[835],[19488,19493],{"type":150,"tag":181,"props":19489,"children":19491},{"className":19490},[835],[19492],{"type":155,"value":802},{"type":150,"tag":181,"props":19494,"children":19496},{"className":19495},[307],[19497],{"type":150,"tag":181,"props":19498,"children":19500},{"className":19499},[312],[19501],{"type":150,"tag":181,"props":19502,"children":19504},{"className":19503},[318],[19505],{"type":150,"tag":181,"props":19506,"children":19508},{"className":19507,"style":3821},[323],[19509],{"type":150,"tag":181,"props":19510,"children":19511},{"style":3730},[19512,19516],{"type":150,"tag":181,"props":19513,"children":19515},{"className":19514,"style":334},[333],[],{"type":150,"tag":181,"props":19517,"children":19519},{"className":19518},[339,340,341,342],[19520],{"type":150,"tag":181,"props":19521,"children":19523},{"className":19522},[296,342],[19524],{"type":155,"value":1874},{"type":150,"tag":181,"props":19526,"children":19528},{"className":19527,"style":658},[377],[],{"type":150,"tag":181,"props":19530,"children":19532},{"className":19531},[663],[19533],{"type":155,"value":865},{"type":150,"tag":181,"props":19535,"children":19537},{"className":19536,"style":658},[377],[],{"type":150,"tag":181,"props":19539,"children":19541},{"className":19540},[285],[19542,19546,19603,19608,19695,19767,19859,19864],{"type":150,"tag":181,"props":19543,"children":19545},{"className":19544,"style":14751},[290],[],{"type":150,"tag":181,"props":19547,"children":19549},{"className":19548},[296],[19550,19555],{"type":150,"tag":181,"props":19551,"children":19553},{"className":19552},[296,301],[19554],{"type":155,"value":213},{"type":150,"tag":181,"props":19556,"children":19558},{"className":19557},[307],[19559],{"type":150,"tag":181,"props":19560,"children":19562},{"className":19561},[312,313],[19563,19592],{"type":150,"tag":181,"props":19564,"children":19566},{"className":19565},[318],[19567,19587],{"type":150,"tag":181,"props":19568,"children":19570},{"className":19569,"style":1191},[323],[19571],{"type":150,"tag":181,"props":19572,"children":19573},{"style":328},[19574,19578],{"type":150,"tag":181,"props":19575,"children":19577},{"className":19576,"style":334},[333],[],{"type":150,"tag":181,"props":19579,"children":19581},{"className":19580},[339,340,341,342],[19582],{"type":150,"tag":181,"props":19583,"children":19585},{"className":19584},[296,301,342],[19586],{"type":155,"value":1035},{"type":150,"tag":181,"props":19588,"children":19590},{"className":19589},[352],[19591],{"type":155,"value":355},{"type":150,"tag":181,"props":19593,"children":19595},{"className":19594},[318],[19596],{"type":150,"tag":181,"props":19597,"children":19599},{"className":19598,"style":363},[323],[19600],{"type":150,"tag":181,"props":19601,"children":19602},{},[],{"type":150,"tag":181,"props":19604,"children":19606},{"className":19605},[824],[19607],{"type":155,"value":792},{"type":150,"tag":181,"props":19609,"children":19611},{"className":19610},[296],[19612,19617],{"type":150,"tag":181,"props":19613,"children":19615},{"className":19614,"style":3691},[296,301],[19616],{"type":155,"value":3265},{"type":150,"tag":181,"props":19618,"children":19620},{"className":19619},[307],[19621],{"type":150,"tag":181,"props":19622,"children":19624},{"className":19623},[312,313],[19625,19684],{"type":150,"tag":181,"props":19626,"children":19628},{"className":19627},[318],[19629,19679],{"type":150,"tag":181,"props":19630,"children":19632},{"className":19631,"style":3709},[323],[19633,19649],{"type":150,"tag":181,"props":19634,"children":19635},{"style":3713},[19636,19640],{"type":150,"tag":181,"props":19637,"children":19639},{"className":19638,"style":334},[333],[],{"type":150,"tag":181,"props":19641,"children":19643},{"className":19642},[339,340,341,342],[19644],{"type":150,"tag":181,"props":19645,"children":19647},{"className":19646,"style":2151},[296,301,342],[19648],{"type":155,"value":1667},{"type":150,"tag":181,"props":19650,"children":19651},{"style":3730},[19652,19656],{"type":150,"tag":181,"props":19653,"children":19655},{"className":19654,"style":334},[333],[],{"type":150,"tag":181,"props":19657,"children":19659},{"className":19658},[339,340,341,342],[19660],{"type":150,"tag":181,"props":19661,"children":19663},{"className":19662},[296,342],[19664,19669,19674],{"type":150,"tag":181,"props":19665,"children":19667},{"className":19666},[824,342],[19668],{"type":155,"value":792},{"type":150,"tag":181,"props":19670,"children":19672},{"className":19671},[296,301,342],[19673],{"type":155,"value":797},{"type":150,"tag":181,"props":19675,"children":19677},{"className":19676},[835,342],[19678],{"type":155,"value":802},{"type":150,"tag":181,"props":19680,"children":19682},{"className":19681},[352],[19683],{"type":155,"value":355},{"type":150,"tag":181,"props":19685,"children":19687},{"className":19686},[318],[19688],{"type":150,"tag":181,"props":19689,"children":19691},{"className":19690,"style":3771},[323],[19692],{"type":150,"tag":181,"props":19693,"children":19694},{},[],{"type":150,"tag":181,"props":19696,"children":19698},{"className":19697},[296],[19699,19704],{"type":150,"tag":181,"props":19700,"children":19702},{"className":19701,"style":19221},[296,301],[19703],{"type":155,"value":573},{"type":150,"tag":181,"props":19705,"children":19707},{"className":19706},[307],[19708],{"type":150,"tag":181,"props":19709,"children":19711},{"className":19710},[312,313],[19712,19755],{"type":150,"tag":181,"props":19713,"children":19715},{"className":19714},[318],[19716,19750],{"type":150,"tag":181,"props":19717,"children":19719},{"className":19718,"style":621},[323],[19720],{"type":150,"tag":181,"props":19721,"children":19722},{"style":19242},[19723,19727],{"type":150,"tag":181,"props":19724,"children":19726},{"className":19725,"style":334},[333],[],{"type":150,"tag":181,"props":19728,"children":19730},{"className":19729},[339,340,341,342],[19731],{"type":150,"tag":181,"props":19732,"children":19734},{"className":19733},[296,342],[19735,19740,19745],{"type":150,"tag":181,"props":19736,"children":19738},{"className":19737},[296,301,342],[19739],{"type":155,"value":1035},{"type":150,"tag":181,"props":19741,"children":19743},{"className":19742},[4998,342],[19744],{"type":155,"value":18794},{"type":150,"tag":181,"props":19746,"children":19748},{"className":19747},[296,301,342],[19749],{"type":155,"value":558},{"type":150,"tag":181,"props":19751,"children":19753},{"className":19752},[352],[19754],{"type":155,"value":355},{"type":150,"tag":181,"props":19756,"children":19758},{"className":19757},[318],[19759],{"type":150,"tag":181,"props":19760,"children":19763},{"className":19761,"style":19762},[323],"height:0.2083em;",[19764],{"type":150,"tag":181,"props":19765,"children":19766},{},[],{"type":150,"tag":181,"props":19768,"children":19770},{"className":19769},[296],[19771,19776],{"type":150,"tag":181,"props":19772,"children":19774},{"className":19773,"style":3691},[296,301],[19775],{"type":155,"value":3265},{"type":150,"tag":181,"props":19777,"children":19779},{"className":19778},[307],[19780],{"type":150,"tag":181,"props":19781,"children":19783},{"className":19782},[312,313],[19784,19848],{"type":150,"tag":181,"props":19785,"children":19787},{"className":19786},[318],[19788,19843],{"type":150,"tag":181,"props":19789,"children":19791},{"className":19790,"style":1276},[323],[19792,19808],{"type":150,"tag":181,"props":19793,"children":19794},{"style":4108},[19795,19799],{"type":150,"tag":181,"props":19796,"children":19798},{"className":19797,"style":334},[333],[],{"type":150,"tag":181,"props":19800,"children":19802},{"className":19801},[339,340,341,342],[19803],{"type":150,"tag":181,"props":19804,"children":19806},{"className":19805,"style":2249},[296,301,342],[19807],{"type":155,"value":1698},{"type":150,"tag":181,"props":19809,"children":19810},{"style":1297},[19811,19815],{"type":150,"tag":181,"props":19812,"children":19814},{"className":19813,"style":334},[333],[],{"type":150,"tag":181,"props":19816,"children":19818},{"className":19817},[339,340,341,342],[19819],{"type":150,"tag":181,"props":19820,"children":19822},{"className":19821},[296,342],[19823,19828,19833,19838],{"type":150,"tag":181,"props":19824,"children":19826},{"className":19825},[824,342],[19827],{"type":155,"value":792},{"type":150,"tag":181,"props":19829,"children":19831},{"className":19830},[296,301,342],[19832],{"type":155,"value":797},{"type":150,"tag":181,"props":19834,"children":19836},{"className":19835},[835,342],[19837],{"type":155,"value":802},{"type":150,"tag":181,"props":19839,"children":19841},{"className":19840},[296,342],[19842],{"type":155,"value":1874},{"type":150,"tag":181,"props":19844,"children":19846},{"className":19845},[352],[19847],{"type":155,"value":355},{"type":150,"tag":181,"props":19849,"children":19851},{"className":19850},[318],[19852],{"type":150,"tag":181,"props":19853,"children":19855},{"className":19854,"style":4165},[323],[19856],{"type":150,"tag":181,"props":19857,"children":19858},{},[],{"type":150,"tag":181,"props":19860,"children":19862},{"className":19861},[835],[19863],{"type":155,"value":802},{"type":150,"tag":181,"props":19865,"children":19867},{"className":19866},[296],[19868,19873],{"type":150,"tag":181,"props":19869,"children":19871},{"className":19870},[296,301],[19872],{"type":155,"value":9859},{"type":150,"tag":181,"props":19874,"children":19876},{"className":19875},[307],[19877],{"type":150,"tag":181,"props":19878,"children":19880},{"className":19879},[312,313],[19881,19926],{"type":150,"tag":181,"props":19882,"children":19884},{"className":19883},[318],[19885,19921],{"type":150,"tag":181,"props":19886,"children":19888},{"className":19887,"style":3821},[323],[19889,19905],{"type":150,"tag":181,"props":19890,"children":19891},{"style":15391},[19892,19896],{"type":150,"tag":181,"props":19893,"children":19895},{"className":19894,"style":334},[333],[],{"type":150,"tag":181,"props":19897,"children":19899},{"className":19898},[339,340,341,342],[19900],{"type":150,"tag":181,"props":19901,"children":19903},{"className":19902},[296,301,342],[19904],{"type":155,"value":558},{"type":150,"tag":181,"props":19906,"children":19907},{"style":3730},[19908,19912],{"type":150,"tag":181,"props":19909,"children":19911},{"className":19910,"style":334},[333],[],{"type":150,"tag":181,"props":19913,"children":19915},{"className":19914},[339,340,341,342],[19916],{"type":150,"tag":181,"props":19917,"children":19919},{"className":19918},[296,342],[19920],{"type":155,"value":1874},{"type":150,"tag":181,"props":19922,"children":19924},{"className":19923},[352],[19925],{"type":155,"value":355},{"type":150,"tag":181,"props":19927,"children":19929},{"className":19928},[318],[19930],{"type":150,"tag":181,"props":19931,"children":19933},{"className":19932,"style":4550},[323],[19934],{"type":150,"tag":181,"props":19935,"children":19936},{},[],{"type":150,"tag":151,"props":19938,"children":19939},{},[19940,20302,20304,20382,20384,20487],{"type":150,"tag":181,"props":19941,"children":19943},{"className":19942},[184],[19944,20040],{"type":150,"tag":181,"props":19945,"children":19947},{"className":19946},[189],[19948],{"type":150,"tag":192,"props":19949,"children":19950},{"xmlns":194},[19951],{"type":150,"tag":197,"props":19952,"children":19953},{},[19954,20035],{"type":150,"tag":201,"props":19955,"children":19956},{},[19957,19983,20005],{"type":150,"tag":1046,"props":19958,"children":19959},{},[19960,19964,19968],{"type":150,"tag":209,"props":19961,"children":19962},{},[19963],{"type":155,"value":3265},{"type":150,"tag":209,"props":19965,"children":19966},{},[19967],{"type":155,"value":1667},{"type":150,"tag":201,"props":19969,"children":19970},{},[19971,19975,19979],{"type":150,"tag":221,"props":19972,"children":19973},{"stretchy":789},[19974],{"type":155,"value":792},{"type":150,"tag":209,"props":19976,"children":19977},{},[19978],{"type":155,"value":797},{"type":150,"tag":221,"props":19980,"children":19981},{"stretchy":789},[19982],{"type":155,"value":802},{"type":150,"tag":205,"props":19984,"children":19985},{},[19986,19990],{"type":150,"tag":209,"props":19987,"children":19988},{},[19989],{"type":155,"value":573},{"type":150,"tag":201,"props":19991,"children":19992},{},[19993,19997,20001],{"type":150,"tag":209,"props":19994,"children":19995},{},[19996],{"type":155,"value":1035},{"type":150,"tag":221,"props":19998,"children":19999},{},[20000],{"type":155,"value":18794},{"type":150,"tag":209,"props":20002,"children":20003},{},[20004],{"type":155,"value":558},{"type":150,"tag":1046,"props":20006,"children":20007},{},[20008,20012,20016],{"type":150,"tag":209,"props":20009,"children":20010},{},[20011],{"type":155,"value":3265},{"type":150,"tag":209,"props":20013,"children":20014},{},[20015],{"type":155,"value":1698},{"type":150,"tag":201,"props":20017,"children":20018},{},[20019,20023,20027,20031],{"type":150,"tag":221,"props":20020,"children":20021},{"stretchy":789},[20022],{"type":155,"value":792},{"type":150,"tag":209,"props":20024,"children":20025},{},[20026],{"type":155,"value":797},{"type":150,"tag":221,"props":20028,"children":20029},{"stretchy":789},[20030],{"type":155,"value":802},{"type":150,"tag":209,"props":20032,"children":20033},{"mathvariant":1618},[20034],{"type":155,"value":1874},{"type":150,"tag":271,"props":20036,"children":20037},{"encoding":273},[20038],{"type":155,"value":20039},"W_q^{(s)} R_{t-i} W_k^{(s)\\top}",{"type":150,"tag":181,"props":20041,"children":20043},{"className":20042,"ariaHidden":223},[280],[20044],{"type":150,"tag":181,"props":20045,"children":20047},{"className":20046},[285],[20048,20052,20139,20210],{"type":150,"tag":181,"props":20049,"children":20051},{"className":20050,"style":4022},[290],[],{"type":150,"tag":181,"props":20053,"children":20055},{"className":20054},[296],[20056,20061],{"type":150,"tag":181,"props":20057,"children":20059},{"className":20058,"style":3691},[296,301],[20060],{"type":155,"value":3265},{"type":150,"tag":181,"props":20062,"children":20064},{"className":20063},[307],[20065],{"type":150,"tag":181,"props":20066,"children":20068},{"className":20067},[312,313],[20069,20128],{"type":150,"tag":181,"props":20070,"children":20072},{"className":20071},[318],[20073,20123],{"type":150,"tag":181,"props":20074,"children":20076},{"className":20075,"style":1276},[323],[20077,20093],{"type":150,"tag":181,"props":20078,"children":20079},{"style":15560},[20080,20084],{"type":150,"tag":181,"props":20081,"children":20083},{"className":20082,"style":334},[333],[],{"type":150,"tag":181,"props":20085,"children":20087},{"className":20086},[339,340,341,342],[20088],{"type":150,"tag":181,"props":20089,"children":20091},{"className":20090,"style":2151},[296,301,342],[20092],{"type":155,"value":1667},{"type":150,"tag":181,"props":20094,"children":20095},{"style":1297},[20096,20100],{"type":150,"tag":181,"props":20097,"children":20099},{"className":20098,"style":334},[333],[],{"type":150,"tag":181,"props":20101,"children":20103},{"className":20102},[339,340,341,342],[20104],{"type":150,"tag":181,"props":20105,"children":20107},{"className":20106},[296,342],[20108,20113,20118],{"type":150,"tag":181,"props":20109,"children":20111},{"className":20110},[824,342],[20112],{"type":155,"value":792},{"type":150,"tag":181,"props":20114,"children":20116},{"className":20115},[296,301,342],[20117],{"type":155,"value":797},{"type":150,"tag":181,"props":20119,"children":20121},{"className":20120},[835,342],[20122],{"type":155,"value":802},{"type":150,"tag":181,"props":20124,"children":20126},{"className":20125},[352],[20127],{"type":155,"value":355},{"type":150,"tag":181,"props":20129,"children":20131},{"className":20130},[318],[20132],{"type":150,"tag":181,"props":20133,"children":20135},{"className":20134,"style":15617},[323],[20136],{"type":150,"tag":181,"props":20137,"children":20138},{},[],{"type":150,"tag":181,"props":20140,"children":20142},{"className":20141},[296],[20143,20148],{"type":150,"tag":181,"props":20144,"children":20146},{"className":20145,"style":19221},[296,301],[20147],{"type":155,"value":573},{"type":150,"tag":181,"props":20149,"children":20151},{"className":20150},[307],[20152],{"type":150,"tag":181,"props":20153,"children":20155},{"className":20154},[312,313],[20156,20199],{"type":150,"tag":181,"props":20157,"children":20159},{"className":20158},[318],[20160,20194],{"type":150,"tag":181,"props":20161,"children":20163},{"className":20162,"style":621},[323],[20164],{"type":150,"tag":181,"props":20165,"children":20166},{"style":19242},[20167,20171],{"type":150,"tag":181,"props":20168,"children":20170},{"className":20169,"style":334},[333],[],{"type":150,"tag":181,"props":20172,"children":20174},{"className":20173},[339,340,341,342],[20175],{"type":150,"tag":181,"props":20176,"children":20178},{"className":20177},[296,342],[20179,20184,20189],{"type":150,"tag":181,"props":20180,"children":20182},{"className":20181},[296,301,342],[20183],{"type":155,"value":1035},{"type":150,"tag":181,"props":20185,"children":20187},{"className":20186},[4998,342],[20188],{"type":155,"value":18794},{"type":150,"tag":181,"props":20190,"children":20192},{"className":20191},[296,301,342],[20193],{"type":155,"value":558},{"type":150,"tag":181,"props":20195,"children":20197},{"className":20196},[352],[20198],{"type":155,"value":355},{"type":150,"tag":181,"props":20200,"children":20202},{"className":20201},[318],[20203],{"type":150,"tag":181,"props":20204,"children":20206},{"className":20205,"style":19762},[323],[20207],{"type":150,"tag":181,"props":20208,"children":20209},{},[],{"type":150,"tag":181,"props":20211,"children":20213},{"className":20212},[296],[20214,20219],{"type":150,"tag":181,"props":20215,"children":20217},{"className":20216,"style":3691},[296,301],[20218],{"type":155,"value":3265},{"type":150,"tag":181,"props":20220,"children":20222},{"className":20221},[307],[20223],{"type":150,"tag":181,"props":20224,"children":20226},{"className":20225},[312,313],[20227,20291],{"type":150,"tag":181,"props":20228,"children":20230},{"className":20229},[318],[20231,20286],{"type":150,"tag":181,"props":20232,"children":20234},{"className":20233,"style":1276},[323],[20235,20251],{"type":150,"tag":181,"props":20236,"children":20237},{"style":4108},[20238,20242],{"type":150,"tag":181,"props":20239,"children":20241},{"className":20240,"style":334},[333],[],{"type":150,"tag":181,"props":20243,"children":20245},{"className":20244},[339,340,341,342],[20246],{"type":150,"tag":181,"props":20247,"children":20249},{"className":20248,"style":2249},[296,301,342],[20250],{"type":155,"value":1698},{"type":150,"tag":181,"props":20252,"children":20253},{"style":1297},[20254,20258],{"type":150,"tag":181,"props":20255,"children":20257},{"className":20256,"style":334},[333],[],{"type":150,"tag":181,"props":20259,"children":20261},{"className":20260},[339,340,341,342],[20262],{"type":150,"tag":181,"props":20263,"children":20265},{"className":20264},[296,342],[20266,20271,20276,20281],{"type":150,"tag":181,"props":20267,"children":20269},{"className":20268},[824,342],[20270],{"type":155,"value":792},{"type":150,"tag":181,"props":20272,"children":20274},{"className":20273},[296,301,342],[20275],{"type":155,"value":797},{"type":150,"tag":181,"props":20277,"children":20279},{"className":20278},[835,342],[20280],{"type":155,"value":802},{"type":150,"tag":181,"props":20282,"children":20284},{"className":20283},[296,342],[20285],{"type":155,"value":1874},{"type":150,"tag":181,"props":20287,"children":20289},{"className":20288},[352],[20290],{"type":155,"value":355},{"type":150,"tag":181,"props":20292,"children":20294},{"className":20293},[318],[20295],{"type":150,"tag":181,"props":20296,"children":20298},{"className":20297,"style":4165},[323],[20299],{"type":150,"tag":181,"props":20300,"children":20301},{},[],{"type":155,"value":20303}," 跟位置差 ",{"type":150,"tag":181,"props":20305,"children":20307},{"className":20306},[184],[20308,20338],{"type":150,"tag":181,"props":20309,"children":20311},{"className":20310},[189],[20312],{"type":150,"tag":192,"props":20313,"children":20314},{"xmlns":194},[20315],{"type":150,"tag":197,"props":20316,"children":20317},{},[20318,20333],{"type":150,"tag":201,"props":20319,"children":20320},{},[20321,20325,20329],{"type":150,"tag":209,"props":20322,"children":20323},{},[20324],{"type":155,"value":1035},{"type":150,"tag":221,"props":20326,"children":20327},{},[20328],{"type":155,"value":18794},{"type":150,"tag":209,"props":20330,"children":20331},{},[20332],{"type":155,"value":558},{"type":150,"tag":271,"props":20334,"children":20335},{"encoding":273},[20336],{"type":155,"value":20337},"t - i",{"type":150,"tag":181,"props":20339,"children":20341},{"className":20340,"ariaHidden":223},[280],[20342,20368],{"type":150,"tag":181,"props":20343,"children":20345},{"className":20344},[285],[20346,20350,20355,20359,20364],{"type":150,"tag":181,"props":20347,"children":20349},{"className":20348,"style":4983},[290],[],{"type":150,"tag":181,"props":20351,"children":20353},{"className":20352},[296,301],[20354],{"type":155,"value":1035},{"type":150,"tag":181,"props":20356,"children":20358},{"className":20357,"style":4993},[377],[],{"type":150,"tag":181,"props":20360,"children":20362},{"className":20361},[4998],[20363],{"type":155,"value":18794},{"type":150,"tag":181,"props":20365,"children":20367},{"className":20366,"style":4993},[377],[],{"type":150,"tag":181,"props":20369,"children":20371},{"className":20370},[285],[20372,20377],{"type":150,"tag":181,"props":20373,"children":20376},{"className":20374,"style":20375},[290],"height:0.6595em;",[],{"type":150,"tag":181,"props":20378,"children":20380},{"className":20379},[296,301],[20381],{"type":155,"value":558},{"type":155,"value":20383}," 相关，无法合并成一个固定矩阵。放弃 RoPE 换 ALIBI 之类的 attention bias 是一种办法，但实验显示明显不如 RoPE。最后 MLA 采取的是混合方案：每个 head 的 Q、K 新增 ",{"type":150,"tag":181,"props":20385,"children":20387},{"className":20386},[184],[20388,20418],{"type":150,"tag":181,"props":20389,"children":20391},{"className":20390},[189],[20392],{"type":150,"tag":192,"props":20393,"children":20394},{"xmlns":194},[20395],{"type":150,"tag":197,"props":20396,"children":20397},{},[20398,20413],{"type":150,"tag":201,"props":20399,"children":20400},{},[20401],{"type":150,"tag":205,"props":20402,"children":20403},{},[20404,20408],{"type":150,"tag":209,"props":20405,"children":20406},{},[20407],{"type":155,"value":578},{"type":150,"tag":209,"props":20409,"children":20410},{},[20411],{"type":155,"value":20412},"r",{"type":150,"tag":271,"props":20414,"children":20415},{"encoding":273},[20416],{"type":155,"value":20417},"d_r",{"type":150,"tag":181,"props":20419,"children":20421},{"className":20420,"ariaHidden":223},[280],[20422],{"type":150,"tag":181,"props":20423,"children":20425},{"className":20424},[285],[20426,20430],{"type":150,"tag":181,"props":20427,"children":20429},{"className":20428,"style":4761},[290],[],{"type":150,"tag":181,"props":20431,"children":20433},{"className":20432},[296],[20434,20439],{"type":150,"tag":181,"props":20435,"children":20437},{"className":20436},[296,301],[20438],{"type":155,"value":578},{"type":150,"tag":181,"props":20440,"children":20442},{"className":20441},[307],[20443],{"type":150,"tag":181,"props":20444,"children":20446},{"className":20445},[312,313],[20447,20476],{"type":150,"tag":181,"props":20448,"children":20450},{"className":20449},[318],[20451,20471],{"type":150,"tag":181,"props":20452,"children":20454},{"className":20453,"style":4865},[323],[20455],{"type":150,"tag":181,"props":20456,"children":20457},{"style":328},[20458,20462],{"type":150,"tag":181,"props":20459,"children":20461},{"className":20460,"style":334},[333],[],{"type":150,"tag":181,"props":20463,"children":20465},{"className":20464},[339,340,341,342],[20466],{"type":150,"tag":181,"props":20467,"children":20469},{"className":20468,"style":6794},[296,301,342],[20470],{"type":155,"value":20412},{"type":150,"tag":181,"props":20472,"children":20474},{"className":20473},[352],[20475],{"type":155,"value":355},{"type":150,"tag":181,"props":20477,"children":20479},{"className":20478},[318],[20480],{"type":150,"tag":181,"props":20481,"children":20483},{"className":20482,"style":363},[323],[20484],{"type":150,"tag":181,"props":20485,"children":20486},{},[],{"type":155,"value":20488}," 个维度专门用来加 RoPE，其中 K 新增的这部分在 head 间共享。",{"type":150,"tag":151,"props":20490,"children":20491},{},[20492,20494,20595,20597,20748,20750,20887],{"type":155,"value":20493},"这样没有 RoPE 的维度继续走前面的压缩，带 RoPE 的维度补充位置信息。",{"type":150,"tag":181,"props":20495,"children":20497},{"className":20496},[184],[20498,20526],{"type":150,"tag":181,"props":20499,"children":20501},{"className":20500},[189],[20502],{"type":150,"tag":192,"props":20503,"children":20504},{"xmlns":194},[20505],{"type":150,"tag":197,"props":20506,"children":20507},{},[20508,20522],{"type":150,"tag":201,"props":20509,"children":20510},{},[20511],{"type":150,"tag":205,"props":20512,"children":20513},{},[20514,20518],{"type":150,"tag":209,"props":20515,"children":20516},{},[20517],{"type":155,"value":578},{"type":150,"tag":209,"props":20519,"children":20520},{},[20521],{"type":155,"value":20412},{"type":150,"tag":271,"props":20523,"children":20524},{"encoding":273},[20525],{"type":155,"value":20417},{"type":150,"tag":181,"props":20527,"children":20529},{"className":20528,"ariaHidden":223},[280],[20530],{"type":150,"tag":181,"props":20531,"children":20533},{"className":20532},[285],[20534,20538],{"type":150,"tag":181,"props":20535,"children":20537},{"className":20536,"style":4761},[290],[],{"type":150,"tag":181,"props":20539,"children":20541},{"className":20540},[296],[20542,20547],{"type":150,"tag":181,"props":20543,"children":20545},{"className":20544},[296,301],[20546],{"type":155,"value":578},{"type":150,"tag":181,"props":20548,"children":20550},{"className":20549},[307],[20551],{"type":150,"tag":181,"props":20552,"children":20554},{"className":20553},[312,313],[20555,20584],{"type":150,"tag":181,"props":20556,"children":20558},{"className":20557},[318],[20559,20579],{"type":150,"tag":181,"props":20560,"children":20562},{"className":20561,"style":4865},[323],[20563],{"type":150,"tag":181,"props":20564,"children":20565},{"style":328},[20566,20570],{"type":150,"tag":181,"props":20567,"children":20569},{"className":20568,"style":334},[333],[],{"type":150,"tag":181,"props":20571,"children":20573},{"className":20572},[339,340,341,342],[20574],{"type":150,"tag":181,"props":20575,"children":20577},{"className":20576,"style":6794},[296,301,342],[20578],{"type":155,"value":20412},{"type":150,"tag":181,"props":20580,"children":20582},{"className":20581},[352],[20583],{"type":155,"value":355},{"type":150,"tag":181,"props":20585,"children":20587},{"className":20586},[318],[20588],{"type":150,"tag":181,"props":20589,"children":20591},{"className":20590,"style":363},[323],[20592],{"type":150,"tag":181,"props":20593,"children":20594},{},[],{"type":155,"value":20596}," 取 ",{"type":150,"tag":181,"props":20598,"children":20600},{"className":20599},[184],[20601,20647],{"type":150,"tag":181,"props":20602,"children":20604},{"className":20603},[189],[20605],{"type":150,"tag":192,"props":20606,"children":20607},{"xmlns":194},[20608],{"type":150,"tag":197,"props":20609,"children":20610},{},[20611,20642],{"type":150,"tag":201,"props":20612,"children":20613},{},[20614,20625,20629,20633,20637],{"type":150,"tag":205,"props":20615,"children":20616},{},[20617,20621],{"type":150,"tag":209,"props":20618,"children":20619},{},[20620],{"type":155,"value":578},{"type":150,"tag":209,"props":20622,"children":20623},{},[20624],{"type":155,"value":1698},{"type":150,"tag":209,"props":20626,"children":20627},{"mathvariant":1618},[20628],{"type":155,"value":4740},{"type":150,"tag":215,"props":20630,"children":20631},{},[20632],{"type":155,"value":238},{"type":150,"tag":221,"props":20634,"children":20635},{},[20636],{"type":155,"value":865},{"type":150,"tag":215,"props":20638,"children":20639},{},[20640],{"type":155,"value":20641},"64",{"type":150,"tag":271,"props":20643,"children":20644},{"encoding":273},[20645],{"type":155,"value":20646},"d_k\u002F2 = 64",{"type":150,"tag":181,"props":20648,"children":20650},{"className":20649,"ariaHidden":223},[280],[20651,20735],{"type":150,"tag":181,"props":20652,"children":20654},{"className":20653},[285],[20655,20659,20716,20722,20726,20731],{"type":150,"tag":181,"props":20656,"children":20658},{"className":20657,"style":819},[290],[],{"type":150,"tag":181,"props":20660,"children":20662},{"className":20661},[296],[20663,20668],{"type":150,"tag":181,"props":20664,"children":20666},{"className":20665},[296,301],[20667],{"type":155,"value":578},{"type":150,"tag":181,"props":20669,"children":20671},{"className":20670},[307],[20672],{"type":150,"tag":181,"props":20673,"children":20675},{"className":20674},[312,313],[20676,20705],{"type":150,"tag":181,"props":20677,"children":20679},{"className":20678},[318],[20680,20700],{"type":150,"tag":181,"props":20681,"children":20683},{"className":20682,"style":493},[323],[20684],{"type":150,"tag":181,"props":20685,"children":20686},{"style":328},[20687,20691],{"type":150,"tag":181,"props":20688,"children":20690},{"className":20689,"style":334},[333],[],{"type":150,"tag":181,"props":20692,"children":20694},{"className":20693},[339,340,341,342],[20695],{"type":150,"tag":181,"props":20696,"children":20698},{"className":20697,"style":2249},[296,301,342],[20699],{"type":155,"value":1698},{"type":150,"tag":181,"props":20701,"children":20703},{"className":20702},[352],[20704],{"type":155,"value":355},{"type":150,"tag":181,"props":20706,"children":20708},{"className":20707},[318],[20709],{"type":150,"tag":181,"props":20710,"children":20712},{"className":20711,"style":363},[323],[20713],{"type":150,"tag":181,"props":20714,"children":20715},{},[],{"type":150,"tag":181,"props":20717,"children":20719},{"className":20718},[296],[20720],{"type":155,"value":20721},"\u002F2",{"type":150,"tag":181,"props":20723,"children":20725},{"className":20724,"style":658},[377],[],{"type":150,"tag":181,"props":20727,"children":20729},{"className":20728},[663],[20730],{"type":155,"value":865},{"type":150,"tag":181,"props":20732,"children":20734},{"className":20733,"style":658},[377],[],{"type":150,"tag":181,"props":20736,"children":20738},{"className":20737},[285],[20739,20743],{"type":150,"tag":181,"props":20740,"children":20742},{"className":20741,"style":5012},[290],[],{"type":150,"tag":181,"props":20744,"children":20746},{"className":20745},[296],[20747],{"type":155,"value":20641},{"type":155,"value":20749},"，相比 ",{"type":150,"tag":181,"props":20751,"children":20753},{"className":20752},[184],[20754,20792],{"type":150,"tag":181,"props":20755,"children":20757},{"className":20756},[189],[20758],{"type":150,"tag":192,"props":20759,"children":20760},{"xmlns":194},[20761],{"type":150,"tag":197,"props":20762,"children":20763},{},[20764,20787],{"type":150,"tag":201,"props":20765,"children":20766},{},[20767,20778,20782],{"type":150,"tag":205,"props":20768,"children":20769},{},[20770,20774],{"type":150,"tag":209,"props":20771,"children":20772},{},[20773],{"type":155,"value":578},{"type":150,"tag":209,"props":20775,"children":20776},{},[20777],{"type":155,"value":9859},{"type":150,"tag":221,"props":20779,"children":20780},{},[20781],{"type":155,"value":865},{"type":150,"tag":215,"props":20783,"children":20784},{},[20785],{"type":155,"value":20786},"512",{"type":150,"tag":271,"props":20788,"children":20789},{"encoding":273},[20790],{"type":155,"value":20791},"d_c = 512",{"type":150,"tag":181,"props":20793,"children":20795},{"className":20794,"ariaHidden":223},[280],[20796,20874],{"type":150,"tag":181,"props":20797,"children":20799},{"className":20798},[285],[20800,20804,20861,20865,20870],{"type":150,"tag":181,"props":20801,"children":20803},{"className":20802,"style":4761},[290],[],{"type":150,"tag":181,"props":20805,"children":20807},{"className":20806},[296],[20808,20813],{"type":150,"tag":181,"props":20809,"children":20811},{"className":20810},[296,301],[20812],{"type":155,"value":578},{"type":150,"tag":181,"props":20814,"children":20816},{"className":20815},[307],[20817],{"type":150,"tag":181,"props":20818,"children":20820},{"className":20819},[312,313],[20821,20850],{"type":150,"tag":181,"props":20822,"children":20824},{"className":20823},[318],[20825,20845],{"type":150,"tag":181,"props":20826,"children":20828},{"className":20827,"style":4865},[323],[20829],{"type":150,"tag":181,"props":20830,"children":20831},{"style":328},[20832,20836],{"type":150,"tag":181,"props":20833,"children":20835},{"className":20834,"style":334},[333],[],{"type":150,"tag":181,"props":20837,"children":20839},{"className":20838},[339,340,341,342],[20840],{"type":150,"tag":181,"props":20841,"children":20843},{"className":20842},[296,301,342],[20844],{"type":155,"value":9859},{"type":150,"tag":181,"props":20846,"children":20848},{"className":20847},[352],[20849],{"type":155,"value":355},{"type":150,"tag":181,"props":20851,"children":20853},{"className":20852},[318],[20854],{"type":150,"tag":181,"props":20855,"children":20857},{"className":20856,"style":363},[323],[20858],{"type":150,"tag":181,"props":20859,"children":20860},{},[],{"type":150,"tag":181,"props":20862,"children":20864},{"className":20863,"style":658},[377],[],{"type":150,"tag":181,"props":20866,"children":20868},{"className":20867},[663],[20869],{"type":155,"value":865},{"type":150,"tag":181,"props":20871,"children":20873},{"className":20872,"style":658},[377],[],{"type":150,"tag":181,"props":20875,"children":20877},{"className":20876},[285],[20878,20882],{"type":150,"tag":181,"props":20879,"children":20881},{"className":20880,"style":5012},[290],[],{"type":150,"tag":181,"props":20883,"children":20885},{"className":20884},[296],[20886],{"type":155,"value":20786},{"type":155,"value":20888},"，增加幅度不大。",{"type":150,"tag":151,"props":20890,"children":20891},{},[20892,20894,21024,21026,21161,21163,21358,21360,21462],{"type":155,"value":20893},"还有一个细节：MLA 最终版把 Q 的输入也改成了低秩投影。这跟减少 KV Cache 无关，主要是为了减少训练期间的参数量和梯度所占显存。",{"type":150,"tag":181,"props":20895,"children":20897},{"className":20896},[184],[20898,20933],{"type":150,"tag":181,"props":20899,"children":20901},{"className":20900},[189],[20902],{"type":150,"tag":192,"props":20903,"children":20904},{"xmlns":194},[20905],{"type":150,"tag":197,"props":20906,"children":20907},{},[20908,20928],{"type":150,"tag":201,"props":20909,"children":20910},{},[20911],{"type":150,"tag":1046,"props":20912,"children":20913},{},[20914,20918,20922],{"type":150,"tag":209,"props":20915,"children":20916},{},[20917],{"type":155,"value":578},{"type":150,"tag":209,"props":20919,"children":20920},{},[20921],{"type":155,"value":9859},{"type":150,"tag":221,"props":20923,"children":20925},{"mathvariant":1618,"lspace":20924,"rspace":20924},"0em",[20926],{"type":155,"value":20927},"′",{"type":150,"tag":271,"props":20929,"children":20930},{"encoding":273},[20931],{"type":155,"value":20932},"d_c'",{"type":150,"tag":181,"props":20934,"children":20936},{"className":20935,"ariaHidden":223},[280],[20937],{"type":150,"tag":181,"props":20938,"children":20940},{"className":20939},[285],[20941,20946],{"type":150,"tag":181,"props":20942,"children":20945},{"className":20943,"style":20944},[290],"height:0.9989em;vertical-align:-0.247em;",[],{"type":150,"tag":181,"props":20947,"children":20949},{"className":20948},[296],[20950,20955],{"type":150,"tag":181,"props":20951,"children":20953},{"className":20952},[296,301],[20954],{"type":155,"value":578},{"type":150,"tag":181,"props":20956,"children":20958},{"className":20957},[307],[20959],{"type":150,"tag":181,"props":20960,"children":20962},{"className":20961},[312,313],[20963,21013],{"type":150,"tag":181,"props":20964,"children":20966},{"className":20965},[318],[20967,21008],{"type":150,"tag":181,"props":20968,"children":20971},{"className":20969,"style":20970},[323],"height:0.7519em;",[20972,20988],{"type":150,"tag":181,"props":20973,"children":20974},{"style":15391},[20975,20979],{"type":150,"tag":181,"props":20976,"children":20978},{"className":20977,"style":334},[333],[],{"type":150,"tag":181,"props":20980,"children":20982},{"className":20981},[339,340,341,342],[20983],{"type":150,"tag":181,"props":20984,"children":20986},{"className":20985},[296,301,342],[20987],{"type":155,"value":9859},{"type":150,"tag":181,"props":20989,"children":20990},{"style":707},[20991,20995],{"type":150,"tag":181,"props":20992,"children":20994},{"className":20993,"style":334},[333],[],{"type":150,"tag":181,"props":20996,"children":20998},{"className":20997},[339,340,341,342],[20999],{"type":150,"tag":181,"props":21000,"children":21002},{"className":21001},[296,342],[21003],{"type":150,"tag":181,"props":21004,"children":21006},{"className":21005},[296,342],[21007],{"type":155,"value":20927},{"type":150,"tag":181,"props":21009,"children":21011},{"className":21010},[352],[21012],{"type":155,"value":355},{"type":150,"tag":181,"props":21014,"children":21016},{"className":21015},[318],[21017],{"type":150,"tag":181,"props":21018,"children":21020},{"className":21019,"style":4550},[323],[21021],{"type":150,"tag":181,"props":21022,"children":21023},{},[],{"type":155,"value":21025}," 取 1536，和 ",{"type":150,"tag":181,"props":21027,"children":21029},{"className":21028},[184],[21030,21066],{"type":150,"tag":181,"props":21031,"children":21033},{"className":21032},[189],[21034],{"type":150,"tag":192,"props":21035,"children":21036},{"xmlns":194},[21037],{"type":150,"tag":197,"props":21038,"children":21039},{},[21040,21062],{"type":150,"tag":201,"props":21041,"children":21042},{},[21043,21054,21058],{"type":150,"tag":205,"props":21044,"children":21045},{},[21046,21050],{"type":150,"tag":209,"props":21047,"children":21048},{},[21049],{"type":155,"value":578},{"type":150,"tag":209,"props":21051,"children":21052},{},[21053],{"type":155,"value":9859},{"type":150,"tag":221,"props":21055,"children":21056},{},[21057],{"type":155,"value":865},{"type":150,"tag":215,"props":21059,"children":21060},{},[21061],{"type":155,"value":20786},{"type":150,"tag":271,"props":21063,"children":21064},{"encoding":273},[21065],{"type":155,"value":20791},{"type":150,"tag":181,"props":21067,"children":21069},{"className":21068,"ariaHidden":223},[280],[21070,21148],{"type":150,"tag":181,"props":21071,"children":21073},{"className":21072},[285],[21074,21078,21135,21139,21144],{"type":150,"tag":181,"props":21075,"children":21077},{"className":21076,"style":4761},[290],[],{"type":150,"tag":181,"props":21079,"children":21081},{"className":21080},[296],[21082,21087],{"type":150,"tag":181,"props":21083,"children":21085},{"className":21084},[296,301],[21086],{"type":155,"value":578},{"type":150,"tag":181,"props":21088,"children":21090},{"className":21089},[307],[21091],{"type":150,"tag":181,"props":21092,"children":21094},{"className":21093},[312,313],[21095,21124],{"type":150,"tag":181,"props":21096,"children":21098},{"className":21097},[318],[21099,21119],{"type":150,"tag":181,"props":21100,"children":21102},{"className":21101,"style":4865},[323],[21103],{"type":150,"tag":181,"props":21104,"children":21105},{"style":328},[21106,21110],{"type":150,"tag":181,"props":21107,"children":21109},{"className":21108,"style":334},[333],[],{"type":150,"tag":181,"props":21111,"children":21113},{"className":21112},[339,340,341,342],[21114],{"type":150,"tag":181,"props":21115,"children":21117},{"className":21116},[296,301,342],[21118],{"type":155,"value":9859},{"type":150,"tag":181,"props":21120,"children":21122},{"className":21121},[352],[21123],{"type":155,"value":355},{"type":150,"tag":181,"props":21125,"children":21127},{"className":21126},[318],[21128],{"type":150,"tag":181,"props":21129,"children":21131},{"className":21130,"style":363},[323],[21132],{"type":150,"tag":181,"props":21133,"children":21134},{},[],{"type":150,"tag":181,"props":21136,"children":21138},{"className":21137,"style":658},[377],[],{"type":150,"tag":181,"props":21140,"children":21142},{"className":21141},[663],[21143],{"type":155,"value":865},{"type":150,"tag":181,"props":21145,"children":21147},{"className":21146,"style":658},[377],[],{"type":150,"tag":181,"props":21149,"children":21151},{"className":21150},[285],[21152,21156],{"type":150,"tag":181,"props":21153,"children":21155},{"className":21154,"style":5012},[290],[],{"type":150,"tag":181,"props":21157,"children":21159},{"className":21158},[296],[21160],{"type":155,"value":20786},{"type":155,"value":21162}," 不同。于是解码阶段的 MLA 是 MQA 形式，Q、K 的 head size 变成 ",{"type":150,"tag":181,"props":21164,"children":21166},{"className":21165},[184],[21167,21211],{"type":150,"tag":181,"props":21168,"children":21170},{"className":21169},[189],[21171],{"type":150,"tag":192,"props":21172,"children":21173},{"xmlns":194},[21174],{"type":150,"tag":197,"props":21175,"children":21176},{},[21177,21206],{"type":150,"tag":201,"props":21178,"children":21179},{},[21180,21191,21195],{"type":150,"tag":205,"props":21181,"children":21182},{},[21183,21187],{"type":150,"tag":209,"props":21184,"children":21185},{},[21186],{"type":155,"value":578},{"type":150,"tag":209,"props":21188,"children":21189},{},[21190],{"type":155,"value":9859},{"type":150,"tag":221,"props":21192,"children":21193},{},[21194],{"type":155,"value":4962},{"type":150,"tag":205,"props":21196,"children":21197},{},[21198,21202],{"type":150,"tag":209,"props":21199,"children":21200},{},[21201],{"type":155,"value":578},{"type":150,"tag":209,"props":21203,"children":21204},{},[21205],{"type":155,"value":20412},{"type":150,"tag":271,"props":21207,"children":21208},{"encoding":273},[21209],{"type":155,"value":21210},"d_c + d_r",{"type":150,"tag":181,"props":21212,"children":21214},{"className":21213,"ariaHidden":223},[280],[21215,21293],{"type":150,"tag":181,"props":21216,"children":21218},{"className":21217},[285],[21219,21223,21280,21284,21289],{"type":150,"tag":181,"props":21220,"children":21222},{"className":21221,"style":4761},[290],[],{"type":150,"tag":181,"props":21224,"children":21226},{"className":21225},[296],[21227,21232],{"type":150,"tag":181,"props":21228,"children":21230},{"className":21229},[296,301],[21231],{"type":155,"value":578},{"type":150,"tag":181,"props":21233,"children":21235},{"className":21234},[307],[21236],{"type":150,"tag":181,"props":21237,"children":21239},{"className":21238},[312,313],[21240,21269],{"type":150,"tag":181,"props":21241,"children":21243},{"className":21242},[318],[21244,21264],{"type":150,"tag":181,"props":21245,"children":21247},{"className":21246,"style":4865},[323],[21248],{"type":150,"tag":181,"props":21249,"children":21250},{"style":328},[21251,21255],{"type":150,"tag":181,"props":21252,"children":21254},{"className":21253,"style":334},[333],[],{"type":150,"tag":181,"props":21256,"children":21258},{"className":21257},[339,340,341,342],[21259],{"type":150,"tag":181,"props":21260,"children":21262},{"className":21261},[296,301,342],[21263],{"type":155,"value":9859},{"type":150,"tag":181,"props":21265,"children":21267},{"className":21266},[352],[21268],{"type":155,"value":355},{"type":150,"tag":181,"props":21270,"children":21272},{"className":21271},[318],[21273],{"type":150,"tag":181,"props":21274,"children":21276},{"className":21275,"style":363},[323],[21277],{"type":150,"tag":181,"props":21278,"children":21279},{},[],{"type":150,"tag":181,"props":21281,"children":21283},{"className":21282,"style":4993},[377],[],{"type":150,"tag":181,"props":21285,"children":21287},{"className":21286},[4998],[21288],{"type":155,"value":4962},{"type":150,"tag":181,"props":21290,"children":21292},{"className":21291,"style":4993},[377],[],{"type":150,"tag":181,"props":21294,"children":21296},{"className":21295},[285],[21297,21301],{"type":150,"tag":181,"props":21298,"children":21300},{"className":21299,"style":4761},[290],[],{"type":150,"tag":181,"props":21302,"children":21304},{"className":21303},[296],[21305,21310],{"type":150,"tag":181,"props":21306,"children":21308},{"className":21307},[296,301],[21309],{"type":155,"value":578},{"type":150,"tag":181,"props":21311,"children":21313},{"className":21312},[307],[21314],{"type":150,"tag":181,"props":21315,"children":21317},{"className":21316},[312,313],[21318,21347],{"type":150,"tag":181,"props":21319,"children":21321},{"className":21320},[318],[21322,21342],{"type":150,"tag":181,"props":21323,"children":21325},{"className":21324,"style":4865},[323],[21326],{"type":150,"tag":181,"props":21327,"children":21328},{"style":328},[21329,21333],{"type":150,"tag":181,"props":21330,"children":21332},{"className":21331,"style":334},[333],[],{"type":150,"tag":181,"props":21334,"children":21336},{"className":21335},[339,340,341,342],[21337],{"type":150,"tag":181,"props":21338,"children":21340},{"className":21339,"style":6794},[296,301,342],[21341],{"type":155,"value":20412},{"type":150,"tag":181,"props":21343,"children":21345},{"className":21344},[352],[21346],{"type":155,"value":355},{"type":150,"tag":181,"props":21348,"children":21350},{"className":21349},[318],[21351],{"type":150,"tag":181,"props":21352,"children":21354},{"className":21353,"style":363},[323],[21355],{"type":150,"tag":181,"props":21356,"children":21357},{},[],{"type":155,"value":21359},"，是 ",{"type":150,"tag":181,"props":21361,"children":21363},{"className":21362},[184],[21364,21393],{"type":150,"tag":181,"props":21365,"children":21367},{"className":21366},[189],[21368],{"type":150,"tag":192,"props":21369,"children":21370},{"xmlns":194},[21371],{"type":150,"tag":197,"props":21372,"children":21373},{},[21374,21388],{"type":150,"tag":201,"props":21375,"children":21376},{},[21377],{"type":150,"tag":205,"props":21378,"children":21379},{},[21380,21384],{"type":150,"tag":209,"props":21381,"children":21382},{},[21383],{"type":155,"value":578},{"type":150,"tag":209,"props":21385,"children":21386},{},[21387],{"type":155,"value":1698},{"type":150,"tag":271,"props":21389,"children":21390},{"encoding":273},[21391],{"type":155,"value":21392},"d_k",{"type":150,"tag":181,"props":21394,"children":21396},{"className":21395,"ariaHidden":223},[280],[21397],{"type":150,"tag":181,"props":21398,"children":21400},{"className":21399},[285],[21401,21405],{"type":150,"tag":181,"props":21402,"children":21404},{"className":21403,"style":4761},[290],[],{"type":150,"tag":181,"props":21406,"children":21408},{"className":21407},[296],[21409,21414],{"type":150,"tag":181,"props":21410,"children":21412},{"className":21411},[296,301],[21413],{"type":155,"value":578},{"type":150,"tag":181,"props":21415,"children":21417},{"className":21416},[307],[21418],{"type":150,"tag":181,"props":21419,"children":21421},{"className":21420},[312,313],[21422,21451],{"type":150,"tag":181,"props":21423,"children":21425},{"className":21424},[318],[21426,21446],{"type":150,"tag":181,"props":21427,"children":21429},{"className":21428,"style":493},[323],[21430],{"type":150,"tag":181,"props":21431,"children":21432},{"style":328},[21433,21437],{"type":150,"tag":181,"props":21434,"children":21436},{"className":21435,"style":334},[333],[],{"type":150,"tag":181,"props":21438,"children":21440},{"className":21439},[339,340,341,342],[21441],{"type":150,"tag":181,"props":21442,"children":21444},{"className":21443,"style":2249},[296,301,342],[21445],{"type":155,"value":1698},{"type":150,"tag":181,"props":21447,"children":21449},{"className":21448},[352],[21450],{"type":155,"value":355},{"type":150,"tag":181,"props":21452,"children":21454},{"className":21453},[318],[21455],{"type":150,"tag":181,"props":21456,"children":21458},{"className":21457,"style":363},[323],[21459],{"type":150,"tag":181,"props":21460,"children":21461},{},[],{"type":155,"value":21463}," 的 4 倍。",{"type":150,"tag":151,"props":21465,"children":21466},{},[21467],{"type":150,"tag":158,"props":21468,"children":21469},{},[21470],{"type":155,"value":21471},"解码的计算量其实是增加的。",{"type":150,"tag":151,"props":21473,"children":21474},{},[21475,21477,21555,21557,21634,21636,21772,21774,21909,21910,22046,22048,22126,22128,22170],{"type":155,"value":21476},"用 DeepSeek-V2 的配置把这笔账算一遍。",{"type":150,"tag":181,"props":21478,"children":21480},{"className":21479},[184],[21481,21512],{"type":150,"tag":181,"props":21482,"children":21484},{"className":21483},[189],[21485],{"type":150,"tag":192,"props":21486,"children":21487},{"xmlns":194},[21488],{"type":150,"tag":197,"props":21489,"children":21490},{},[21491,21507],{"type":150,"tag":201,"props":21492,"children":21493},{},[21494,21498,21502],{"type":150,"tag":209,"props":21495,"children":21496},{},[21497],{"type":155,"value":5385},{"type":150,"tag":221,"props":21499,"children":21500},{},[21501],{"type":155,"value":865},{"type":150,"tag":215,"props":21503,"children":21504},{},[21505],{"type":155,"value":21506},"60",{"type":150,"tag":271,"props":21508,"children":21509},{"encoding":273},[21510],{"type":155,"value":21511},"L = 60",{"type":150,"tag":181,"props":21513,"children":21515},{"className":21514,"ariaHidden":223},[280],[21516,21542],{"type":150,"tag":181,"props":21517,"children":21519},{"className":21518},[285],[21520,21524,21529,21533,21538],{"type":150,"tag":181,"props":21521,"children":21523},{"className":21522,"style":5680},[290],[],{"type":150,"tag":181,"props":21525,"children":21527},{"className":21526},[296,301],[21528],{"type":155,"value":5385},{"type":150,"tag":181,"props":21530,"children":21532},{"className":21531,"style":658},[377],[],{"type":150,"tag":181,"props":21534,"children":21536},{"className":21535},[663],[21537],{"type":155,"value":865},{"type":150,"tag":181,"props":21539,"children":21541},{"className":21540,"style":658},[377],[],{"type":150,"tag":181,"props":21543,"children":21545},{"className":21544},[285],[21546,21550],{"type":150,"tag":181,"props":21547,"children":21549},{"className":21548,"style":5012},[290],[],{"type":150,"tag":181,"props":21551,"children":21553},{"className":21552},[296],[21554],{"type":155,"value":21506},{"type":155,"value":21556}," 层，",{"type":150,"tag":181,"props":21558,"children":21560},{"className":21559},[184],[21561,21591],{"type":150,"tag":181,"props":21562,"children":21564},{"className":21563},[189],[21565],{"type":150,"tag":192,"props":21566,"children":21567},{"xmlns":194},[21568],{"type":150,"tag":197,"props":21569,"children":21570},{},[21571,21586],{"type":150,"tag":201,"props":21572,"children":21573},{},[21574,21578,21582],{"type":150,"tag":209,"props":21575,"children":21576},{},[21577],{"type":155,"value":745},{"type":150,"tag":221,"props":21579,"children":21580},{},[21581],{"type":155,"value":865},{"type":150,"tag":215,"props":21583,"children":21584},{},[21585],{"type":155,"value":5871},{"type":150,"tag":271,"props":21587,"children":21588},{"encoding":273},[21589],{"type":155,"value":21590},"h = 128",{"type":150,"tag":181,"props":21592,"children":21594},{"className":21593,"ariaHidden":223},[280],[21595,21621],{"type":150,"tag":181,"props":21596,"children":21598},{"className":21597},[285],[21599,21603,21608,21612,21617],{"type":150,"tag":181,"props":21600,"children":21602},{"className":21601,"style":761},[290],[],{"type":150,"tag":181,"props":21604,"children":21606},{"className":21605},[296,301],[21607],{"type":155,"value":745},{"type":150,"tag":181,"props":21609,"children":21611},{"className":21610,"style":658},[377],[],{"type":150,"tag":181,"props":21613,"children":21615},{"className":21614},[663],[21616],{"type":155,"value":865},{"type":150,"tag":181,"props":21618,"children":21620},{"className":21619,"style":658},[377],[],{"type":150,"tag":181,"props":21622,"children":21624},{"className":21623},[285],[21625,21629],{"type":150,"tag":181,"props":21626,"children":21628},{"className":21627,"style":5012},[290],[],{"type":150,"tag":181,"props":21630,"children":21632},{"className":21631},[296],[21633],{"type":155,"value":5871},{"type":155,"value":21635}," 个 head，",{"type":150,"tag":181,"props":21637,"children":21639},{"className":21638},[184],[21640,21677],{"type":150,"tag":181,"props":21641,"children":21643},{"className":21642},[189],[21644],{"type":150,"tag":192,"props":21645,"children":21646},{"xmlns":194},[21647],{"type":150,"tag":197,"props":21648,"children":21649},{},[21650,21672],{"type":150,"tag":201,"props":21651,"children":21652},{},[21653,21664,21668],{"type":150,"tag":205,"props":21654,"children":21655},{},[21656,21660],{"type":150,"tag":209,"props":21657,"children":21658},{},[21659],{"type":155,"value":578},{"type":150,"tag":209,"props":21661,"children":21662},{},[21663],{"type":155,"value":1698},{"type":150,"tag":221,"props":21665,"children":21666},{},[21667],{"type":155,"value":865},{"type":150,"tag":215,"props":21669,"children":21670},{},[21671],{"type":155,"value":5871},{"type":150,"tag":271,"props":21673,"children":21674},{"encoding":273},[21675],{"type":155,"value":21676},"d_k = 128",{"type":150,"tag":181,"props":21678,"children":21680},{"className":21679,"ariaHidden":223},[280],[21681,21759],{"type":150,"tag":181,"props":21682,"children":21684},{"className":21683},[285],[21685,21689,21746,21750,21755],{"type":150,"tag":181,"props":21686,"children":21688},{"className":21687,"style":4761},[290],[],{"type":150,"tag":181,"props":21690,"children":21692},{"className":21691},[296],[21693,21698],{"type":150,"tag":181,"props":21694,"children":21696},{"className":21695},[296,301],[21697],{"type":155,"value":578},{"type":150,"tag":181,"props":21699,"children":21701},{"className":21700},[307],[21702],{"type":150,"tag":181,"props":21703,"children":21705},{"className":21704},[312,313],[21706,21735],{"type":150,"tag":181,"props":21707,"children":21709},{"className":21708},[318],[21710,21730],{"type":150,"tag":181,"props":21711,"children":21713},{"className":21712,"style":493},[323],[21714],{"type":150,"tag":181,"props":21715,"children":21716},{"style":328},[21717,21721],{"type":150,"tag":181,"props":21718,"children":21720},{"className":21719,"style":334},[333],[],{"type":150,"tag":181,"props":21722,"children":21724},{"className":21723},[339,340,341,342],[21725],{"type":150,"tag":181,"props":21726,"children":21728},{"className":21727,"style":2249},[296,301,342],[21729],{"type":155,"value":1698},{"type":150,"tag":181,"props":21731,"children":21733},{"className":21732},[352],[21734],{"type":155,"value":355},{"type":150,"tag":181,"props":21736,"children":21738},{"className":21737},[318],[21739],{"type":150,"tag":181,"props":21740,"children":21742},{"className":21741,"style":363},[323],[21743],{"type":150,"tag":181,"props":21744,"children":21745},{},[],{"type":150,"tag":181,"props":21747,"children":21749},{"className":21748,"style":658},[377],[],{"type":150,"tag":181,"props":21751,"children":21753},{"className":21752},[663],[21754],{"type":155,"value":865},{"type":150,"tag":181,"props":21756,"children":21758},{"className":21757,"style":658},[377],[],{"type":150,"tag":181,"props":21760,"children":21762},{"className":21761},[285],[21763,21767],{"type":150,"tag":181,"props":21764,"children":21766},{"className":21765,"style":5012},[290],[],{"type":150,"tag":181,"props":21768,"children":21770},{"className":21769},[296],[21771],{"type":155,"value":5871},{"type":155,"value":21773},"，MLA 这边 ",{"type":150,"tag":181,"props":21775,"children":21777},{"className":21776},[184],[21778,21814],{"type":150,"tag":181,"props":21779,"children":21781},{"className":21780},[189],[21782],{"type":150,"tag":192,"props":21783,"children":21784},{"xmlns":194},[21785],{"type":150,"tag":197,"props":21786,"children":21787},{},[21788,21810],{"type":150,"tag":201,"props":21789,"children":21790},{},[21791,21802,21806],{"type":150,"tag":205,"props":21792,"children":21793},{},[21794,21798],{"type":150,"tag":209,"props":21795,"children":21796},{},[21797],{"type":155,"value":578},{"type":150,"tag":209,"props":21799,"children":21800},{},[21801],{"type":155,"value":9859},{"type":150,"tag":221,"props":21803,"children":21804},{},[21805],{"type":155,"value":865},{"type":150,"tag":215,"props":21807,"children":21808},{},[21809],{"type":155,"value":20786},{"type":150,"tag":271,"props":21811,"children":21812},{"encoding":273},[21813],{"type":155,"value":20791},{"type":150,"tag":181,"props":21815,"children":21817},{"className":21816,"ariaHidden":223},[280],[21818,21896],{"type":150,"tag":181,"props":21819,"children":21821},{"className":21820},[285],[21822,21826,21883,21887,21892],{"type":150,"tag":181,"props":21823,"children":21825},{"className":21824,"style":4761},[290],[],{"type":150,"tag":181,"props":21827,"children":21829},{"className":21828},[296],[21830,21835],{"type":150,"tag":181,"props":21831,"children":21833},{"className":21832},[296,301],[21834],{"type":155,"value":578},{"type":150,"tag":181,"props":21836,"children":21838},{"className":21837},[307],[21839],{"type":150,"tag":181,"props":21840,"children":21842},{"className":21841},[312,313],[21843,21872],{"type":150,"tag":181,"props":21844,"children":21846},{"className":21845},[318],[21847,21867],{"type":150,"tag":181,"props":21848,"children":21850},{"className":21849,"style":4865},[323],[21851],{"type":150,"tag":181,"props":21852,"children":21853},{"style":328},[21854,21858],{"type":150,"tag":181,"props":21855,"children":21857},{"className":21856,"style":334},[333],[],{"type":150,"tag":181,"props":21859,"children":21861},{"className":21860},[339,340,341,342],[21862],{"type":150,"tag":181,"props":21863,"children":21865},{"className":21864},[296,301,342],[21866],{"type":155,"value":9859},{"type":150,"tag":181,"props":21868,"children":21870},{"className":21869},[352],[21871],{"type":155,"value":355},{"type":150,"tag":181,"props":21873,"children":21875},{"className":21874},[318],[21876],{"type":150,"tag":181,"props":21877,"children":21879},{"className":21878,"style":363},[323],[21880],{"type":150,"tag":181,"props":21881,"children":21882},{},[],{"type":150,"tag":181,"props":21884,"children":21886},{"className":21885,"style":658},[377],[],{"type":150,"tag":181,"props":21888,"children":21890},{"className":21889},[663],[21891],{"type":155,"value":865},{"type":150,"tag":181,"props":21893,"children":21895},{"className":21894,"style":658},[377],[],{"type":150,"tag":181,"props":21897,"children":21899},{"className":21898},[285],[21900,21904],{"type":150,"tag":181,"props":21901,"children":21903},{"className":21902,"style":5012},[290],[],{"type":150,"tag":181,"props":21905,"children":21907},{"className":21906},[296],[21908],{"type":155,"value":20786},{"type":155,"value":9252},{"type":150,"tag":181,"props":21911,"children":21913},{"className":21912},[184],[21914,21951],{"type":150,"tag":181,"props":21915,"children":21917},{"className":21916},[189],[21918],{"type":150,"tag":192,"props":21919,"children":21920},{"xmlns":194},[21921],{"type":150,"tag":197,"props":21922,"children":21923},{},[21924,21946],{"type":150,"tag":201,"props":21925,"children":21926},{},[21927,21938,21942],{"type":150,"tag":205,"props":21928,"children":21929},{},[21930,21934],{"type":150,"tag":209,"props":21931,"children":21932},{},[21933],{"type":155,"value":578},{"type":150,"tag":209,"props":21935,"children":21936},{},[21937],{"type":155,"value":20412},{"type":150,"tag":221,"props":21939,"children":21940},{},[21941],{"type":155,"value":865},{"type":150,"tag":215,"props":21943,"children":21944},{},[21945],{"type":155,"value":20641},{"type":150,"tag":271,"props":21947,"children":21948},{"encoding":273},[21949],{"type":155,"value":21950},"d_r = 64",{"type":150,"tag":181,"props":21952,"children":21954},{"className":21953,"ariaHidden":223},[280],[21955,22033],{"type":150,"tag":181,"props":21956,"children":21958},{"className":21957},[285],[21959,21963,22020,22024,22029],{"type":150,"tag":181,"props":21960,"children":21962},{"className":21961,"style":4761},[290],[],{"type":150,"tag":181,"props":21964,"children":21966},{"className":21965},[296],[21967,21972],{"type":150,"tag":181,"props":21968,"children":21970},{"className":21969},[296,301],[21971],{"type":155,"value":578},{"type":150,"tag":181,"props":21973,"children":21975},{"className":21974},[307],[21976],{"type":150,"tag":181,"props":21977,"children":21979},{"className":21978},[312,313],[21980,22009],{"type":150,"tag":181,"props":21981,"children":21983},{"className":21982},[318],[21984,22004],{"type":150,"tag":181,"props":21985,"children":21987},{"className":21986,"style":4865},[323],[21988],{"type":150,"tag":181,"props":21989,"children":21990},{"style":328},[21991,21995],{"type":150,"tag":181,"props":21992,"children":21994},{"className":21993,"style":334},[333],[],{"type":150,"tag":181,"props":21996,"children":21998},{"className":21997},[339,340,341,342],[21999],{"type":150,"tag":181,"props":22000,"children":22002},{"className":22001,"style":6794},[296,301,342],[22003],{"type":155,"value":20412},{"type":150,"tag":181,"props":22005,"children":22007},{"className":22006},[352],[22008],{"type":155,"value":355},{"type":150,"tag":181,"props":22010,"children":22012},{"className":22011},[318],[22013],{"type":150,"tag":181,"props":22014,"children":22016},{"className":22015,"style":363},[323],[22017],{"type":150,"tag":181,"props":22018,"children":22019},{},[],{"type":150,"tag":181,"props":22021,"children":22023},{"className":22022,"style":658},[377],[],{"type":150,"tag":181,"props":22025,"children":22027},{"className":22026},[663],[22028],{"type":155,"value":865},{"type":150,"tag":181,"props":22030,"children":22032},{"className":22031,"style":658},[377],[],{"type":150,"tag":181,"props":22034,"children":22036},{"className":22035},[285],[22037,22041],{"type":150,"tag":181,"props":22038,"children":22040},{"className":22039,"style":5012},[290],[],{"type":150,"tag":181,"props":22042,"children":22044},{"className":22043},[296],[22045],{"type":155,"value":20641},{"type":155,"value":22047},"，BF16 存储，所以每参数 ",{"type":150,"tag":181,"props":22049,"children":22051},{"className":22050},[184],[22052,22083],{"type":150,"tag":181,"props":22053,"children":22055},{"className":22054},[189],[22056],{"type":150,"tag":192,"props":22057,"children":22058},{"xmlns":194},[22059],{"type":150,"tag":197,"props":22060,"children":22061},{},[22062,22078],{"type":150,"tag":201,"props":22063,"children":22064},{},[22065,22070,22074],{"type":150,"tag":209,"props":22066,"children":22067},{},[22068],{"type":155,"value":22069},"b",{"type":150,"tag":221,"props":22071,"children":22072},{},[22073],{"type":155,"value":865},{"type":150,"tag":215,"props":22075,"children":22076},{},[22077],{"type":155,"value":238},{"type":150,"tag":271,"props":22079,"children":22080},{"encoding":273},[22081],{"type":155,"value":22082},"b = 2",{"type":150,"tag":181,"props":22084,"children":22086},{"className":22085,"ariaHidden":223},[280],[22087,22113],{"type":150,"tag":181,"props":22088,"children":22090},{"className":22089},[285],[22091,22095,22100,22104,22109],{"type":150,"tag":181,"props":22092,"children":22094},{"className":22093,"style":761},[290],[],{"type":150,"tag":181,"props":22096,"children":22098},{"className":22097},[296,301],[22099],{"type":155,"value":22069},{"type":150,"tag":181,"props":22101,"children":22103},{"className":22102,"style":658},[377],[],{"type":150,"tag":181,"props":22105,"children":22107},{"className":22106},[663],[22108],{"type":155,"value":865},{"type":150,"tag":181,"props":22110,"children":22112},{"className":22111,"style":658},[377],[],{"type":150,"tag":181,"props":22114,"children":22116},{"className":22115},[285],[22117,22121],{"type":150,"tag":181,"props":22118,"children":22120},{"className":22119,"style":5012},[290],[],{"type":150,"tag":181,"props":22122,"children":22124},{"className":22123},[296],[22125],{"type":155,"value":238},{"type":155,"value":22127}," 字节。decode 到第 ",{"type":150,"tag":181,"props":22129,"children":22131},{"className":22130},[184],[22132,22153],{"type":150,"tag":181,"props":22133,"children":22135},{"className":22134},[189],[22136],{"type":150,"tag":192,"props":22137,"children":22138},{"xmlns":194},[22139],{"type":150,"tag":197,"props":22140,"children":22141},{},[22142,22149],{"type":150,"tag":201,"props":22143,"children":22144},{},[22145],{"type":150,"tag":209,"props":22146,"children":22147},{},[22148],{"type":155,"value":5417},{"type":150,"tag":271,"props":22150,"children":22151},{"encoding":273},[22152],{"type":155,"value":5417},{"type":150,"tag":181,"props":22154,"children":22156},{"className":22155,"ariaHidden":223},[280],[22157],{"type":150,"tag":181,"props":22158,"children":22160},{"className":22159},[285],[22161,22165],{"type":150,"tag":181,"props":22162,"children":22164},{"className":22163,"style":5680},[290],[],{"type":150,"tag":181,"props":22166,"children":22168},{"className":22167,"style":5606},[296,301],[22169],{"type":155,"value":5417},{"type":155,"value":22171}," 个位置时，每 token 每层：",{"type":150,"tag":22173,"props":22174,"children":22175},"table",{},[22176,22198],{"type":150,"tag":22177,"props":22178,"children":22179},"thead",{},[22180],{"type":150,"tag":22181,"props":22182,"children":22183},"tr",{},[22184,22188,22193],{"type":150,"tag":22185,"props":22186,"children":22187},"th",{},[],{"type":150,"tag":22185,"props":22189,"children":22190},{},[22191],{"type":155,"value":22192},"KV 读取",{"type":150,"tag":22185,"props":22194,"children":22195},{},[22196],{"type":155,"value":22197},"attention FLOPs",{"type":150,"tag":22199,"props":22200,"children":22201},"tbody",{},[22202,22485],{"type":150,"tag":22181,"props":22203,"children":22204},{},[22205,22211,22352],{"type":150,"tag":22206,"props":22207,"children":22208},"td",{},[22209],{"type":155,"value":22210},"MHA",{"type":150,"tag":22206,"props":22212,"children":22213},{},[22214],{"type":150,"tag":181,"props":22215,"children":22217},{"className":22216},[184],[22218,22263],{"type":150,"tag":181,"props":22219,"children":22221},{"className":22220},[189],[22222],{"type":150,"tag":192,"props":22223,"children":22224},{"xmlns":194},[22225],{"type":150,"tag":197,"props":22226,"children":22227},{},[22228,22258],{"type":150,"tag":201,"props":22229,"children":22230},{},[22231,22235,22239,22250,22254],{"type":150,"tag":215,"props":22232,"children":22233},{},[22234],{"type":155,"value":238},{"type":150,"tag":209,"props":22236,"children":22237},{},[22238],{"type":155,"value":745},{"type":150,"tag":205,"props":22240,"children":22241},{},[22242,22246],{"type":150,"tag":209,"props":22243,"children":22244},{},[22245],{"type":155,"value":578},{"type":150,"tag":209,"props":22247,"children":22248},{},[22249],{"type":155,"value":1698},{"type":150,"tag":209,"props":22251,"children":22252},{},[22253],{"type":155,"value":5417},{"type":150,"tag":209,"props":22255,"children":22256},{},[22257],{"type":155,"value":22069},{"type":150,"tag":271,"props":22259,"children":22260},{"encoding":273},[22261],{"type":155,"value":22262},"2 h d_k N b",{"type":150,"tag":181,"props":22264,"children":22266},{"className":22265,"ariaHidden":223},[280],[22267],{"type":150,"tag":181,"props":22268,"children":22270},{"className":22269},[285],[22271,22275,22280,22285,22342,22347],{"type":150,"tag":181,"props":22272,"children":22274},{"className":22273,"style":4761},[290],[],{"type":150,"tag":181,"props":22276,"children":22278},{"className":22277},[296],[22279],{"type":155,"value":238},{"type":150,"tag":181,"props":22281,"children":22283},{"className":22282},[296,301],[22284],{"type":155,"value":745},{"type":150,"tag":181,"props":22286,"children":22288},{"className":22287},[296],[22289,22294],{"type":150,"tag":181,"props":22290,"children":22292},{"className":22291},[296,301],[22293],{"type":155,"value":578},{"type":150,"tag":181,"props":22295,"children":22297},{"className":22296},[307],[22298],{"type":150,"tag":181,"props":22299,"children":22301},{"className":22300},[312,313],[22302,22331],{"type":150,"tag":181,"props":22303,"children":22305},{"className":22304},[318],[22306,22326],{"type":150,"tag":181,"props":22307,"children":22309},{"className":22308,"style":493},[323],[22310],{"type":150,"tag":181,"props":22311,"children":22312},{"style":328},[22313,22317],{"type":150,"tag":181,"props":22314,"children":22316},{"className":22315,"style":334},[333],[],{"type":150,"tag":181,"props":22318,"children":22320},{"className":22319},[339,340,341,342],[22321],{"type":150,"tag":181,"props":22322,"children":22324},{"className":22323,"style":2249},[296,301,342],[22325],{"type":155,"value":1698},{"type":150,"tag":181,"props":22327,"children":22329},{"className":22328},[352],[22330],{"type":155,"value":355},{"type":150,"tag":181,"props":22332,"children":22334},{"className":22333},[318],[22335],{"type":150,"tag":181,"props":22336,"children":22338},{"className":22337,"style":363},[323],[22339],{"type":150,"tag":181,"props":22340,"children":22341},{},[],{"type":150,"tag":181,"props":22343,"children":22345},{"className":22344,"style":5606},[296,301],[22346],{"type":155,"value":5417},{"type":150,"tag":181,"props":22348,"children":22350},{"className":22349},[296,301],[22351],{"type":155,"value":22069},{"type":150,"tag":22206,"props":22353,"children":22354},{},[22355],{"type":150,"tag":181,"props":22356,"children":22358},{"className":22357},[184],[22359,22401],{"type":150,"tag":181,"props":22360,"children":22362},{"className":22361},[189],[22363],{"type":150,"tag":192,"props":22364,"children":22365},{"xmlns":194},[22366],{"type":150,"tag":197,"props":22367,"children":22368},{},[22369,22396],{"type":150,"tag":201,"props":22370,"children":22371},{},[22372,22377,22381,22392],{"type":150,"tag":215,"props":22373,"children":22374},{},[22375],{"type":155,"value":22376},"4",{"type":150,"tag":209,"props":22378,"children":22379},{},[22380],{"type":155,"value":745},{"type":150,"tag":205,"props":22382,"children":22383},{},[22384,22388],{"type":150,"tag":209,"props":22385,"children":22386},{},[22387],{"type":155,"value":578},{"type":150,"tag":209,"props":22389,"children":22390},{},[22391],{"type":155,"value":1698},{"type":150,"tag":209,"props":22393,"children":22394},{},[22395],{"type":155,"value":5417},{"type":150,"tag":271,"props":22397,"children":22398},{"encoding":273},[22399],{"type":155,"value":22400},"4 h d_k N",{"type":150,"tag":181,"props":22402,"children":22404},{"className":22403,"ariaHidden":223},[280],[22405],{"type":150,"tag":181,"props":22406,"children":22408},{"className":22407},[285],[22409,22413,22418,22423,22480],{"type":150,"tag":181,"props":22410,"children":22412},{"className":22411,"style":4761},[290],[],{"type":150,"tag":181,"props":22414,"children":22416},{"className":22415},[296],[22417],{"type":155,"value":22376},{"type":150,"tag":181,"props":22419,"children":22421},{"className":22420},[296,301],[22422],{"type":155,"value":745},{"type":150,"tag":181,"props":22424,"children":22426},{"className":22425},[296],[22427,22432],{"type":150,"tag":181,"props":22428,"children":22430},{"className":22429},[296,301],[22431],{"type":155,"value":578},{"type":150,"tag":181,"props":22433,"children":22435},{"className":22434},[307],[22436],{"type":150,"tag":181,"props":22437,"children":22439},{"className":22438},[312,313],[22440,22469],{"type":150,"tag":181,"props":22441,"children":22443},{"className":22442},[318],[22444,22464],{"type":150,"tag":181,"props":22445,"children":22447},{"className":22446,"style":493},[323],[22448],{"type":150,"tag":181,"props":22449,"children":22450},{"style":328},[22451,22455],{"type":150,"tag":181,"props":22452,"children":22454},{"className":22453,"style":334},[333],[],{"type":150,"tag":181,"props":22456,"children":22458},{"className":22457},[339,340,341,342],[22459],{"type":150,"tag":181,"props":22460,"children":22462},{"className":22461,"style":2249},[296,301,342],[22463],{"type":155,"value":1698},{"type":150,"tag":181,"props":22465,"children":22467},{"className":22466},[352],[22468],{"type":155,"value":355},{"type":150,"tag":181,"props":22470,"children":22472},{"className":22471},[318],[22473],{"type":150,"tag":181,"props":22474,"children":22476},{"className":22475,"style":363},[323],[22477],{"type":150,"tag":181,"props":22478,"children":22479},{},[],{"type":150,"tag":181,"props":22481,"children":22483},{"className":22482,"style":5606},[296,301],[22484],{"type":155,"value":5417},{"type":150,"tag":22181,"props":22486,"children":22487},{},[22488,22493,22727],{"type":150,"tag":22206,"props":22489,"children":22490},{},[22491],{"type":155,"value":22492},"MLA decode",{"type":150,"tag":22206,"props":22494,"children":22495},{},[22496],{"type":150,"tag":181,"props":22497,"children":22499},{"className":22498},[184],[22500,22560],{"type":150,"tag":181,"props":22501,"children":22503},{"className":22502},[189],[22504],{"type":150,"tag":192,"props":22505,"children":22506},{"xmlns":194},[22507],{"type":150,"tag":197,"props":22508,"children":22509},{},[22510,22555],{"type":150,"tag":201,"props":22511,"children":22512},{},[22513,22517,22528,22532,22543,22547,22551],{"type":150,"tag":221,"props":22514,"children":22515},{"stretchy":789},[22516],{"type":155,"value":792},{"type":150,"tag":205,"props":22518,"children":22519},{},[22520,22524],{"type":150,"tag":209,"props":22521,"children":22522},{},[22523],{"type":155,"value":578},{"type":150,"tag":209,"props":22525,"children":22526},{},[22527],{"type":155,"value":9859},{"type":150,"tag":221,"props":22529,"children":22530},{},[22531],{"type":155,"value":4962},{"type":150,"tag":205,"props":22533,"children":22534},{},[22535,22539],{"type":150,"tag":209,"props":22536,"children":22537},{},[22538],{"type":155,"value":578},{"type":150,"tag":209,"props":22540,"children":22541},{},[22542],{"type":155,"value":20412},{"type":150,"tag":221,"props":22544,"children":22545},{"stretchy":789},[22546],{"type":155,"value":802},{"type":150,"tag":209,"props":22548,"children":22549},{},[22550],{"type":155,"value":5417},{"type":150,"tag":209,"props":22552,"children":22553},{},[22554],{"type":155,"value":22069},{"type":150,"tag":271,"props":22556,"children":22557},{"encoding":273},[22558],{"type":155,"value":22559},"(d_c + d_r) N b",{"type":150,"tag":181,"props":22561,"children":22563},{"className":22562,"ariaHidden":223},[280],[22564,22647],{"type":150,"tag":181,"props":22565,"children":22567},{"className":22566},[285],[22568,22572,22577,22634,22638,22643],{"type":150,"tag":181,"props":22569,"children":22571},{"className":22570,"style":819},[290],[],{"type":150,"tag":181,"props":22573,"children":22575},{"className":22574},[824],[22576],{"type":155,"value":792},{"type":150,"tag":181,"props":22578,"children":22580},{"className":22579},[296],[22581,22586],{"type":150,"tag":181,"props":22582,"children":22584},{"className":22583},[296,301],[22585],{"type":155,"value":578},{"type":150,"tag":181,"props":22587,"children":22589},{"className":22588},[307],[22590],{"type":150,"tag":181,"props":22591,"children":22593},{"className":22592},[312,313],[22594,22623],{"type":150,"tag":181,"props":22595,"children":22597},{"className":22596},[318],[22598,22618],{"type":150,"tag":181,"props":22599,"children":22601},{"className":22600,"style":4865},[323],[22602],{"type":150,"tag":181,"props":22603,"children":22604},{"style":328},[22605,22609],{"type":150,"tag":181,"props":22606,"children":22608},{"className":22607,"style":334},[333],[],{"type":150,"tag":181,"props":22610,"children":22612},{"className":22611},[339,340,341,342],[22613],{"type":150,"tag":181,"props":22614,"children":22616},{"className":22615},[296,301,342],[22617],{"type":155,"value":9859},{"type":150,"tag":181,"props":22619,"children":22621},{"className":22620},[352],[22622],{"type":155,"value":355},{"type":150,"tag":181,"props":22624,"children":22626},{"className":22625},[318],[22627],{"type":150,"tag":181,"props":22628,"children":22630},{"className":22629,"style":363},[323],[22631],{"type":150,"tag":181,"props":22632,"children":22633},{},[],{"type":150,"tag":181,"props":22635,"children":22637},{"className":22636,"style":4993},[377],[],{"type":150,"tag":181,"props":22639,"children":22641},{"className":22640},[4998],[22642],{"type":155,"value":4962},{"type":150,"tag":181,"props":22644,"children":22646},{"className":22645,"style":4993},[377],[],{"type":150,"tag":181,"props":22648,"children":22650},{"className":22649},[285],[22651,22655,22712,22717,22722],{"type":150,"tag":181,"props":22652,"children":22654},{"className":22653,"style":819},[290],[],{"type":150,"tag":181,"props":22656,"children":22658},{"className":22657},[296],[22659,22664],{"type":150,"tag":181,"props":22660,"children":22662},{"className":22661},[296,301],[22663],{"type":155,"value":578},{"type":150,"tag":181,"props":22665,"children":22667},{"className":22666},[307],[22668],{"type":150,"tag":181,"props":22669,"children":22671},{"className":22670},[312,313],[22672,22701],{"type":150,"tag":181,"props":22673,"children":22675},{"className":22674},[318],[22676,22696],{"type":150,"tag":181,"props":22677,"children":22679},{"className":22678,"style":4865},[323],[22680],{"type":150,"tag":181,"props":22681,"children":22682},{"style":328},[22683,22687],{"type":150,"tag":181,"props":22684,"children":22686},{"className":22685,"style":334},[333],[],{"type":150,"tag":181,"props":22688,"children":22690},{"className":22689},[339,340,341,342],[22691],{"type":150,"tag":181,"props":22692,"children":22694},{"className":22693,"style":6794},[296,301,342],[22695],{"type":155,"value":20412},{"type":150,"tag":181,"props":22697,"children":22699},{"className":22698},[352],[22700],{"type":155,"value":355},{"type":150,"tag":181,"props":22702,"children":22704},{"className":22703},[318],[22705],{"type":150,"tag":181,"props":22706,"children":22708},{"className":22707,"style":363},[323],[22709],{"type":150,"tag":181,"props":22710,"children":22711},{},[],{"type":150,"tag":181,"props":22713,"children":22715},{"className":22714},[835],[22716],{"type":155,"value":802},{"type":150,"tag":181,"props":22718,"children":22720},{"className":22719,"style":5606},[296,301],[22721],{"type":155,"value":5417},{"type":150,"tag":181,"props":22723,"children":22725},{"className":22724},[296,301],[22726],{"type":155,"value":22069},{"type":150,"tag":22206,"props":22728,"children":22729},{},[22730],{"type":150,"tag":181,"props":22731,"children":22733},{"className":22732},[184],[22734,22825],{"type":150,"tag":181,"props":22735,"children":22737},{"className":22736},[189],[22738],{"type":150,"tag":192,"props":22739,"children":22740},{"xmlns":194},[22741],{"type":150,"tag":197,"props":22742,"children":22743},{},[22744,22820],{"type":150,"tag":201,"props":22745,"children":22746},{},[22747,22751,22755,22759,22770,22774,22785,22789,22793,22797,22801,22805,22816],{"type":150,"tag":215,"props":22748,"children":22749},{},[22750],{"type":155,"value":238},{"type":150,"tag":209,"props":22752,"children":22753},{},[22754],{"type":155,"value":745},{"type":150,"tag":221,"props":22756,"children":22757},{"stretchy":789},[22758],{"type":155,"value":792},{"type":150,"tag":205,"props":22760,"children":22761},{},[22762,22766],{"type":150,"tag":209,"props":22763,"children":22764},{},[22765],{"type":155,"value":578},{"type":150,"tag":209,"props":22767,"children":22768},{},[22769],{"type":155,"value":9859},{"type":150,"tag":221,"props":22771,"children":22772},{},[22773],{"type":155,"value":4962},{"type":150,"tag":205,"props":22775,"children":22776},{},[22777,22781],{"type":150,"tag":209,"props":22778,"children":22779},{},[22780],{"type":155,"value":578},{"type":150,"tag":209,"props":22782,"children":22783},{},[22784],{"type":155,"value":20412},{"type":150,"tag":221,"props":22786,"children":22787},{"stretchy":789},[22788],{"type":155,"value":802},{"type":150,"tag":209,"props":22790,"children":22791},{},[22792],{"type":155,"value":5417},{"type":150,"tag":221,"props":22794,"children":22795},{},[22796],{"type":155,"value":4962},{"type":150,"tag":215,"props":22798,"children":22799},{},[22800],{"type":155,"value":238},{"type":150,"tag":209,"props":22802,"children":22803},{},[22804],{"type":155,"value":745},{"type":150,"tag":205,"props":22806,"children":22807},{},[22808,22812],{"type":150,"tag":209,"props":22809,"children":22810},{},[22811],{"type":155,"value":578},{"type":150,"tag":209,"props":22813,"children":22814},{},[22815],{"type":155,"value":9859},{"type":150,"tag":209,"props":22817,"children":22818},{},[22819],{"type":155,"value":5417},{"type":150,"tag":271,"props":22821,"children":22822},{"encoding":273},[22823],{"type":155,"value":22824},"2h(d_c+d_r)N + 2hd_cN",{"type":150,"tag":181,"props":22826,"children":22828},{"className":22827,"ariaHidden":223},[280],[22829,22922,23010],{"type":150,"tag":181,"props":22830,"children":22832},{"className":22831},[285],[22833,22837,22842,22847,22852,22909,22913,22918],{"type":150,"tag":181,"props":22834,"children":22836},{"className":22835,"style":819},[290],[],{"type":150,"tag":181,"props":22838,"children":22840},{"className":22839},[296],[22841],{"type":155,"value":238},{"type":150,"tag":181,"props":22843,"children":22845},{"className":22844},[296,301],[22846],{"type":155,"value":745},{"type":150,"tag":181,"props":22848,"children":22850},{"className":22849},[824],[22851],{"type":155,"value":792},{"type":150,"tag":181,"props":22853,"children":22855},{"className":22854},[296],[22856,22861],{"type":150,"tag":181,"props":22857,"children":22859},{"className":22858},[296,301],[22860],{"type":155,"value":578},{"type":150,"tag":181,"props":22862,"children":22864},{"className":22863},[307],[22865],{"type":150,"tag":181,"props":22866,"children":22868},{"className":22867},[312,313],[22869,22898],{"type":150,"tag":181,"props":22870,"children":22872},{"className":22871},[318],[22873,22893],{"type":150,"tag":181,"props":22874,"children":22876},{"className":22875,"style":4865},[323],[22877],{"type":150,"tag":181,"props":22878,"children":22879},{"style":328},[22880,22884],{"type":150,"tag":181,"props":22881,"children":22883},{"className":22882,"style":334},[333],[],{"type":150,"tag":181,"props":22885,"children":22887},{"className":22886},[339,340,341,342],[22888],{"type":150,"tag":181,"props":22889,"children":22891},{"className":22890},[296,301,342],[22892],{"type":155,"value":9859},{"type":150,"tag":181,"props":22894,"children":22896},{"className":22895},[352],[22897],{"type":155,"value":355},{"type":150,"tag":181,"props":22899,"children":22901},{"className":22900},[318],[22902],{"type":150,"tag":181,"props":22903,"children":22905},{"className":22904,"style":363},[323],[22906],{"type":150,"tag":181,"props":22907,"children":22908},{},[],{"type":150,"tag":181,"props":22910,"children":22912},{"className":22911,"style":4993},[377],[],{"type":150,"tag":181,"props":22914,"children":22916},{"className":22915},[4998],[22917],{"type":155,"value":4962},{"type":150,"tag":181,"props":22919,"children":22921},{"className":22920,"style":4993},[377],[],{"type":150,"tag":181,"props":22923,"children":22925},{"className":22924},[285],[22926,22930,22987,22992,22997,23001,23006],{"type":150,"tag":181,"props":22927,"children":22929},{"className":22928,"style":819},[290],[],{"type":150,"tag":181,"props":22931,"children":22933},{"className":22932},[296],[22934,22939],{"type":150,"tag":181,"props":22935,"children":22937},{"className":22936},[296,301],[22938],{"type":155,"value":578},{"type":150,"tag":181,"props":22940,"children":22942},{"className":22941},[307],[22943],{"type":150,"tag":181,"props":22944,"children":22946},{"className":22945},[312,313],[22947,22976],{"type":150,"tag":181,"props":22948,"children":22950},{"className":22949},[318],[22951,22971],{"type":150,"tag":181,"props":22952,"children":22954},{"className":22953,"style":4865},[323],[22955],{"type":150,"tag":181,"props":22956,"children":22957},{"style":328},[22958,22962],{"type":150,"tag":181,"props":22959,"children":22961},{"className":22960,"style":334},[333],[],{"type":150,"tag":181,"props":22963,"children":22965},{"className":22964},[339,340,341,342],[22966],{"type":150,"tag":181,"props":22967,"children":22969},{"className":22968,"style":6794},[296,301,342],[22970],{"type":155,"value":20412},{"type":150,"tag":181,"props":22972,"children":22974},{"className":22973},[352],[22975],{"type":155,"value":355},{"type":150,"tag":181,"props":22977,"children":22979},{"className":22978},[318],[22980],{"type":150,"tag":181,"props":22981,"children":22983},{"className":22982,"style":363},[323],[22984],{"type":150,"tag":181,"props":22985,"children":22986},{},[],{"type":150,"tag":181,"props":22988,"children":22990},{"className":22989},[835],[22991],{"type":155,"value":802},{"type":150,"tag":181,"props":22993,"children":22995},{"className":22994,"style":5606},[296,301],[22996],{"type":155,"value":5417},{"type":150,"tag":181,"props":22998,"children":23000},{"className":22999,"style":4993},[377],[],{"type":150,"tag":181,"props":23002,"children":23004},{"className":23003},[4998],[23005],{"type":155,"value":4962},{"type":150,"tag":181,"props":23007,"children":23009},{"className":23008,"style":4993},[377],[],{"type":150,"tag":181,"props":23011,"children":23013},{"className":23012},[285],[23014,23018,23023,23028,23085],{"type":150,"tag":181,"props":23015,"children":23017},{"className":23016,"style":4761},[290],[],{"type":150,"tag":181,"props":23019,"children":23021},{"className":23020},[296],[23022],{"type":155,"value":238},{"type":150,"tag":181,"props":23024,"children":23026},{"className":23025},[296,301],[23027],{"type":155,"value":745},{"type":150,"tag":181,"props":23029,"children":23031},{"className":23030},[296],[23032,23037],{"type":150,"tag":181,"props":23033,"children":23035},{"className":23034},[296,301],[23036],{"type":155,"value":578},{"type":150,"tag":181,"props":23038,"children":23040},{"className":23039},[307],[23041],{"type":150,"tag":181,"props":23042,"children":23044},{"className":23043},[312,313],[23045,23074],{"type":150,"tag":181,"props":23046,"children":23048},{"className":23047},[318],[23049,23069],{"type":150,"tag":181,"props":23050,"children":23052},{"className":23051,"style":4865},[323],[23053],{"type":150,"tag":181,"props":23054,"children":23055},{"style":328},[23056,23060],{"type":150,"tag":181,"props":23057,"children":23059},{"className":23058,"style":334},[333],[],{"type":150,"tag":181,"props":23061,"children":23063},{"className":23062},[339,340,341,342],[23064],{"type":150,"tag":181,"props":23065,"children":23067},{"className":23066},[296,301,342],[23068],{"type":155,"value":9859},{"type":150,"tag":181,"props":23070,"children":23072},{"className":23071},[352],[23073],{"type":155,"value":355},{"type":150,"tag":181,"props":23075,"children":23077},{"className":23076},[318],[23078],{"type":150,"tag":181,"props":23079,"children":23081},{"className":23080,"style":363},[323],[23082],{"type":150,"tag":181,"props":23083,"children":23084},{},[],{"type":150,"tag":181,"props":23086,"children":23088},{"className":23087,"style":5606},[296,301],[23089],{"type":155,"value":5417},{"type":150,"tag":151,"props":23091,"children":23092},{},[23093],{"type":155,"value":23094},"代进数字：",{"type":150,"tag":22173,"props":23096,"children":23097},{},[23098,23120],{"type":150,"tag":22177,"props":23099,"children":23100},{},[23101],{"type":150,"tag":22181,"props":23102,"children":23103},{},[23104,23107,23111,23115],{"type":150,"tag":22185,"props":23105,"children":23106},{},[],{"type":150,"tag":22185,"props":23108,"children":23109},{},[23110],{"type":155,"value":22210},{"type":150,"tag":22185,"props":23112,"children":23113},{},[23114],{"type":155,"value":13},{"type":150,"tag":22185,"props":23116,"children":23117},{},[23118],{"type":155,"value":23119},"比值",{"type":150,"tag":22199,"props":23121,"children":23122},{},[23123,23254],{"type":150,"tag":22181,"props":23124,"children":23125},{},[23126,23131,23189,23246],{"type":150,"tag":22206,"props":23127,"children":23128},{},[23129],{"type":155,"value":23130},"KV 读取 \u002F 层 \u002F token",{"type":150,"tag":22206,"props":23132,"children":23133},{},[23134,23187],{"type":150,"tag":181,"props":23135,"children":23137},{"className":23136},[184],[23138,23165],{"type":150,"tag":181,"props":23139,"children":23141},{"className":23140},[189],[23142],{"type":150,"tag":192,"props":23143,"children":23144},{"xmlns":194},[23145],{"type":150,"tag":197,"props":23146,"children":23147},{},[23148,23160],{"type":150,"tag":201,"props":23149,"children":23150},{},[23151,23156],{"type":150,"tag":215,"props":23152,"children":23153},{},[23154],{"type":155,"value":23155},"65536",{"type":150,"tag":209,"props":23157,"children":23158},{},[23159],{"type":155,"value":5417},{"type":150,"tag":271,"props":23161,"children":23162},{"encoding":273},[23163],{"type":155,"value":23164},"65536N",{"type":150,"tag":181,"props":23166,"children":23168},{"className":23167,"ariaHidden":223},[280],[23169],{"type":150,"tag":181,"props":23170,"children":23172},{"className":23171},[285],[23173,23177,23182],{"type":150,"tag":181,"props":23174,"children":23176},{"className":23175,"style":5680},[290],[],{"type":150,"tag":181,"props":23178,"children":23180},{"className":23179},[296],[23181],{"type":155,"value":23155},{"type":150,"tag":181,"props":23183,"children":23185},{"className":23184,"style":5606},[296,301],[23186],{"type":155,"value":5417},{"type":155,"value":23188},"B",{"type":150,"tag":22206,"props":23190,"children":23191},{},[23192,23245],{"type":150,"tag":181,"props":23193,"children":23195},{"className":23194},[184],[23196,23223],{"type":150,"tag":181,"props":23197,"children":23199},{"className":23198},[189],[23200],{"type":150,"tag":192,"props":23201,"children":23202},{"xmlns":194},[23203],{"type":150,"tag":197,"props":23204,"children":23205},{},[23206,23218],{"type":150,"tag":201,"props":23207,"children":23208},{},[23209,23214],{"type":150,"tag":215,"props":23210,"children":23211},{},[23212],{"type":155,"value":23213},"1152",{"type":150,"tag":209,"props":23215,"children":23216},{},[23217],{"type":155,"value":5417},{"type":150,"tag":271,"props":23219,"children":23220},{"encoding":273},[23221],{"type":155,"value":23222},"1152N",{"type":150,"tag":181,"props":23224,"children":23226},{"className":23225,"ariaHidden":223},[280],[23227],{"type":150,"tag":181,"props":23228,"children":23230},{"className":23229},[285],[23231,23235,23240],{"type":150,"tag":181,"props":23232,"children":23234},{"className":23233,"style":5680},[290],[],{"type":150,"tag":181,"props":23236,"children":23238},{"className":23237},[296],[23239],{"type":155,"value":23213},{"type":150,"tag":181,"props":23241,"children":23243},{"className":23242,"style":5606},[296,301],[23244],{"type":155,"value":5417},{"type":155,"value":23188},{"type":150,"tag":22206,"props":23247,"children":23248},{},[23249],{"type":150,"tag":158,"props":23250,"children":23251},{},[23252],{"type":155,"value":23253},"1 : 56.9",{"type":150,"tag":22181,"props":23255,"children":23256},{},[23257,23262,23316,23372],{"type":150,"tag":22206,"props":23258,"children":23259},{},[23260],{"type":155,"value":23261},"attention FLOPs \u002F 层 \u002F token",{"type":150,"tag":22206,"props":23263,"children":23264},{},[23265],{"type":150,"tag":181,"props":23266,"children":23268},{"className":23267},[184],[23269,23294],{"type":150,"tag":181,"props":23270,"children":23272},{"className":23271},[189],[23273],{"type":150,"tag":192,"props":23274,"children":23275},{"xmlns":194},[23276],{"type":150,"tag":197,"props":23277,"children":23278},{},[23279,23290],{"type":150,"tag":201,"props":23280,"children":23281},{},[23282,23286],{"type":150,"tag":215,"props":23283,"children":23284},{},[23285],{"type":155,"value":23155},{"type":150,"tag":209,"props":23287,"children":23288},{},[23289],{"type":155,"value":5417},{"type":150,"tag":271,"props":23291,"children":23292},{"encoding":273},[23293],{"type":155,"value":23164},{"type":150,"tag":181,"props":23295,"children":23297},{"className":23296,"ariaHidden":223},[280],[23298],{"type":150,"tag":181,"props":23299,"children":23301},{"className":23300},[285],[23302,23306,23311],{"type":150,"tag":181,"props":23303,"children":23305},{"className":23304,"style":5680},[290],[],{"type":150,"tag":181,"props":23307,"children":23309},{"className":23308},[296],[23310],{"type":155,"value":23155},{"type":150,"tag":181,"props":23312,"children":23314},{"className":23313,"style":5606},[296,301],[23315],{"type":155,"value":5417},{"type":150,"tag":22206,"props":23317,"children":23318},{},[23319],{"type":150,"tag":181,"props":23320,"children":23322},{"className":23321},[184],[23323,23350],{"type":150,"tag":181,"props":23324,"children":23326},{"className":23325},[189],[23327],{"type":150,"tag":192,"props":23328,"children":23329},{"xmlns":194},[23330],{"type":150,"tag":197,"props":23331,"children":23332},{},[23333,23345],{"type":150,"tag":201,"props":23334,"children":23335},{},[23336,23341],{"type":150,"tag":215,"props":23337,"children":23338},{},[23339],{"type":155,"value":23340},"278528",{"type":150,"tag":209,"props":23342,"children":23343},{},[23344],{"type":155,"value":5417},{"type":150,"tag":271,"props":23346,"children":23347},{"encoding":273},[23348],{"type":155,"value":23349},"278528N",{"type":150,"tag":181,"props":23351,"children":23353},{"className":23352,"ariaHidden":223},[280],[23354],{"type":150,"tag":181,"props":23355,"children":23357},{"className":23356},[285],[23358,23362,23367],{"type":150,"tag":181,"props":23359,"children":23361},{"className":23360,"style":5680},[290],[],{"type":150,"tag":181,"props":23363,"children":23365},{"className":23364},[296],[23366],{"type":155,"value":23340},{"type":150,"tag":181,"props":23368,"children":23370},{"className":23369,"style":5606},[296,301],[23371],{"type":155,"value":5417},{"type":150,"tag":22206,"props":23373,"children":23374},{},[23375],{"type":150,"tag":158,"props":23376,"children":23377},{},[23378],{"type":155,"value":23379},"4.25 : 1",{"type":150,"tag":151,"props":23381,"children":23382},{},[23383],{"type":155,"value":23384},"用 roofline 分析一下。H100 SXM 的 BF16 算力约 989 TFLOPS，HBM3 带宽 3.35 TB\u002Fs，拐点在：",{"type":150,"tag":181,"props":23386,"children":23388},{"className":23387},[1003],[23389],{"type":150,"tag":181,"props":23390,"children":23392},{"className":23391},[184],[23393,23454],{"type":150,"tag":181,"props":23394,"children":23396},{"className":23395},[189],[23397],{"type":150,"tag":192,"props":23398,"children":23399},{"xmlns":194,"display":1015},[23400],{"type":150,"tag":197,"props":23401,"children":23402},{},[23403,23449],{"type":150,"tag":201,"props":23404,"children":23405},{},[23406,23435,23439,23444],{"type":150,"tag":1774,"props":23407,"children":23408},{},[23409,23422],{"type":150,"tag":201,"props":23410,"children":23411},{},[23412,23417],{"type":150,"tag":215,"props":23413,"children":23414},{},[23415],{"type":155,"value":23416},"989",{"type":150,"tag":249,"props":23418,"children":23419},{},[23420],{"type":155,"value":23421}," TFLOPS",{"type":150,"tag":201,"props":23423,"children":23424},{},[23425,23430],{"type":150,"tag":215,"props":23426,"children":23427},{},[23428],{"type":155,"value":23429},"3.35",{"type":150,"tag":249,"props":23431,"children":23432},{},[23433],{"type":155,"value":23434}," TB\u002Fs",{"type":150,"tag":221,"props":23436,"children":23437},{},[23438],{"type":155,"value":5876},{"type":150,"tag":215,"props":23440,"children":23441},{},[23442],{"type":155,"value":23443},"295",{"type":150,"tag":249,"props":23445,"children":23446},{},[23447],{"type":155,"value":23448}," FLOP\u002Fbyte",{"type":150,"tag":271,"props":23450,"children":23451},{"encoding":273},[23452],{"type":155,"value":23453},"\\frac{989\\ \\text{TFLOPS}}{3.35\\ \\text{TB\u002Fs}} \\approx 295\\ \\text{FLOP\u002Fbyte}",{"type":150,"tag":181,"props":23455,"children":23457},{"className":23456,"ariaHidden":223},[280],[23458,23599],{"type":150,"tag":181,"props":23459,"children":23461},{"className":23460},[285],[23462,23467,23586,23590,23595],{"type":150,"tag":181,"props":23463,"children":23466},{"className":23464,"style":23465},[290],"height:2.2963em;vertical-align:-0.936em;",[],{"type":150,"tag":181,"props":23468,"children":23470},{"className":23469},[296],[23471,23475,23582],{"type":150,"tag":181,"props":23472,"children":23474},{"className":23473},[824,2482],[],{"type":150,"tag":181,"props":23476,"children":23478},{"className":23477},[1774],[23479],{"type":150,"tag":181,"props":23480,"children":23482},{"className":23481},[312,313],[23483,23570],{"type":150,"tag":181,"props":23484,"children":23486},{"className":23485},[318],[23487,23565],{"type":150,"tag":181,"props":23488,"children":23491},{"className":23489,"style":23490},[323],"height:1.3603em;",[23492,23523,23534],{"type":150,"tag":181,"props":23493,"children":23494},{"style":8235},[23495,23499],{"type":150,"tag":181,"props":23496,"children":23498},{"className":23497,"style":8240},[333],[],{"type":150,"tag":181,"props":23500,"children":23502},{"className":23501},[296],[23503,23508,23513],{"type":150,"tag":181,"props":23504,"children":23506},{"className":23505},[296],[23507],{"type":155,"value":23429},{"type":150,"tag":181,"props":23509,"children":23511},{"className":23510},[377],[23512],{"type":155,"value":8576},{"type":150,"tag":181,"props":23514,"children":23516},{"className":23515},[296,155],[23517],{"type":150,"tag":181,"props":23518,"children":23520},{"className":23519},[296],[23521],{"type":155,"value":23522},"TB\u002Fs",{"type":150,"tag":181,"props":23524,"children":23525},{"style":8253},[23526,23530],{"type":150,"tag":181,"props":23527,"children":23529},{"className":23528,"style":8240},[333],[],{"type":150,"tag":181,"props":23531,"children":23533},{"className":23532,"style":2804},[2803],[],{"type":150,"tag":181,"props":23535,"children":23536},{"style":8265},[23537,23541],{"type":150,"tag":181,"props":23538,"children":23540},{"className":23539,"style":8240},[333],[],{"type":150,"tag":181,"props":23542,"children":23544},{"className":23543},[296],[23545,23550,23555],{"type":150,"tag":181,"props":23546,"children":23548},{"className":23547},[296],[23549],{"type":155,"value":23416},{"type":150,"tag":181,"props":23551,"children":23553},{"className":23552},[377],[23554],{"type":155,"value":8576},{"type":150,"tag":181,"props":23556,"children":23558},{"className":23557},[296,155],[23559],{"type":150,"tag":181,"props":23560,"children":23562},{"className":23561},[296],[23563],{"type":155,"value":23564},"TFLOPS",{"type":150,"tag":181,"props":23566,"children":23568},{"className":23567},[352],[23569],{"type":155,"value":355},{"type":150,"tag":181,"props":23571,"children":23573},{"className":23572},[318],[23574],{"type":150,"tag":181,"props":23575,"children":23578},{"className":23576,"style":23577},[323],"height:0.936em;",[23579],{"type":150,"tag":181,"props":23580,"children":23581},{},[],{"type":150,"tag":181,"props":23583,"children":23585},{"className":23584},[835,2482],[],{"type":150,"tag":181,"props":23587,"children":23589},{"className":23588,"style":658},[377],[],{"type":150,"tag":181,"props":23591,"children":23593},{"className":23592},[663],[23594],{"type":155,"value":5876},{"type":150,"tag":181,"props":23596,"children":23598},{"className":23597,"style":658},[377],[],{"type":150,"tag":181,"props":23600,"children":23602},{"className":23601},[285],[23603,23607,23612,23617],{"type":150,"tag":181,"props":23604,"children":23606},{"className":23605,"style":819},[290],[],{"type":150,"tag":181,"props":23608,"children":23610},{"className":23609},[296],[23611],{"type":155,"value":23443},{"type":150,"tag":181,"props":23613,"children":23615},{"className":23614},[377],[23616],{"type":155,"value":8576},{"type":150,"tag":181,"props":23618,"children":23620},{"className":23619},[296,155],[23621],{"type":150,"tag":181,"props":23622,"children":23624},{"className":23623},[296],[23625],{"type":155,"value":23626},"FLOP\u002Fbyte",{"type":150,"tag":151,"props":23628,"children":23629},{},[23630],{"type":155,"value":23631},"算术强度低于 295 就是 memory bound，高于才是 compute bound。再看 MLA decode 这一侧，别忘了每 token 还要把权重读一遍。",{"type":150,"tag":23633,"props":23634,"children":23635},"ul",{},[23636,23684,23765,24197,24681],{"type":150,"tag":23637,"props":23638,"children":23639},"li",{},[23640,23682],{"type":150,"tag":181,"props":23641,"children":23643},{"className":23642},[184],[23644,23665],{"type":150,"tag":181,"props":23645,"children":23647},{"className":23646},[189],[23648],{"type":150,"tag":192,"props":23649,"children":23650},{"xmlns":194},[23651],{"type":150,"tag":197,"props":23652,"children":23653},{},[23654,23661],{"type":150,"tag":201,"props":23655,"children":23656},{},[23657],{"type":150,"tag":209,"props":23658,"children":23659},{},[23660],{"type":155,"value":5417},{"type":150,"tag":271,"props":23662,"children":23663},{"encoding":273},[23664],{"type":155,"value":5417},{"type":150,"tag":181,"props":23666,"children":23668},{"className":23667,"ariaHidden":223},[280],[23669],{"type":150,"tag":181,"props":23670,"children":23672},{"className":23671},[285],[23673,23677],{"type":150,"tag":181,"props":23674,"children":23676},{"className":23675,"style":5680},[290],[],{"type":150,"tag":181,"props":23678,"children":23680},{"className":23679,"style":5606},[296,301],[23681],{"type":155,"value":5417},{"type":155,"value":23683},"：当前 context 长度，也就是 decode 到这一步时要看多少个历史 token",{"type":150,"tag":23637,"props":23685,"children":23686},{},[23687,23763],{"type":150,"tag":181,"props":23688,"children":23690},{"className":23689},[184],[23691,23720],{"type":150,"tag":181,"props":23692,"children":23694},{"className":23693},[189],[23695],{"type":150,"tag":192,"props":23696,"children":23697},{"xmlns":194},[23698],{"type":150,"tag":197,"props":23699,"children":23700},{},[23701,23716],{"type":150,"tag":201,"props":23702,"children":23703},{},[23704,23708,23712],{"type":150,"tag":209,"props":23705,"children":23706},{},[23707],{"type":155,"value":5385},{"type":150,"tag":221,"props":23709,"children":23710},{},[23711],{"type":155,"value":865},{"type":150,"tag":215,"props":23713,"children":23714},{},[23715],{"type":155,"value":21506},{"type":150,"tag":271,"props":23717,"children":23718},{"encoding":273},[23719],{"type":155,"value":21511},{"type":150,"tag":181,"props":23721,"children":23723},{"className":23722,"ariaHidden":223},[280],[23724,23750],{"type":150,"tag":181,"props":23725,"children":23727},{"className":23726},[285],[23728,23732,23737,23741,23746],{"type":150,"tag":181,"props":23729,"children":23731},{"className":23730,"style":5680},[290],[],{"type":150,"tag":181,"props":23733,"children":23735},{"className":23734},[296,301],[23736],{"type":155,"value":5385},{"type":150,"tag":181,"props":23738,"children":23740},{"className":23739,"style":658},[377],[],{"type":150,"tag":181,"props":23742,"children":23744},{"className":23743},[663],[23745],{"type":155,"value":865},{"type":150,"tag":181,"props":23747,"children":23749},{"className":23748,"style":658},[377],[],{"type":150,"tag":181,"props":23751,"children":23753},{"className":23752},[285],[23754,23758],{"type":150,"tag":181,"props":23755,"children":23757},{"className":23756,"style":5012},[290],[],{"type":150,"tag":181,"props":23759,"children":23761},{"className":23760},[296],[23762],{"type":155,"value":21506},{"type":155,"value":23764},"：层数",{"type":150,"tag":23637,"props":23766,"children":23767},{},[23768,24195],{"type":150,"tag":181,"props":23769,"children":23771},{"className":23770},[184],[23772,23869],{"type":150,"tag":181,"props":23773,"children":23775},{"className":23774},[189],[23776],{"type":150,"tag":192,"props":23777,"children":23778},{"xmlns":194},[23779],{"type":150,"tag":197,"props":23780,"children":23781},{},[23782,23864],{"type":150,"tag":201,"props":23783,"children":23784},{},[23785,23797,23801,23805,23816,23820,23831,23835,23839,23843,23848,23852,23856,23860],{"type":150,"tag":205,"props":23786,"children":23787},{},[23788,23792],{"type":150,"tag":209,"props":23789,"children":23790},{},[23791],{"type":155,"value":9859},{"type":150,"tag":249,"props":23793,"children":23794},{},[23795],{"type":155,"value":23796},"kv",{"type":150,"tag":221,"props":23798,"children":23799},{},[23800],{"type":155,"value":865},{"type":150,"tag":221,"props":23802,"children":23803},{"stretchy":789},[23804],{"type":155,"value":792},{"type":150,"tag":205,"props":23806,"children":23807},{},[23808,23812],{"type":150,"tag":209,"props":23809,"children":23810},{},[23811],{"type":155,"value":578},{"type":150,"tag":209,"props":23813,"children":23814},{},[23815],{"type":155,"value":9859},{"type":150,"tag":221,"props":23817,"children":23818},{},[23819],{"type":155,"value":4962},{"type":150,"tag":205,"props":23821,"children":23822},{},[23823,23827],{"type":150,"tag":209,"props":23824,"children":23825},{},[23826],{"type":155,"value":578},{"type":150,"tag":209,"props":23828,"children":23829},{},[23830],{"type":155,"value":20412},{"type":150,"tag":221,"props":23832,"children":23833},{"stretchy":789},[23834],{"type":155,"value":802},{"type":150,"tag":209,"props":23836,"children":23837},{},[23838],{"type":155,"value":22069},{"type":150,"tag":221,"props":23840,"children":23841},{},[23842],{"type":155,"value":865},{"type":150,"tag":215,"props":23844,"children":23845},{},[23846],{"type":155,"value":23847},"576",{"type":150,"tag":221,"props":23849,"children":23850},{},[23851],{"type":155,"value":5380},{"type":150,"tag":215,"props":23853,"children":23854},{},[23855],{"type":155,"value":238},{"type":150,"tag":221,"props":23857,"children":23858},{},[23859],{"type":155,"value":865},{"type":150,"tag":215,"props":23861,"children":23862},{},[23863],{"type":155,"value":23213},{"type":150,"tag":271,"props":23865,"children":23866},{"encoding":273},[23867],{"type":155,"value":23868},"c_{\\text{kv}} = (d_c + d_r)b = 576 \\times 2 = 1152",{"type":150,"tag":181,"props":23870,"children":23872},{"className":23871,"ariaHidden":223},[280],[23873,23959,24042,24130,24156,24182],{"type":150,"tag":181,"props":23874,"children":23876},{"className":23875},[285],[23877,23881,23946,23950,23955],{"type":150,"tag":181,"props":23878,"children":23880},{"className":23879,"style":1165},[290],[],{"type":150,"tag":181,"props":23882,"children":23884},{"className":23883},[296],[23885,23890],{"type":150,"tag":181,"props":23886,"children":23888},{"className":23887},[296,301],[23889],{"type":155,"value":9859},{"type":150,"tag":181,"props":23891,"children":23893},{"className":23892},[307],[23894],{"type":150,"tag":181,"props":23895,"children":23897},{"className":23896},[312,313],[23898,23935],{"type":150,"tag":181,"props":23899,"children":23901},{"className":23900},[318],[23902,23930],{"type":150,"tag":181,"props":23903,"children":23905},{"className":23904,"style":493},[323],[23906],{"type":150,"tag":181,"props":23907,"children":23908},{"style":328},[23909,23913],{"type":150,"tag":181,"props":23910,"children":23912},{"className":23911,"style":334},[333],[],{"type":150,"tag":181,"props":23914,"children":23916},{"className":23915},[339,340,341,342],[23917],{"type":150,"tag":181,"props":23918,"children":23920},{"className":23919},[296,342],[23921],{"type":150,"tag":181,"props":23922,"children":23924},{"className":23923},[296,155,342],[23925],{"type":150,"tag":181,"props":23926,"children":23928},{"className":23927},[296,342],[23929],{"type":155,"value":23796},{"type":150,"tag":181,"props":23931,"children":23933},{"className":23932},[352],[23934],{"type":155,"value":355},{"type":150,"tag":181,"props":23936,"children":23938},{"className":23937},[318],[23939],{"type":150,"tag":181,"props":23940,"children":23942},{"className":23941,"style":363},[323],[23943],{"type":150,"tag":181,"props":23944,"children":23945},{},[],{"type":150,"tag":181,"props":23947,"children":23949},{"className":23948,"style":658},[377],[],{"type":150,"tag":181,"props":23951,"children":23953},{"className":23952},[663],[23954],{"type":155,"value":865},{"type":150,"tag":181,"props":23956,"children":23958},{"className":23957,"style":658},[377],[],{"type":150,"tag":181,"props":23960,"children":23962},{"className":23961},[285],[23963,23967,23972,24029,24033,24038],{"type":150,"tag":181,"props":23964,"children":23966},{"className":23965,"style":819},[290],[],{"type":150,"tag":181,"props":23968,"children":23970},{"className":23969},[824],[23971],{"type":155,"value":792},{"type":150,"tag":181,"props":23973,"children":23975},{"className":23974},[296],[23976,23981],{"type":150,"tag":181,"props":23977,"children":23979},{"className":23978},[296,301],[23980],{"type":155,"value":578},{"type":150,"tag":181,"props":23982,"children":23984},{"className":23983},[307],[23985],{"type":150,"tag":181,"props":23986,"children":23988},{"className":23987},[312,313],[23989,24018],{"type":150,"tag":181,"props":23990,"children":23992},{"className":23991},[318],[23993,24013],{"type":150,"tag":181,"props":23994,"children":23996},{"className":23995,"style":4865},[323],[23997],{"type":150,"tag":181,"props":23998,"children":23999},{"style":328},[24000,24004],{"type":150,"tag":181,"props":24001,"children":24003},{"className":24002,"style":334},[333],[],{"type":150,"tag":181,"props":24005,"children":24007},{"className":24006},[339,340,341,342],[24008],{"type":150,"tag":181,"props":24009,"children":24011},{"className":24010},[296,301,342],[24012],{"type":155,"value":9859},{"type":150,"tag":181,"props":24014,"children":24016},{"className":24015},[352],[24017],{"type":155,"value":355},{"type":150,"tag":181,"props":24019,"children":24021},{"className":24020},[318],[24022],{"type":150,"tag":181,"props":24023,"children":24025},{"className":24024,"style":363},[323],[24026],{"type":150,"tag":181,"props":24027,"children":24028},{},[],{"type":150,"tag":181,"props":24030,"children":24032},{"className":24031,"style":4993},[377],[],{"type":150,"tag":181,"props":24034,"children":24036},{"className":24035},[4998],[24037],{"type":155,"value":4962},{"type":150,"tag":181,"props":24039,"children":24041},{"className":24040,"style":4993},[377],[],{"type":150,"tag":181,"props":24043,"children":24045},{"className":24044},[285],[24046,24050,24107,24112,24117,24121,24126],{"type":150,"tag":181,"props":24047,"children":24049},{"className":24048,"style":819},[290],[],{"type":150,"tag":181,"props":24051,"children":24053},{"className":24052},[296],[24054,24059],{"type":150,"tag":181,"props":24055,"children":24057},{"className":24056},[296,301],[24058],{"type":155,"value":578},{"type":150,"tag":181,"props":24060,"children":24062},{"className":24061},[307],[24063],{"type":150,"tag":181,"props":24064,"children":24066},{"className":24065},[312,313],[24067,24096],{"type":150,"tag":181,"props":24068,"children":24070},{"className":24069},[318],[24071,24091],{"type":150,"tag":181,"props":24072,"children":24074},{"className":24073,"style":4865},[323],[24075],{"type":150,"tag":181,"props":24076,"children":24077},{"style":328},[24078,24082],{"type":150,"tag":181,"props":24079,"children":24081},{"className":24080,"style":334},[333],[],{"type":150,"tag":181,"props":24083,"children":24085},{"className":24084},[339,340,341,342],[24086],{"type":150,"tag":181,"props":24087,"children":24089},{"className":24088,"style":6794},[296,301,342],[24090],{"type":155,"value":20412},{"type":150,"tag":181,"props":24092,"children":24094},{"className":24093},[352],[24095],{"type":155,"value":355},{"type":150,"tag":181,"props":24097,"children":24099},{"className":24098},[318],[24100],{"type":150,"tag":181,"props":24101,"children":24103},{"className":24102,"style":363},[323],[24104],{"type":150,"tag":181,"props":24105,"children":24106},{},[],{"type":150,"tag":181,"props":24108,"children":24110},{"className":24109},[835],[24111],{"type":155,"value":802},{"type":150,"tag":181,"props":24113,"children":24115},{"className":24114},[296,301],[24116],{"type":155,"value":22069},{"type":150,"tag":181,"props":24118,"children":24120},{"className":24119,"style":658},[377],[],{"type":150,"tag":181,"props":24122,"children":24124},{"className":24123},[663],[24125],{"type":155,"value":865},{"type":150,"tag":181,"props":24127,"children":24129},{"className":24128,"style":658},[377],[],{"type":150,"tag":181,"props":24131,"children":24133},{"className":24132},[285],[24134,24138,24143,24147,24152],{"type":150,"tag":181,"props":24135,"children":24137},{"className":24136,"style":5443},[290],[],{"type":150,"tag":181,"props":24139,"children":24141},{"className":24140},[296],[24142],{"type":155,"value":23847},{"type":150,"tag":181,"props":24144,"children":24146},{"className":24145,"style":4993},[377],[],{"type":150,"tag":181,"props":24148,"children":24150},{"className":24149},[4998],[24151],{"type":155,"value":5380},{"type":150,"tag":181,"props":24153,"children":24155},{"className":24154,"style":4993},[377],[],{"type":150,"tag":181,"props":24157,"children":24159},{"className":24158},[285],[24160,24164,24169,24173,24178],{"type":150,"tag":181,"props":24161,"children":24163},{"className":24162,"style":5012},[290],[],{"type":150,"tag":181,"props":24165,"children":24167},{"className":24166},[296],[24168],{"type":155,"value":238},{"type":150,"tag":181,"props":24170,"children":24172},{"className":24171,"style":658},[377],[],{"type":150,"tag":181,"props":24174,"children":24176},{"className":24175},[663],[24177],{"type":155,"value":865},{"type":150,"tag":181,"props":24179,"children":24181},{"className":24180,"style":658},[377],[],{"type":150,"tag":181,"props":24183,"children":24185},{"className":24184},[285],[24186,24190],{"type":150,"tag":181,"props":24187,"children":24189},{"className":24188,"style":5012},[290],[],{"type":150,"tag":181,"props":24191,"children":24193},{"className":24192},[296],[24194],{"type":155,"value":23213},{"type":155,"value":24196}," 字节：每层、每 token、每多看一个历史位置，要多读多少 KV",{"type":150,"tag":23637,"props":24198,"children":24199},{},[24200,24679],{"type":150,"tag":181,"props":24201,"children":24203},{"className":24202},[184],[24204,24311],{"type":150,"tag":181,"props":24205,"children":24207},{"className":24206},[189],[24208],{"type":150,"tag":192,"props":24209,"children":24210},{"xmlns":194},[24211],{"type":150,"tag":197,"props":24212,"children":24213},{},[24214,24306],{"type":150,"tag":201,"props":24215,"children":24216},{},[24217,24229,24233,24237,24241,24245,24256,24260,24271,24275,24279,24283,24287,24298,24302],{"type":150,"tag":205,"props":24218,"children":24219},{},[24220,24224],{"type":150,"tag":209,"props":24221,"children":24222},{},[24223],{"type":155,"value":9859},{"type":150,"tag":249,"props":24225,"children":24226},{},[24227],{"type":155,"value":24228},"flop",{"type":150,"tag":221,"props":24230,"children":24231},{},[24232],{"type":155,"value":865},{"type":150,"tag":215,"props":24234,"children":24235},{},[24236],{"type":155,"value":238},{"type":150,"tag":209,"props":24238,"children":24239},{},[24240],{"type":155,"value":745},{"type":150,"tag":221,"props":24242,"children":24243},{"stretchy":789},[24244],{"type":155,"value":792},{"type":150,"tag":205,"props":24246,"children":24247},{},[24248,24252],{"type":150,"tag":209,"props":24249,"children":24250},{},[24251],{"type":155,"value":578},{"type":150,"tag":209,"props":24253,"children":24254},{},[24255],{"type":155,"value":9859},{"type":150,"tag":221,"props":24257,"children":24258},{},[24259],{"type":155,"value":4962},{"type":150,"tag":205,"props":24261,"children":24262},{},[24263,24267],{"type":150,"tag":209,"props":24264,"children":24265},{},[24266],{"type":155,"value":578},{"type":150,"tag":209,"props":24268,"children":24269},{},[24270],{"type":155,"value":20412},{"type":150,"tag":221,"props":24272,"children":24273},{"stretchy":789},[24274],{"type":155,"value":802},{"type":150,"tag":221,"props":24276,"children":24277},{},[24278],{"type":155,"value":4962},{"type":150,"tag":215,"props":24280,"children":24281},{},[24282],{"type":155,"value":238},{"type":150,"tag":209,"props":24284,"children":24285},{},[24286],{"type":155,"value":745},{"type":150,"tag":205,"props":24288,"children":24289},{},[24290,24294],{"type":150,"tag":209,"props":24291,"children":24292},{},[24293],{"type":155,"value":578},{"type":150,"tag":209,"props":24295,"children":24296},{},[24297],{"type":155,"value":9859},{"type":150,"tag":221,"props":24299,"children":24300},{},[24301],{"type":155,"value":865},{"type":150,"tag":215,"props":24303,"children":24304},{},[24305],{"type":155,"value":23340},{"type":150,"tag":271,"props":24307,"children":24308},{"encoding":273},[24309],{"type":155,"value":24310},"c_{\\text{flop}} = 2h(d_c + d_r) + 2hd_c = 278528",{"type":150,"tag":181,"props":24312,"children":24314},{"className":24313,"ariaHidden":223},[280],[24315,24402,24495,24578,24666],{"type":150,"tag":181,"props":24316,"children":24318},{"className":24317},[285],[24319,24324,24389,24393,24398],{"type":150,"tag":181,"props":24320,"children":24323},{"className":24321,"style":24322},[290],"height:0.7167em;vertical-align:-0.2861em;",[],{"type":150,"tag":181,"props":24325,"children":24327},{"className":24326},[296],[24328,24333],{"type":150,"tag":181,"props":24329,"children":24331},{"className":24330},[296,301],[24332],{"type":155,"value":9859},{"type":150,"tag":181,"props":24334,"children":24336},{"className":24335},[307],[24337],{"type":150,"tag":181,"props":24338,"children":24340},{"className":24339},[312,313],[24341,24378],{"type":150,"tag":181,"props":24342,"children":24344},{"className":24343},[318],[24345,24373],{"type":150,"tag":181,"props":24346,"children":24348},{"className":24347,"style":493},[323],[24349],{"type":150,"tag":181,"props":24350,"children":24351},{"style":328},[24352,24356],{"type":150,"tag":181,"props":24353,"children":24355},{"className":24354,"style":334},[333],[],{"type":150,"tag":181,"props":24357,"children":24359},{"className":24358},[339,340,341,342],[24360],{"type":150,"tag":181,"props":24361,"children":24363},{"className":24362},[296,342],[24364],{"type":150,"tag":181,"props":24365,"children":24367},{"className":24366},[296,155,342],[24368],{"type":150,"tag":181,"props":24369,"children":24371},{"className":24370},[296,342],[24372],{"type":155,"value":24228},{"type":150,"tag":181,"props":24374,"children":24376},{"className":24375},[352],[24377],{"type":155,"value":355},{"type":150,"tag":181,"props":24379,"children":24381},{"className":24380},[318],[24382],{"type":150,"tag":181,"props":24383,"children":24385},{"className":24384,"style":6518},[323],[24386],{"type":150,"tag":181,"props":24387,"children":24388},{},[],{"type":150,"tag":181,"props":24390,"children":24392},{"className":24391,"style":658},[377],[],{"type":150,"tag":181,"props":24394,"children":24396},{"className":24395},[663],[24397],{"type":155,"value":865},{"type":150,"tag":181,"props":24399,"children":24401},{"className":24400,"style":658},[377],[],{"type":150,"tag":181,"props":24403,"children":24405},{"className":24404},[285],[24406,24410,24415,24420,24425,24482,24486,24491],{"type":150,"tag":181,"props":24407,"children":24409},{"className":24408,"style":819},[290],[],{"type":150,"tag":181,"props":24411,"children":24413},{"className":24412},[296],[24414],{"type":155,"value":238},{"type":150,"tag":181,"props":24416,"children":24418},{"className":24417},[296,301],[24419],{"type":155,"value":745},{"type":150,"tag":181,"props":24421,"children":24423},{"className":24422},[824],[24424],{"type":155,"value":792},{"type":150,"tag":181,"props":24426,"children":24428},{"className":24427},[296],[24429,24434],{"type":150,"tag":181,"props":24430,"children":24432},{"className":24431},[296,301],[24433],{"type":155,"value":578},{"type":150,"tag":181,"props":24435,"children":24437},{"className":24436},[307],[24438],{"type":150,"tag":181,"props":24439,"children":24441},{"className":24440},[312,313],[24442,24471],{"type":150,"tag":181,"props":24443,"children":24445},{"className":24444},[318],[24446,24466],{"type":150,"tag":181,"props":24447,"children":24449},{"className":24448,"style":4865},[323],[24450],{"type":150,"tag":181,"props":24451,"children":24452},{"style":328},[24453,24457],{"type":150,"tag":181,"props":24454,"children":24456},{"className":24455,"style":334},[333],[],{"type":150,"tag":181,"props":24458,"children":24460},{"className":24459},[339,340,341,342],[24461],{"type":150,"tag":181,"props":24462,"children":24464},{"className":24463},[296,301,342],[24465],{"type":155,"value":9859},{"type":150,"tag":181,"props":24467,"children":24469},{"className":24468},[352],[24470],{"type":155,"value":355},{"type":150,"tag":181,"props":24472,"children":24474},{"className":24473},[318],[24475],{"type":150,"tag":181,"props":24476,"children":24478},{"className":24477,"style":363},[323],[24479],{"type":150,"tag":181,"props":24480,"children":24481},{},[],{"type":150,"tag":181,"props":24483,"children":24485},{"className":24484,"style":4993},[377],[],{"type":150,"tag":181,"props":24487,"children":24489},{"className":24488},[4998],[24490],{"type":155,"value":4962},{"type":150,"tag":181,"props":24492,"children":24494},{"className":24493,"style":4993},[377],[],{"type":150,"tag":181,"props":24496,"children":24498},{"className":24497},[285],[24499,24503,24560,24565,24569,24574],{"type":150,"tag":181,"props":24500,"children":24502},{"className":24501,"style":819},[290],[],{"type":150,"tag":181,"props":24504,"children":24506},{"className":24505},[296],[24507,24512],{"type":150,"tag":181,"props":24508,"children":24510},{"className":24509},[296,301],[24511],{"type":155,"value":578},{"type":150,"tag":181,"props":24513,"children":24515},{"className":24514},[307],[24516],{"type":150,"tag":181,"props":24517,"children":24519},{"className":24518},[312,313],[24520,24549],{"type":150,"tag":181,"props":24521,"children":24523},{"className":24522},[318],[24524,24544],{"type":150,"tag":181,"props":24525,"children":24527},{"className":24526,"style":4865},[323],[24528],{"type":150,"tag":181,"props":24529,"children":24530},{"style":328},[24531,24535],{"type":150,"tag":181,"props":24532,"children":24534},{"className":24533,"style":334},[333],[],{"type":150,"tag":181,"props":24536,"children":24538},{"className":24537},[339,340,341,342],[24539],{"type":150,"tag":181,"props":24540,"children":24542},{"className":24541,"style":6794},[296,301,342],[24543],{"type":155,"value":20412},{"type":150,"tag":181,"props":24545,"children":24547},{"className":24546},[352],[24548],{"type":155,"value":355},{"type":150,"tag":181,"props":24550,"children":24552},{"className":24551},[318],[24553],{"type":150,"tag":181,"props":24554,"children":24556},{"className":24555,"style":363},[323],[24557],{"type":150,"tag":181,"props":24558,"children":24559},{},[],{"type":150,"tag":181,"props":24561,"children":24563},{"className":24562},[835],[24564],{"type":155,"value":802},{"type":150,"tag":181,"props":24566,"children":24568},{"className":24567,"style":4993},[377],[],{"type":150,"tag":181,"props":24570,"children":24572},{"className":24571},[4998],[24573],{"type":155,"value":4962},{"type":150,"tag":181,"props":24575,"children":24577},{"className":24576,"style":4993},[377],[],{"type":150,"tag":181,"props":24579,"children":24581},{"className":24580},[285],[24582,24586,24591,24596,24653,24657,24662],{"type":150,"tag":181,"props":24583,"children":24585},{"className":24584,"style":4761},[290],[],{"type":150,"tag":181,"props":24587,"children":24589},{"className":24588},[296],[24590],{"type":155,"value":238},{"type":150,"tag":181,"props":24592,"children":24594},{"className":24593},[296,301],[24595],{"type":155,"value":745},{"type":150,"tag":181,"props":24597,"children":24599},{"className":24598},[296],[24600,24605],{"type":150,"tag":181,"props":24601,"children":24603},{"className":24602},[296,301],[24604],{"type":155,"value":578},{"type":150,"tag":181,"props":24606,"children":24608},{"className":24607},[307],[24609],{"type":150,"tag":181,"props":24610,"children":24612},{"className":24611},[312,313],[24613,24642],{"type":150,"tag":181,"props":24614,"children":24616},{"className":24615},[318],[24617,24637],{"type":150,"tag":181,"props":24618,"children":24620},{"className":24619,"style":4865},[323],[24621],{"type":150,"tag":181,"props":24622,"children":24623},{"style":328},[24624,24628],{"type":150,"tag":181,"props":24625,"children":24627},{"className":24626,"style":334},[333],[],{"type":150,"tag":181,"props":24629,"children":24631},{"className":24630},[339,340,341,342],[24632],{"type":150,"tag":181,"props":24633,"children":24635},{"className":24634},[296,301,342],[24636],{"type":155,"value":9859},{"type":150,"tag":181,"props":24638,"children":24640},{"className":24639},[352],[24641],{"type":155,"value":355},{"type":150,"tag":181,"props":24643,"children":24645},{"className":24644},[318],[24646],{"type":150,"tag":181,"props":24647,"children":24649},{"className":24648,"style":363},[323],[24650],{"type":150,"tag":181,"props":24651,"children":24652},{},[],{"type":150,"tag":181,"props":24654,"children":24656},{"className":24655,"style":658},[377],[],{"type":150,"tag":181,"props":24658,"children":24660},{"className":24659},[663],[24661],{"type":155,"value":865},{"type":150,"tag":181,"props":24663,"children":24665},{"className":24664,"style":658},[377],[],{"type":150,"tag":181,"props":24667,"children":24669},{"className":24668},[285],[24670,24674],{"type":150,"tag":181,"props":24671,"children":24673},{"className":24672,"style":5012},[290],[],{"type":150,"tag":181,"props":24675,"children":24677},{"className":24676},[296],[24678],{"type":155,"value":23340},{"type":155,"value":24680},"：每层、每 token、每多看一个历史位置，要多做多少次浮点运算",{"type":150,"tag":23637,"props":24682,"children":24683},{},[24684,24865,24867,24909],{"type":150,"tag":181,"props":24685,"children":24687},{"className":24686},[184],[24688,24745],{"type":150,"tag":181,"props":24689,"children":24691},{"className":24690},[189],[24692],{"type":150,"tag":192,"props":24693,"children":24694},{"xmlns":194},[24695],{"type":150,"tag":197,"props":24696,"children":24697},{},[24698,24740],{"type":150,"tag":201,"props":24699,"children":24700},{},[24701,24705,24709,24714,24718,24722,24726,24731,24735],{"type":150,"tag":209,"props":24702,"children":24703},{},[24704],{"type":155,"value":3265},{"type":150,"tag":221,"props":24706,"children":24707},{},[24708],{"type":155,"value":865},{"type":150,"tag":215,"props":24710,"children":24711},{},[24712],{"type":155,"value":24713},"21",{"type":150,"tag":249,"props":24715,"children":24716},{},[24717],{"type":155,"value":23188},{"type":150,"tag":221,"props":24719,"children":24720},{},[24721],{"type":155,"value":5380},{"type":150,"tag":215,"props":24723,"children":24724},{},[24725],{"type":155,"value":238},{"type":150,"tag":249,"props":24727,"children":24728},{},[24729],{"type":155,"value":24730}," 字节",{"type":150,"tag":221,"props":24732,"children":24733},{},[24734],{"type":155,"value":865},{"type":150,"tag":215,"props":24736,"children":24737},{},[24738],{"type":155,"value":24739},"42",{"type":150,"tag":271,"props":24741,"children":24742},{"encoding":273},[24743],{"type":155,"value":24744},"W = 21\\text{B} \\times 2\\ \\text{字节} = 42",{"type":150,"tag":181,"props":24746,"children":24748},{"className":24747,"ariaHidden":223},[280],[24749,24775,24810,24852],{"type":150,"tag":181,"props":24750,"children":24752},{"className":24751},[285],[24753,24757,24762,24766,24771],{"type":150,"tag":181,"props":24754,"children":24756},{"className":24755,"style":5680},[290],[],{"type":150,"tag":181,"props":24758,"children":24760},{"className":24759,"style":3691},[296,301],[24761],{"type":155,"value":3265},{"type":150,"tag":181,"props":24763,"children":24765},{"className":24764,"style":658},[377],[],{"type":150,"tag":181,"props":24767,"children":24769},{"className":24768},[663],[24770],{"type":155,"value":865},{"type":150,"tag":181,"props":24772,"children":24774},{"className":24773,"style":658},[377],[],{"type":150,"tag":181,"props":24776,"children":24778},{"className":24777},[285],[24779,24783,24788,24797,24801,24806],{"type":150,"tag":181,"props":24780,"children":24782},{"className":24781,"style":5470},[290],[],{"type":150,"tag":181,"props":24784,"children":24786},{"className":24785},[296],[24787],{"type":155,"value":24713},{"type":150,"tag":181,"props":24789,"children":24791},{"className":24790},[296,155],[24792],{"type":150,"tag":181,"props":24793,"children":24795},{"className":24794},[296],[24796],{"type":155,"value":23188},{"type":150,"tag":181,"props":24798,"children":24800},{"className":24799,"style":4993},[377],[],{"type":150,"tag":181,"props":24802,"children":24804},{"className":24803},[4998],[24805],{"type":155,"value":5380},{"type":150,"tag":181,"props":24807,"children":24809},{"className":24808,"style":4993},[377],[],{"type":150,"tag":181,"props":24811,"children":24813},{"className":24812},[285],[24814,24818,24823,24828,24839,24843,24848],{"type":150,"tag":181,"props":24815,"children":24817},{"className":24816,"style":5680},[290],[],{"type":150,"tag":181,"props":24819,"children":24821},{"className":24820},[296],[24822],{"type":155,"value":238},{"type":150,"tag":181,"props":24824,"children":24826},{"className":24825},[377],[24827],{"type":155,"value":8576},{"type":150,"tag":181,"props":24829,"children":24831},{"className":24830},[296,155],[24832],{"type":150,"tag":181,"props":24833,"children":24836},{"className":24834},[296,24835],"cjk_fallback",[24837],{"type":155,"value":24838},"字节",{"type":150,"tag":181,"props":24840,"children":24842},{"className":24841,"style":658},[377],[],{"type":150,"tag":181,"props":24844,"children":24846},{"className":24845},[663],[24847],{"type":155,"value":865},{"type":150,"tag":181,"props":24849,"children":24851},{"className":24850,"style":658},[377],[],{"type":150,"tag":181,"props":24853,"children":24855},{"className":24854},[285],[24856,24860],{"type":150,"tag":181,"props":24857,"children":24859},{"className":24858,"style":5012},[290],[],{"type":150,"tag":181,"props":24861,"children":24863},{"className":24862},[296],[24864],{"type":155,"value":24739},{"type":155,"value":24866}," GB：每 token 要读的权重字节数（21B 是 activated 参数量），它不随 ",{"type":150,"tag":181,"props":24868,"children":24870},{"className":24869},[184],[24871,24892],{"type":150,"tag":181,"props":24872,"children":24874},{"className":24873},[189],[24875],{"type":150,"tag":192,"props":24876,"children":24877},{"xmlns":194},[24878],{"type":150,"tag":197,"props":24879,"children":24880},{},[24881,24888],{"type":150,"tag":201,"props":24882,"children":24883},{},[24884],{"type":150,"tag":209,"props":24885,"children":24886},{},[24887],{"type":155,"value":5417},{"type":150,"tag":271,"props":24889,"children":24890},{"encoding":273},[24891],{"type":155,"value":5417},{"type":150,"tag":181,"props":24893,"children":24895},{"className":24894,"ariaHidden":223},[280],[24896],{"type":150,"tag":181,"props":24897,"children":24899},{"className":24898},[285],[24900,24904],{"type":150,"tag":181,"props":24901,"children":24903},{"className":24902,"style":5680},[290],[],{"type":150,"tag":181,"props":24905,"children":24907},{"className":24906,"style":5606},[296,301],[24908],{"type":155,"value":5417},{"type":155,"value":24910}," 变",{"type":150,"tag":151,"props":24912,"children":24913},{},[24914,24916,24960],{"type":155,"value":24915},"算术强度（记作 ",{"type":150,"tag":181,"props":24917,"children":24919},{"className":24918},[184],[24920,24942],{"type":150,"tag":181,"props":24921,"children":24923},{"className":24922},[189],[24924],{"type":150,"tag":192,"props":24925,"children":24926},{"xmlns":194},[24927],{"type":150,"tag":197,"props":24928,"children":24929},{},[24930,24938],{"type":150,"tag":201,"props":24931,"children":24932},{},[24933],{"type":150,"tag":209,"props":24934,"children":24935},{},[24936],{"type":155,"value":24937},"I",{"type":150,"tag":271,"props":24939,"children":24940},{"encoding":273},[24941],{"type":155,"value":24937},{"type":150,"tag":181,"props":24943,"children":24945},{"className":24944,"ariaHidden":223},[280],[24946],{"type":150,"tag":181,"props":24947,"children":24949},{"className":24948},[285],[24950,24954],{"type":150,"tag":181,"props":24951,"children":24953},{"className":24952,"style":5680},[290],[],{"type":150,"tag":181,"props":24955,"children":24958},{"className":24956,"style":24957},[296,301],"margin-right:0.0785em;",[24959],{"type":155,"value":24937},{"type":155,"value":24961},"）就是一个字节要承担多少次浮点运算：",{"type":150,"tag":181,"props":24963,"children":24965},{"className":24964},[1003],[24966],{"type":150,"tag":181,"props":24967,"children":24969},{"className":24968},[184],[24970,25093],{"type":150,"tag":181,"props":24971,"children":24973},{"className":24972},[189],[24974],{"type":150,"tag":192,"props":24975,"children":24976},{"xmlns":194,"display":1015},[24977],{"type":150,"tag":197,"props":24978,"children":24979},{},[24980,25088],{"type":150,"tag":201,"props":24981,"children":24982},{},[24983,24987,24991,24995,24999,25003],{"type":150,"tag":209,"props":24984,"children":24985},{},[24986],{"type":155,"value":24937},{"type":150,"tag":221,"props":24988,"children":24989},{"stretchy":789},[24990],{"type":155,"value":792},{"type":150,"tag":209,"props":24992,"children":24993},{},[24994],{"type":155,"value":5417},{"type":150,"tag":221,"props":24996,"children":24997},{"stretchy":789},[24998],{"type":155,"value":802},{"type":150,"tag":221,"props":25000,"children":25001},{},[25002],{"type":155,"value":865},{"type":150,"tag":1774,"props":25004,"children":25005},{},[25006,25050],{"type":150,"tag":201,"props":25007,"children":25008},{},[25009,25013,25018,25022,25033,25038,25042,25046],{"type":150,"tag":215,"props":25010,"children":25011},{},[25012],{"type":155,"value":238},{"type":150,"tag":209,"props":25014,"children":25015},{},[25016],{"type":155,"value":25017},"P",{"type":150,"tag":221,"props":25019,"children":25020},{},[25021],{"type":155,"value":4962},{"type":150,"tag":205,"props":25023,"children":25024},{},[25025,25029],{"type":150,"tag":209,"props":25026,"children":25027},{},[25028],{"type":155,"value":9859},{"type":150,"tag":249,"props":25030,"children":25031},{},[25032],{"type":155,"value":24228},{"type":150,"tag":221,"props":25034,"children":25035},{},[25036],{"type":155,"value":25037},"⋅",{"type":150,"tag":209,"props":25039,"children":25040},{},[25041],{"type":155,"value":5385},{"type":150,"tag":221,"props":25043,"children":25044},{},[25045],{"type":155,"value":25037},{"type":150,"tag":209,"props":25047,"children":25048},{},[25049],{"type":155,"value":5417},{"type":150,"tag":201,"props":25051,"children":25052},{},[25053,25057,25061,25072,25076,25080,25084],{"type":150,"tag":209,"props":25054,"children":25055},{},[25056],{"type":155,"value":3265},{"type":150,"tag":221,"props":25058,"children":25059},{},[25060],{"type":155,"value":4962},{"type":150,"tag":205,"props":25062,"children":25063},{},[25064,25068],{"type":150,"tag":209,"props":25065,"children":25066},{},[25067],{"type":155,"value":9859},{"type":150,"tag":249,"props":25069,"children":25070},{},[25071],{"type":155,"value":23796},{"type":150,"tag":221,"props":25073,"children":25074},{},[25075],{"type":155,"value":25037},{"type":150,"tag":209,"props":25077,"children":25078},{},[25079],{"type":155,"value":5385},{"type":150,"tag":221,"props":25081,"children":25082},{},[25083],{"type":155,"value":25037},{"type":150,"tag":209,"props":25085,"children":25086},{},[25087],{"type":155,"value":5417},{"type":150,"tag":271,"props":25089,"children":25090},{"encoding":273},[25091],{"type":155,"value":25092},"I(N) = \\frac{2P + c_{\\text{flop}} \\cdot L \\cdot N}{W + c_{\\text{kv}} \\cdot L \\cdot N}",{"type":150,"tag":181,"props":25094,"children":25096},{"className":25095,"ariaHidden":223},[280],[25097,25138],{"type":150,"tag":181,"props":25098,"children":25100},{"className":25099},[285],[25101,25105,25110,25115,25120,25125,25129,25134],{"type":150,"tag":181,"props":25102,"children":25104},{"className":25103,"style":819},[290],[],{"type":150,"tag":181,"props":25106,"children":25108},{"className":25107,"style":24957},[296,301],[25109],{"type":155,"value":24937},{"type":150,"tag":181,"props":25111,"children":25113},{"className":25112},[824],[25114],{"type":155,"value":792},{"type":150,"tag":181,"props":25116,"children":25118},{"className":25117,"style":5606},[296,301],[25119],{"type":155,"value":5417},{"type":150,"tag":181,"props":25121,"children":25123},{"className":25122},[835],[25124],{"type":155,"value":802},{"type":150,"tag":181,"props":25126,"children":25128},{"className":25127,"style":658},[377],[],{"type":150,"tag":181,"props":25130,"children":25132},{"className":25131},[663],[25133],{"type":155,"value":865},{"type":150,"tag":181,"props":25135,"children":25137},{"className":25136,"style":658},[377],[],{"type":150,"tag":181,"props":25139,"children":25141},{"className":25140},[285],[25142,25147],{"type":150,"tag":181,"props":25143,"children":25146},{"className":25144,"style":25145},[290],"height:2.1963em;vertical-align:-0.836em;",[],{"type":150,"tag":181,"props":25148,"children":25150},{"className":25149},[296],[25151,25155,25464],{"type":150,"tag":181,"props":25152,"children":25154},{"className":25153},[824,2482],[],{"type":150,"tag":181,"props":25156,"children":25158},{"className":25157},[1774],[25159],{"type":150,"tag":181,"props":25160,"children":25162},{"className":25161},[312,313],[25163,25452],{"type":150,"tag":181,"props":25164,"children":25166},{"className":25165},[318],[25167,25447],{"type":150,"tag":181,"props":25168,"children":25170},{"className":25169,"style":23490},[323],[25171,25301,25312],{"type":150,"tag":181,"props":25172,"children":25173},{"style":8235},[25174,25178],{"type":150,"tag":181,"props":25175,"children":25177},{"className":25176,"style":8240},[333],[],{"type":150,"tag":181,"props":25179,"children":25181},{"className":25180},[296],[25182,25187,25191,25196,25200,25265,25269,25274,25278,25283,25287,25292,25296],{"type":150,"tag":181,"props":25183,"children":25185},{"className":25184,"style":3691},[296,301],[25186],{"type":155,"value":3265},{"type":150,"tag":181,"props":25188,"children":25190},{"className":25189,"style":4993},[377],[],{"type":150,"tag":181,"props":25192,"children":25194},{"className":25193},[4998],[25195],{"type":155,"value":4962},{"type":150,"tag":181,"props":25197,"children":25199},{"className":25198,"style":4993},[377],[],{"type":150,"tag":181,"props":25201,"children":25203},{"className":25202},[296],[25204,25209],{"type":150,"tag":181,"props":25205,"children":25207},{"className":25206},[296,301],[25208],{"type":155,"value":9859},{"type":150,"tag":181,"props":25210,"children":25212},{"className":25211},[307],[25213],{"type":150,"tag":181,"props":25214,"children":25216},{"className":25215},[312,313],[25217,25254],{"type":150,"tag":181,"props":25218,"children":25220},{"className":25219},[318],[25221,25249],{"type":150,"tag":181,"props":25222,"children":25224},{"className":25223,"style":493},[323],[25225],{"type":150,"tag":181,"props":25226,"children":25227},{"style":328},[25228,25232],{"type":150,"tag":181,"props":25229,"children":25231},{"className":25230,"style":334},[333],[],{"type":150,"tag":181,"props":25233,"children":25235},{"className":25234},[339,340,341,342],[25236],{"type":150,"tag":181,"props":25237,"children":25239},{"className":25238},[296,342],[25240],{"type":150,"tag":181,"props":25241,"children":25243},{"className":25242},[296,155,342],[25244],{"type":150,"tag":181,"props":25245,"children":25247},{"className":25246},[296,342],[25248],{"type":155,"value":23796},{"type":150,"tag":181,"props":25250,"children":25252},{"className":25251},[352],[25253],{"type":155,"value":355},{"type":150,"tag":181,"props":25255,"children":25257},{"className":25256},[318],[25258],{"type":150,"tag":181,"props":25259,"children":25261},{"className":25260,"style":363},[323],[25262],{"type":150,"tag":181,"props":25263,"children":25264},{},[],{"type":150,"tag":181,"props":25266,"children":25268},{"className":25267,"style":4993},[377],[],{"type":150,"tag":181,"props":25270,"children":25272},{"className":25271},[4998],[25273],{"type":155,"value":25037},{"type":150,"tag":181,"props":25275,"children":25277},{"className":25276,"style":4993},[377],[],{"type":150,"tag":181,"props":25279,"children":25281},{"className":25280},[296,301],[25282],{"type":155,"value":5385},{"type":150,"tag":181,"props":25284,"children":25286},{"className":25285,"style":4993},[377],[],{"type":150,"tag":181,"props":25288,"children":25290},{"className":25289},[4998],[25291],{"type":155,"value":25037},{"type":150,"tag":181,"props":25293,"children":25295},{"className":25294,"style":4993},[377],[],{"type":150,"tag":181,"props":25297,"children":25299},{"className":25298,"style":5606},[296,301],[25300],{"type":155,"value":5417},{"type":150,"tag":181,"props":25302,"children":25303},{"style":8253},[25304,25308],{"type":150,"tag":181,"props":25305,"children":25307},{"className":25306,"style":8240},[333],[],{"type":150,"tag":181,"props":25309,"children":25311},{"className":25310,"style":2804},[2803],[],{"type":150,"tag":181,"props":25313,"children":25314},{"style":8265},[25315,25319],{"type":150,"tag":181,"props":25316,"children":25318},{"className":25317,"style":8240},[333],[],{"type":150,"tag":181,"props":25320,"children":25322},{"className":25321},[296],[25323,25328,25333,25337,25342,25346,25411,25415,25420,25424,25429,25433,25438,25442],{"type":150,"tag":181,"props":25324,"children":25326},{"className":25325},[296],[25327],{"type":155,"value":238},{"type":150,"tag":181,"props":25329,"children":25331},{"className":25330,"style":3691},[296,301],[25332],{"type":155,"value":25017},{"type":150,"tag":181,"props":25334,"children":25336},{"className":25335,"style":4993},[377],[],{"type":150,"tag":181,"props":25338,"children":25340},{"className":25339},[4998],[25341],{"type":155,"value":4962},{"type":150,"tag":181,"props":25343,"children":25345},{"className":25344,"style":4993},[377],[],{"type":150,"tag":181,"props":25347,"children":25349},{"className":25348},[296],[25350,25355],{"type":150,"tag":181,"props":25351,"children":25353},{"className":25352},[296,301],[25354],{"type":155,"value":9859},{"type":150,"tag":181,"props":25356,"children":25358},{"className":25357},[307],[25359],{"type":150,"tag":181,"props":25360,"children":25362},{"className":25361},[312,313],[25363,25400],{"type":150,"tag":181,"props":25364,"children":25366},{"className":25365},[318],[25367,25395],{"type":150,"tag":181,"props":25368,"children":25370},{"className":25369,"style":493},[323],[25371],{"type":150,"tag":181,"props":25372,"children":25373},{"style":328},[25374,25378],{"type":150,"tag":181,"props":25375,"children":25377},{"className":25376,"style":334},[333],[],{"type":150,"tag":181,"props":25379,"children":25381},{"className":25380},[339,340,341,342],[25382],{"type":150,"tag":181,"props":25383,"children":25385},{"className":25384},[296,342],[25386],{"type":150,"tag":181,"props":25387,"children":25389},{"className":25388},[296,155,342],[25390],{"type":150,"tag":181,"props":25391,"children":25393},{"className":25392},[296,342],[25394],{"type":155,"value":24228},{"type":150,"tag":181,"props":25396,"children":25398},{"className":25397},[352],[25399],{"type":155,"value":355},{"type":150,"tag":181,"props":25401,"children":25403},{"className":25402},[318],[25404],{"type":150,"tag":181,"props":25405,"children":25407},{"className":25406,"style":6518},[323],[25408],{"type":150,"tag":181,"props":25409,"children":25410},{},[],{"type":150,"tag":181,"props":25412,"children":25414},{"className":25413,"style":4993},[377],[],{"type":150,"tag":181,"props":25416,"children":25418},{"className":25417},[4998],[25419],{"type":155,"value":25037},{"type":150,"tag":181,"props":25421,"children":25423},{"className":25422,"style":4993},[377],[],{"type":150,"tag":181,"props":25425,"children":25427},{"className":25426},[296,301],[25428],{"type":155,"value":5385},{"type":150,"tag":181,"props":25430,"children":25432},{"className":25431,"style":4993},[377],[],{"type":150,"tag":181,"props":25434,"children":25436},{"className":25435},[4998],[25437],{"type":155,"value":25037},{"type":150,"tag":181,"props":25439,"children":25441},{"className":25440,"style":4993},[377],[],{"type":150,"tag":181,"props":25443,"children":25445},{"className":25444,"style":5606},[296,301],[25446],{"type":155,"value":5417},{"type":150,"tag":181,"props":25448,"children":25450},{"className":25449},[352],[25451],{"type":155,"value":355},{"type":150,"tag":181,"props":25453,"children":25455},{"className":25454},[318],[25456],{"type":150,"tag":181,"props":25457,"children":25460},{"className":25458,"style":25459},[323],"height:0.836em;",[25461],{"type":150,"tag":181,"props":25462,"children":25463},{},[],{"type":150,"tag":181,"props":25465,"children":25467},{"className":25466},[835,2482],[],{"type":150,"tag":151,"props":25469,"children":25470},{},[25471,25473,25525,25527,25569],{"type":155,"value":25472},"分子是每个 token 要做的浮点次数：权重矩阵乘 ",{"type":150,"tag":181,"props":25474,"children":25476},{"className":25475},[184],[25477,25503],{"type":150,"tag":181,"props":25478,"children":25480},{"className":25479},[189],[25481],{"type":150,"tag":192,"props":25482,"children":25483},{"xmlns":194},[25484],{"type":150,"tag":197,"props":25485,"children":25486},{},[25487,25498],{"type":150,"tag":201,"props":25488,"children":25489},{},[25490,25494],{"type":150,"tag":215,"props":25491,"children":25492},{},[25493],{"type":155,"value":238},{"type":150,"tag":209,"props":25495,"children":25496},{},[25497],{"type":155,"value":25017},{"type":150,"tag":271,"props":25499,"children":25500},{"encoding":273},[25501],{"type":155,"value":25502},"2P",{"type":150,"tag":181,"props":25504,"children":25506},{"className":25505,"ariaHidden":223},[280],[25507],{"type":150,"tag":181,"props":25508,"children":25510},{"className":25509},[285],[25511,25515,25520],{"type":150,"tag":181,"props":25512,"children":25514},{"className":25513,"style":5680},[290],[],{"type":150,"tag":181,"props":25516,"children":25518},{"className":25517},[296],[25519],{"type":155,"value":238},{"type":150,"tag":181,"props":25521,"children":25523},{"className":25522,"style":3691},[296,301],[25524],{"type":155,"value":25017},{"type":155,"value":25526},"，加上 attention 那部分。分母是要搬的字节：权重 ",{"type":150,"tag":181,"props":25528,"children":25530},{"className":25529},[184],[25531,25552],{"type":150,"tag":181,"props":25532,"children":25534},{"className":25533},[189],[25535],{"type":150,"tag":192,"props":25536,"children":25537},{"xmlns":194},[25538],{"type":150,"tag":197,"props":25539,"children":25540},{},[25541,25548],{"type":150,"tag":201,"props":25542,"children":25543},{},[25544],{"type":150,"tag":209,"props":25545,"children":25546},{},[25547],{"type":155,"value":3265},{"type":150,"tag":271,"props":25549,"children":25550},{"encoding":273},[25551],{"type":155,"value":3265},{"type":150,"tag":181,"props":25553,"children":25555},{"className":25554,"ariaHidden":223},[280],[25556],{"type":150,"tag":181,"props":25557,"children":25559},{"className":25558},[285],[25560,25564],{"type":150,"tag":181,"props":25561,"children":25563},{"className":25562,"style":5680},[290],[],{"type":150,"tag":181,"props":25565,"children":25567},{"className":25566,"style":3691},[296,301],[25568],{"type":155,"value":3265},{"type":155,"value":25570},"，加上 KV。代进数字：",{"type":150,"tag":181,"props":25572,"children":25574},{"className":25573},[1003],[25575],{"type":150,"tag":181,"props":25576,"children":25578},{"className":25577},[184],[25579,25714],{"type":150,"tag":181,"props":25580,"children":25582},{"className":25581},[189],[25583],{"type":150,"tag":192,"props":25584,"children":25585},{"xmlns":194,"display":1015},[25586],{"type":150,"tag":197,"props":25587,"children":25588},{},[25589,25709],{"type":150,"tag":201,"props":25590,"children":25591},{},[25592,25596,25600,25604,25608,25612],{"type":150,"tag":209,"props":25593,"children":25594},{},[25595],{"type":155,"value":24937},{"type":150,"tag":221,"props":25597,"children":25598},{"stretchy":789},[25599],{"type":155,"value":792},{"type":150,"tag":209,"props":25601,"children":25602},{},[25603],{"type":155,"value":5417},{"type":150,"tag":221,"props":25605,"children":25606},{"stretchy":789},[25607],{"type":155,"value":802},{"type":150,"tag":221,"props":25609,"children":25610},{},[25611],{"type":155,"value":865},{"type":150,"tag":1774,"props":25613,"children":25614},{},[25615,25663],{"type":150,"tag":201,"props":25616,"children":25617},{},[25618,25622,25626,25639,25643,25647,25651,25655,25659],{"type":150,"tag":215,"props":25619,"children":25620},{},[25621],{"type":155,"value":24739},{"type":150,"tag":221,"props":25623,"children":25624},{},[25625],{"type":155,"value":5380},{"type":150,"tag":565,"props":25627,"children":25628},{},[25629,25634],{"type":150,"tag":215,"props":25630,"children":25631},{},[25632],{"type":155,"value":25633},"10",{"type":150,"tag":215,"props":25635,"children":25636},{},[25637],{"type":155,"value":25638},"9",{"type":150,"tag":221,"props":25640,"children":25641},{},[25642],{"type":155,"value":4962},{"type":150,"tag":215,"props":25644,"children":25645},{},[25646],{"type":155,"value":23340},{"type":150,"tag":221,"props":25648,"children":25649},{},[25650],{"type":155,"value":5380},{"type":150,"tag":215,"props":25652,"children":25653},{},[25654],{"type":155,"value":21506},{"type":150,"tag":221,"props":25656,"children":25657},{},[25658],{"type":155,"value":5380},{"type":150,"tag":209,"props":25660,"children":25661},{},[25662],{"type":155,"value":5417},{"type":150,"tag":201,"props":25664,"children":25665},{},[25666,25670,25674,25685,25689,25693,25697,25701,25705],{"type":150,"tag":215,"props":25667,"children":25668},{},[25669],{"type":155,"value":24739},{"type":150,"tag":221,"props":25671,"children":25672},{},[25673],{"type":155,"value":5380},{"type":150,"tag":565,"props":25675,"children":25676},{},[25677,25681],{"type":150,"tag":215,"props":25678,"children":25679},{},[25680],{"type":155,"value":25633},{"type":150,"tag":215,"props":25682,"children":25683},{},[25684],{"type":155,"value":25638},{"type":150,"tag":221,"props":25686,"children":25687},{},[25688],{"type":155,"value":4962},{"type":150,"tag":215,"props":25690,"children":25691},{},[25692],{"type":155,"value":23213},{"type":150,"tag":221,"props":25694,"children":25695},{},[25696],{"type":155,"value":5380},{"type":150,"tag":215,"props":25698,"children":25699},{},[25700],{"type":155,"value":21506},{"type":150,"tag":221,"props":25702,"children":25703},{},[25704],{"type":155,"value":5380},{"type":150,"tag":209,"props":25706,"children":25707},{},[25708],{"type":155,"value":5417},{"type":150,"tag":271,"props":25710,"children":25711},{"encoding":273},[25712],{"type":155,"value":25713},"I(N) = \\frac{42 \\times 10^9 + 278528 \\times 60 \\times N}{42 \\times 10^9 + 1152 \\times 60 \\times N}",{"type":150,"tag":181,"props":25715,"children":25717},{"className":25716,"ariaHidden":223},[280],[25718,25759],{"type":150,"tag":181,"props":25719,"children":25721},{"className":25720},[285],[25722,25726,25731,25736,25741,25746,25750,25755],{"type":150,"tag":181,"props":25723,"children":25725},{"className":25724,"style":819},[290],[],{"type":150,"tag":181,"props":25727,"children":25729},{"className":25728,"style":24957},[296,301],[25730],{"type":155,"value":24937},{"type":150,"tag":181,"props":25732,"children":25734},{"className":25733},[824],[25735],{"type":155,"value":792},{"type":150,"tag":181,"props":25737,"children":25739},{"className":25738,"style":5606},[296,301],[25740],{"type":155,"value":5417},{"type":150,"tag":181,"props":25742,"children":25744},{"className":25743},[835],[25745],{"type":155,"value":802},{"type":150,"tag":181,"props":25747,"children":25749},{"className":25748,"style":658},[377],[],{"type":150,"tag":181,"props":25751,"children":25753},{"className":25752},[663],[25754],{"type":155,"value":865},{"type":150,"tag":181,"props":25756,"children":25758},{"className":25757,"style":658},[377],[],{"type":150,"tag":181,"props":25760,"children":25762},{"className":25761},[285],[25763,25768],{"type":150,"tag":181,"props":25764,"children":25767},{"className":25765,"style":25766},[290],"height:2.2604em;vertical-align:-0.7693em;",[],{"type":150,"tag":181,"props":25769,"children":25771},{"className":25770},[296],[25772,25776,26082],{"type":150,"tag":181,"props":25773,"children":25775},{"className":25774},[824,2482],[],{"type":150,"tag":181,"props":25777,"children":25779},{"className":25778},[1774],[25780],{"type":150,"tag":181,"props":25781,"children":25783},{"className":25782},[312,313],[25784,26070],{"type":150,"tag":181,"props":25785,"children":25787},{"className":25786},[318],[25788,26065],{"type":150,"tag":181,"props":25789,"children":25792},{"className":25790,"style":25791},[323],"height:1.4911em;",[25793,25925,25936],{"type":150,"tag":181,"props":25794,"children":25795},{"style":8235},[25796,25800],{"type":150,"tag":181,"props":25797,"children":25799},{"className":25798,"style":8240},[333],[],{"type":150,"tag":181,"props":25801,"children":25803},{"className":25802},[296],[25804,25809,25813,25818,25822,25827,25871,25875,25880,25884,25889,25893,25898,25902,25907,25911,25916,25920],{"type":150,"tag":181,"props":25805,"children":25807},{"className":25806},[296],[25808],{"type":155,"value":24739},{"type":150,"tag":181,"props":25810,"children":25812},{"className":25811,"style":4993},[377],[],{"type":150,"tag":181,"props":25814,"children":25816},{"className":25815},[4998],[25817],{"type":155,"value":5380},{"type":150,"tag":181,"props":25819,"children":25821},{"className":25820,"style":4993},[377],[],{"type":150,"tag":181,"props":25823,"children":25825},{"className":25824},[296],[25826],{"type":155,"value":219},{"type":150,"tag":181,"props":25828,"children":25830},{"className":25829},[296],[25831,25837],{"type":150,"tag":181,"props":25832,"children":25834},{"className":25833},[296],[25835],{"type":155,"value":25836},"0",{"type":150,"tag":181,"props":25838,"children":25840},{"className":25839},[307],[25841],{"type":150,"tag":181,"props":25842,"children":25844},{"className":25843},[312],[25845],{"type":150,"tag":181,"props":25846,"children":25848},{"className":25847},[318],[25849],{"type":150,"tag":181,"props":25850,"children":25853},{"className":25851,"style":25852},[323],"height:0.7401em;",[25854],{"type":150,"tag":181,"props":25855,"children":25857},{"style":25856},"top:-2.989em;margin-right:0.05em;",[25858,25862],{"type":150,"tag":181,"props":25859,"children":25861},{"className":25860,"style":334},[333],[],{"type":150,"tag":181,"props":25863,"children":25865},{"className":25864},[339,340,341,342],[25866],{"type":150,"tag":181,"props":25867,"children":25869},{"className":25868},[296,342],[25870],{"type":155,"value":25638},{"type":150,"tag":181,"props":25872,"children":25874},{"className":25873,"style":4993},[377],[],{"type":150,"tag":181,"props":25876,"children":25878},{"className":25877},[4998],[25879],{"type":155,"value":4962},{"type":150,"tag":181,"props":25881,"children":25883},{"className":25882,"style":4993},[377],[],{"type":150,"tag":181,"props":25885,"children":25887},{"className":25886},[296],[25888],{"type":155,"value":23213},{"type":150,"tag":181,"props":25890,"children":25892},{"className":25891,"style":4993},[377],[],{"type":150,"tag":181,"props":25894,"children":25896},{"className":25895},[4998],[25897],{"type":155,"value":5380},{"type":150,"tag":181,"props":25899,"children":25901},{"className":25900,"style":4993},[377],[],{"type":150,"tag":181,"props":25903,"children":25905},{"className":25904},[296],[25906],{"type":155,"value":21506},{"type":150,"tag":181,"props":25908,"children":25910},{"className":25909,"style":4993},[377],[],{"type":150,"tag":181,"props":25912,"children":25914},{"className":25913},[4998],[25915],{"type":155,"value":5380},{"type":150,"tag":181,"props":25917,"children":25919},{"className":25918,"style":4993},[377],[],{"type":150,"tag":181,"props":25921,"children":25923},{"className":25922,"style":5606},[296,301],[25924],{"type":155,"value":5417},{"type":150,"tag":181,"props":25926,"children":25927},{"style":8253},[25928,25932],{"type":150,"tag":181,"props":25929,"children":25931},{"className":25930,"style":8240},[333],[],{"type":150,"tag":181,"props":25933,"children":25935},{"className":25934,"style":2804},[2803],[],{"type":150,"tag":181,"props":25937,"children":25938},{"style":8265},[25939,25943],{"type":150,"tag":181,"props":25940,"children":25942},{"className":25941,"style":8240},[333],[],{"type":150,"tag":181,"props":25944,"children":25946},{"className":25945},[296],[25947,25952,25956,25961,25965,25970,26011,26015,26020,26024,26029,26033,26038,26042,26047,26051,26056,26060],{"type":150,"tag":181,"props":25948,"children":25950},{"className":25949},[296],[25951],{"type":155,"value":24739},{"type":150,"tag":181,"props":25953,"children":25955},{"className":25954,"style":4993},[377],[],{"type":150,"tag":181,"props":25957,"children":25959},{"className":25958},[4998],[25960],{"type":155,"value":5380},{"type":150,"tag":181,"props":25962,"children":25964},{"className":25963,"style":4993},[377],[],{"type":150,"tag":181,"props":25966,"children":25968},{"className":25967},[296],[25969],{"type":155,"value":219},{"type":150,"tag":181,"props":25971,"children":25973},{"className":25972},[296],[25974,25979],{"type":150,"tag":181,"props":25975,"children":25977},{"className":25976},[296],[25978],{"type":155,"value":25836},{"type":150,"tag":181,"props":25980,"children":25982},{"className":25981},[307],[25983],{"type":150,"tag":181,"props":25984,"children":25986},{"className":25985},[312],[25987],{"type":150,"tag":181,"props":25988,"children":25990},{"className":25989},[318],[25991],{"type":150,"tag":181,"props":25992,"children":25994},{"className":25993,"style":6831},[323],[25995],{"type":150,"tag":181,"props":25996,"children":25997},{"style":707},[25998,26002],{"type":150,"tag":181,"props":25999,"children":26001},{"className":26000,"style":334},[333],[],{"type":150,"tag":181,"props":26003,"children":26005},{"className":26004},[339,340,341,342],[26006],{"type":150,"tag":181,"props":26007,"children":26009},{"className":26008},[296,342],[26010],{"type":155,"value":25638},{"type":150,"tag":181,"props":26012,"children":26014},{"className":26013,"style":4993},[377],[],{"type":150,"tag":181,"props":26016,"children":26018},{"className":26017},[4998],[26019],{"type":155,"value":4962},{"type":150,"tag":181,"props":26021,"children":26023},{"className":26022,"style":4993},[377],[],{"type":150,"tag":181,"props":26025,"children":26027},{"className":26026},[296],[26028],{"type":155,"value":23340},{"type":150,"tag":181,"props":26030,"children":26032},{"className":26031,"style":4993},[377],[],{"type":150,"tag":181,"props":26034,"children":26036},{"className":26035},[4998],[26037],{"type":155,"value":5380},{"type":150,"tag":181,"props":26039,"children":26041},{"className":26040,"style":4993},[377],[],{"type":150,"tag":181,"props":26043,"children":26045},{"className":26044},[296],[26046],{"type":155,"value":21506},{"type":150,"tag":181,"props":26048,"children":26050},{"className":26049,"style":4993},[377],[],{"type":150,"tag":181,"props":26052,"children":26054},{"className":26053},[4998],[26055],{"type":155,"value":5380},{"type":150,"tag":181,"props":26057,"children":26059},{"className":26058,"style":4993},[377],[],{"type":150,"tag":181,"props":26061,"children":26063},{"className":26062,"style":5606},[296,301],[26064],{"type":155,"value":5417},{"type":150,"tag":181,"props":26066,"children":26068},{"className":26067},[352],[26069],{"type":155,"value":355},{"type":150,"tag":181,"props":26071,"children":26073},{"className":26072},[318],[26074],{"type":150,"tag":181,"props":26075,"children":26078},{"className":26076,"style":26077},[323],"height:0.7693em;",[26079],{"type":150,"tag":181,"props":26080,"children":26081},{},[],{"type":150,"tag":181,"props":26083,"children":26085},{"className":26084},[835,2482],[],{"type":150,"tag":151,"props":26087,"children":26088},{},[26089,26131],{"type":150,"tag":181,"props":26090,"children":26092},{"className":26091},[184],[26093,26114],{"type":150,"tag":181,"props":26094,"children":26096},{"className":26095},[189],[26097],{"type":150,"tag":192,"props":26098,"children":26099},{"xmlns":194},[26100],{"type":150,"tag":197,"props":26101,"children":26102},{},[26103,26110],{"type":150,"tag":201,"props":26104,"children":26105},{},[26106],{"type":150,"tag":209,"props":26107,"children":26108},{},[26109],{"type":155,"value":5417},{"type":150,"tag":271,"props":26111,"children":26112},{"encoding":273},[26113],{"type":155,"value":5417},{"type":150,"tag":181,"props":26115,"children":26117},{"className":26116,"ariaHidden":223},[280],[26118],{"type":150,"tag":181,"props":26119,"children":26121},{"className":26120},[285],[26122,26126],{"type":150,"tag":181,"props":26123,"children":26125},{"className":26124,"style":5680},[290],[],{"type":150,"tag":181,"props":26127,"children":26129},{"className":26128,"style":5606},[296,301],[26130],{"type":155,"value":5417},{"type":155,"value":26132}," 一大，分子分母里那两项权重就被 KV 和 attention 项压过去了，于是有上界：",{"type":150,"tag":181,"props":26134,"children":26136},{"className":26135},[1003],[26137],{"type":150,"tag":181,"props":26138,"children":26140},{"className":26139},[184],[26141,26263],{"type":150,"tag":181,"props":26142,"children":26144},{"className":26143},[189],[26145],{"type":150,"tag":192,"props":26146,"children":26147},{"xmlns":194,"display":1015},[26148],{"type":150,"tag":197,"props":26149,"children":26150},{},[26151,26258],{"type":150,"tag":201,"props":26152,"children":26153},{},[26154,26185,26189,26193,26197,26201,26205,26230,26234,26245,26249,26254],{"type":150,"tag":1781,"props":26155,"children":26156},{},[26157,26169],{"type":150,"tag":201,"props":26158,"children":26159},{},[26160,26165],{"type":150,"tag":209,"props":26161,"children":26162},{},[26163],{"type":155,"value":26164},"lim",{"type":150,"tag":221,"props":26166,"children":26167},{},[26168],{"type":155,"value":1813},{"type":150,"tag":201,"props":26170,"children":26171},{},[26172,26176,26180],{"type":150,"tag":209,"props":26173,"children":26174},{},[26175],{"type":155,"value":5417},{"type":150,"tag":221,"props":26177,"children":26178},{},[26179],{"type":155,"value":12767},{"type":150,"tag":209,"props":26181,"children":26182},{"mathvariant":1618},[26183],{"type":155,"value":26184},"∞",{"type":150,"tag":209,"props":26186,"children":26187},{},[26188],{"type":155,"value":24937},{"type":150,"tag":221,"props":26190,"children":26191},{"stretchy":789},[26192],{"type":155,"value":792},{"type":150,"tag":209,"props":26194,"children":26195},{},[26196],{"type":155,"value":5417},{"type":150,"tag":221,"props":26198,"children":26199},{"stretchy":789},[26200],{"type":155,"value":802},{"type":150,"tag":221,"props":26202,"children":26203},{},[26204],{"type":155,"value":865},{"type":150,"tag":1774,"props":26206,"children":26207},{},[26208,26219],{"type":150,"tag":205,"props":26209,"children":26210},{},[26211,26215],{"type":150,"tag":209,"props":26212,"children":26213},{},[26214],{"type":155,"value":9859},{"type":150,"tag":249,"props":26216,"children":26217},{},[26218],{"type":155,"value":24228},{"type":150,"tag":205,"props":26220,"children":26221},{},[26222,26226],{"type":150,"tag":209,"props":26223,"children":26224},{},[26225],{"type":155,"value":9859},{"type":150,"tag":249,"props":26227,"children":26228},{},[26229],{"type":155,"value":23796},{"type":150,"tag":221,"props":26231,"children":26232},{},[26233],{"type":155,"value":865},{"type":150,"tag":1774,"props":26235,"children":26236},{},[26237,26241],{"type":150,"tag":215,"props":26238,"children":26239},{},[26240],{"type":155,"value":23340},{"type":150,"tag":215,"props":26242,"children":26243},{},[26244],{"type":155,"value":23213},{"type":150,"tag":221,"props":26246,"children":26247},{},[26248],{"type":155,"value":5876},{"type":150,"tag":215,"props":26250,"children":26251},{},[26252],{"type":155,"value":26253},"242",{"type":150,"tag":249,"props":26255,"children":26256},{},[26257],{"type":155,"value":23448},{"type":150,"tag":271,"props":26259,"children":26260},{"encoding":273},[26261],{"type":155,"value":26262},"\\lim_{N \\to \\infty} I(N) = \\frac{c_{\\text{flop}}}{c_{\\text{kv}}} = \\frac{278528}{1152} \\approx 242\\ \\text{FLOP\u002Fbyte}",{"type":150,"tag":181,"props":26264,"children":26266},{"className":26265,"ariaHidden":223},[280],[26267,26394,26623,26733],{"type":150,"tag":181,"props":26268,"children":26270},{"className":26269},[285],[26271,26276,26357,26361,26366,26371,26376,26381,26385,26390],{"type":150,"tag":181,"props":26272,"children":26275},{"className":26273,"style":26274},[290],"height:1.4943em;vertical-align:-0.7443em;",[],{"type":150,"tag":181,"props":26277,"children":26280},{"className":26278},[2517,26279],"op-limits",[26281],{"type":150,"tag":181,"props":26282,"children":26284},{"className":26283},[312,313],[26285,26345],{"type":150,"tag":181,"props":26286,"children":26288},{"className":26287},[318],[26289,26340],{"type":150,"tag":181,"props":26290,"children":26292},{"className":26291,"style":761},[323],[26293,26324],{"type":150,"tag":181,"props":26294,"children":26296},{"style":26295},"top:-2.3557em;margin-left:0em;",[26297,26301],{"type":150,"tag":181,"props":26298,"children":26300},{"className":26299,"style":8240},[333],[],{"type":150,"tag":181,"props":26302,"children":26304},{"className":26303},[339,340,341,342],[26305],{"type":150,"tag":181,"props":26306,"children":26308},{"className":26307},[296,342],[26309,26314,26319],{"type":150,"tag":181,"props":26310,"children":26312},{"className":26311,"style":5606},[296,301,342],[26313],{"type":155,"value":5417},{"type":150,"tag":181,"props":26315,"children":26317},{"className":26316},[663,342],[26318],{"type":155,"value":12767},{"type":150,"tag":181,"props":26320,"children":26322},{"className":26321},[296,342],[26323],{"type":155,"value":26184},{"type":150,"tag":181,"props":26325,"children":26327},{"style":26326},"top:-3em;",[26328,26332],{"type":150,"tag":181,"props":26329,"children":26331},{"className":26330,"style":8240},[333],[],{"type":150,"tag":181,"props":26333,"children":26334},{},[26335],{"type":150,"tag":181,"props":26336,"children":26338},{"className":26337},[2517],[26339],{"type":155,"value":26164},{"type":150,"tag":181,"props":26341,"children":26343},{"className":26342},[352],[26344],{"type":155,"value":355},{"type":150,"tag":181,"props":26346,"children":26348},{"className":26347},[318],[26349],{"type":150,"tag":181,"props":26350,"children":26353},{"className":26351,"style":26352},[323],"height:0.7443em;",[26354],{"type":150,"tag":181,"props":26355,"children":26356},{},[],{"type":150,"tag":181,"props":26358,"children":26360},{"className":26359,"style":378},[377],[],{"type":150,"tag":181,"props":26362,"children":26364},{"className":26363,"style":24957},[296,301],[26365],{"type":155,"value":24937},{"type":150,"tag":181,"props":26367,"children":26369},{"className":26368},[824],[26370],{"type":155,"value":792},{"type":150,"tag":181,"props":26372,"children":26374},{"className":26373,"style":5606},[296,301],[26375],{"type":155,"value":5417},{"type":150,"tag":181,"props":26377,"children":26379},{"className":26378},[835],[26380],{"type":155,"value":802},{"type":150,"tag":181,"props":26382,"children":26384},{"className":26383,"style":658},[377],[],{"type":150,"tag":181,"props":26386,"children":26388},{"className":26387},[663],[26389],{"type":155,"value":865},{"type":150,"tag":181,"props":26391,"children":26393},{"className":26392,"style":658},[377],[],{"type":150,"tag":181,"props":26395,"children":26397},{"className":26396},[285],[26398,26403,26610,26614,26619],{"type":150,"tag":181,"props":26399,"children":26402},{"className":26400,"style":26401},[290],"height:1.9436em;vertical-align:-0.836em;",[],{"type":150,"tag":181,"props":26404,"children":26406},{"className":26405},[296],[26407,26411,26606],{"type":150,"tag":181,"props":26408,"children":26410},{"className":26409},[824,2482],[],{"type":150,"tag":181,"props":26412,"children":26414},{"className":26413},[1774],[26415],{"type":150,"tag":181,"props":26416,"children":26418},{"className":26417},[312,313],[26419,26595],{"type":150,"tag":181,"props":26420,"children":26422},{"className":26421},[318],[26423,26590],{"type":150,"tag":181,"props":26424,"children":26426},{"className":26425,"style":8231},[323],[26427,26503,26514],{"type":150,"tag":181,"props":26428,"children":26429},{"style":8235},[26430,26434],{"type":150,"tag":181,"props":26431,"children":26433},{"className":26432,"style":8240},[333],[],{"type":150,"tag":181,"props":26435,"children":26437},{"className":26436},[296],[26438],{"type":150,"tag":181,"props":26439,"children":26441},{"className":26440},[296],[26442,26447],{"type":150,"tag":181,"props":26443,"children":26445},{"className":26444},[296,301],[26446],{"type":155,"value":9859},{"type":150,"tag":181,"props":26448,"children":26450},{"className":26449},[307],[26451],{"type":150,"tag":181,"props":26452,"children":26454},{"className":26453},[312,313],[26455,26492],{"type":150,"tag":181,"props":26456,"children":26458},{"className":26457},[318],[26459,26487],{"type":150,"tag":181,"props":26460,"children":26462},{"className":26461,"style":493},[323],[26463],{"type":150,"tag":181,"props":26464,"children":26465},{"style":328},[26466,26470],{"type":150,"tag":181,"props":26467,"children":26469},{"className":26468,"style":334},[333],[],{"type":150,"tag":181,"props":26471,"children":26473},{"className":26472},[339,340,341,342],[26474],{"type":150,"tag":181,"props":26475,"children":26477},{"className":26476},[296,342],[26478],{"type":150,"tag":181,"props":26479,"children":26481},{"className":26480},[296,155,342],[26482],{"type":150,"tag":181,"props":26483,"children":26485},{"className":26484},[296,342],[26486],{"type":155,"value":23796},{"type":150,"tag":181,"props":26488,"children":26490},{"className":26489},[352],[26491],{"type":155,"value":355},{"type":150,"tag":181,"props":26493,"children":26495},{"className":26494},[318],[26496],{"type":150,"tag":181,"props":26497,"children":26499},{"className":26498,"style":363},[323],[26500],{"type":150,"tag":181,"props":26501,"children":26502},{},[],{"type":150,"tag":181,"props":26504,"children":26505},{"style":8253},[26506,26510],{"type":150,"tag":181,"props":26507,"children":26509},{"className":26508,"style":8240},[333],[],{"type":150,"tag":181,"props":26511,"children":26513},{"className":26512,"style":2804},[2803],[],{"type":150,"tag":181,"props":26515,"children":26516},{"style":8265},[26517,26521],{"type":150,"tag":181,"props":26518,"children":26520},{"className":26519,"style":8240},[333],[],{"type":150,"tag":181,"props":26522,"children":26524},{"className":26523},[296],[26525],{"type":150,"tag":181,"props":26526,"children":26528},{"className":26527},[296],[26529,26534],{"type":150,"tag":181,"props":26530,"children":26532},{"className":26531},[296,301],[26533],{"type":155,"value":9859},{"type":150,"tag":181,"props":26535,"children":26537},{"className":26536},[307],[26538],{"type":150,"tag":181,"props":26539,"children":26541},{"className":26540},[312,313],[26542,26579],{"type":150,"tag":181,"props":26543,"children":26545},{"className":26544},[318],[26546,26574],{"type":150,"tag":181,"props":26547,"children":26549},{"className":26548,"style":493},[323],[26550],{"type":150,"tag":181,"props":26551,"children":26552},{"style":328},[26553,26557],{"type":150,"tag":181,"props":26554,"children":26556},{"className":26555,"style":334},[333],[],{"type":150,"tag":181,"props":26558,"children":26560},{"className":26559},[339,340,341,342],[26561],{"type":150,"tag":181,"props":26562,"children":26564},{"className":26563},[296,342],[26565],{"type":150,"tag":181,"props":26566,"children":26568},{"className":26567},[296,155,342],[26569],{"type":150,"tag":181,"props":26570,"children":26572},{"className":26571},[296,342],[26573],{"type":155,"value":24228},{"type":150,"tag":181,"props":26575,"children":26577},{"className":26576},[352],[26578],{"type":155,"value":355},{"type":150,"tag":181,"props":26580,"children":26582},{"className":26581},[318],[26583],{"type":150,"tag":181,"props":26584,"children":26586},{"className":26585,"style":6518},[323],[26587],{"type":150,"tag":181,"props":26588,"children":26589},{},[],{"type":150,"tag":181,"props":26591,"children":26593},{"className":26592},[352],[26594],{"type":155,"value":355},{"type":150,"tag":181,"props":26596,"children":26598},{"className":26597},[318],[26599],{"type":150,"tag":181,"props":26600,"children":26602},{"className":26601,"style":25459},[323],[26603],{"type":150,"tag":181,"props":26604,"children":26605},{},[],{"type":150,"tag":181,"props":26607,"children":26609},{"className":26608},[835,2482],[],{"type":150,"tag":181,"props":26611,"children":26613},{"className":26612,"style":658},[377],[],{"type":150,"tag":181,"props":26615,"children":26617},{"className":26616},[663],[26618],{"type":155,"value":865},{"type":150,"tag":181,"props":26620,"children":26622},{"className":26621,"style":658},[377],[],{"type":150,"tag":181,"props":26624,"children":26626},{"className":26625},[285],[26627,26632,26720,26724,26729],{"type":150,"tag":181,"props":26628,"children":26631},{"className":26629,"style":26630},[290],"height:2.0074em;vertical-align:-0.686em;",[],{"type":150,"tag":181,"props":26633,"children":26635},{"className":26634},[296],[26636,26640,26716],{"type":150,"tag":181,"props":26637,"children":26639},{"className":26638},[824,2482],[],{"type":150,"tag":181,"props":26641,"children":26643},{"className":26642},[1774],[26644],{"type":150,"tag":181,"props":26645,"children":26647},{"className":26646},[312,313],[26648,26705],{"type":150,"tag":181,"props":26649,"children":26651},{"className":26650},[318],[26652,26700],{"type":150,"tag":181,"props":26653,"children":26656},{"className":26654,"style":26655},[323],"height:1.3214em;",[26657,26673,26684],{"type":150,"tag":181,"props":26658,"children":26659},{"style":8235},[26660,26664],{"type":150,"tag":181,"props":26661,"children":26663},{"className":26662,"style":8240},[333],[],{"type":150,"tag":181,"props":26665,"children":26667},{"className":26666},[296],[26668],{"type":150,"tag":181,"props":26669,"children":26671},{"className":26670},[296],[26672],{"type":155,"value":23213},{"type":150,"tag":181,"props":26674,"children":26675},{"style":8253},[26676,26680],{"type":150,"tag":181,"props":26677,"children":26679},{"className":26678,"style":8240},[333],[],{"type":150,"tag":181,"props":26681,"children":26683},{"className":26682,"style":2804},[2803],[],{"type":150,"tag":181,"props":26685,"children":26686},{"style":8265},[26687,26691],{"type":150,"tag":181,"props":26688,"children":26690},{"className":26689,"style":8240},[333],[],{"type":150,"tag":181,"props":26692,"children":26694},{"className":26693},[296],[26695],{"type":150,"tag":181,"props":26696,"children":26698},{"className":26697},[296],[26699],{"type":155,"value":23340},{"type":150,"tag":181,"props":26701,"children":26703},{"className":26702},[352],[26704],{"type":155,"value":355},{"type":150,"tag":181,"props":26706,"children":26708},{"className":26707},[318],[26709],{"type":150,"tag":181,"props":26710,"children":26712},{"className":26711,"style":8297},[323],[26713],{"type":150,"tag":181,"props":26714,"children":26715},{},[],{"type":150,"tag":181,"props":26717,"children":26719},{"className":26718},[835,2482],[],{"type":150,"tag":181,"props":26721,"children":26723},{"className":26722,"style":658},[377],[],{"type":150,"tag":181,"props":26725,"children":26727},{"className":26726},[663],[26728],{"type":155,"value":5876},{"type":150,"tag":181,"props":26730,"children":26732},{"className":26731,"style":658},[377],[],{"type":150,"tag":181,"props":26734,"children":26736},{"className":26735},[285],[26737,26741,26746,26751],{"type":150,"tag":181,"props":26738,"children":26740},{"className":26739,"style":819},[290],[],{"type":150,"tag":181,"props":26742,"children":26744},{"className":26743},[296],[26745],{"type":155,"value":26253},{"type":150,"tag":181,"props":26747,"children":26749},{"className":26748},[377],[26750],{"type":155,"value":8576},{"type":150,"tag":181,"props":26752,"children":26754},{"className":26753},[296,155],[26755],{"type":150,"tag":181,"props":26756,"children":26758},{"className":26757},[296],[26759],{"type":155,"value":23626},{"type":150,"tag":151,"props":26761,"children":26762},{},[26763,26765,26807,26809,26814],{"type":155,"value":26764},"注意 ",{"type":150,"tag":181,"props":26766,"children":26768},{"className":26767},[184],[26769,26790],{"type":150,"tag":181,"props":26770,"children":26772},{"className":26771},[189],[26773],{"type":150,"tag":192,"props":26774,"children":26775},{"xmlns":194},[26776],{"type":150,"tag":197,"props":26777,"children":26778},{},[26779,26786],{"type":150,"tag":201,"props":26780,"children":26781},{},[26782],{"type":150,"tag":209,"props":26783,"children":26784},{},[26785],{"type":155,"value":5385},{"type":150,"tag":271,"props":26787,"children":26788},{"encoding":273},[26789],{"type":155,"value":5385},{"type":150,"tag":181,"props":26791,"children":26793},{"className":26792,"ariaHidden":223},[280],[26794],{"type":150,"tag":181,"props":26795,"children":26797},{"className":26796},[285],[26798,26802],{"type":150,"tag":181,"props":26799,"children":26801},{"className":26800,"style":5680},[290],[],{"type":150,"tag":181,"props":26803,"children":26805},{"className":26804},[296,301],[26806],{"type":155,"value":5385},{"type":155,"value":26808}," 被约掉了。",{"type":150,"tag":158,"props":26810,"children":26811},{},[26812],{"type":155,"value":26813},"无论多少层，这个上限都一样",{"type":155,"value":26815},"，因为 KV 和 attention 计算都随层数线性增长。",{"type":150,"tag":151,"props":26817,"children":26818},{},[26819,26824],{"type":150,"tag":158,"props":26820,"children":26821},{},[26822],{"type":155,"value":26823},"242 小于 295。",{"type":155,"value":26825}," 也就是说，不管 context 多长，MLA 的 decode 都落在拐点左边，是 memory bound。那 4.25 倍的计算量加进去就是白加的。算力本来就在空转等内存。",{"type":150,"tag":151,"props":26827,"children":26828},{},[26829],{"type":155,"value":26830},"具体看几个长度：",{"type":150,"tag":22173,"props":26832,"children":26833},{},[26834,26905],{"type":150,"tag":22177,"props":26835,"children":26836},{},[26837],{"type":150,"tag":22181,"props":26838,"children":26839},{},[26840,26885,26890,26895,26900],{"type":150,"tag":22185,"props":26841,"children":26842},{},[26843],{"type":150,"tag":181,"props":26844,"children":26846},{"className":26845},[184],[26847,26868],{"type":150,"tag":181,"props":26848,"children":26850},{"className":26849},[189],[26851],{"type":150,"tag":192,"props":26852,"children":26853},{"xmlns":194},[26854],{"type":150,"tag":197,"props":26855,"children":26856},{},[26857,26864],{"type":150,"tag":201,"props":26858,"children":26859},{},[26860],{"type":150,"tag":209,"props":26861,"children":26862},{},[26863],{"type":155,"value":5417},{"type":150,"tag":271,"props":26865,"children":26866},{"encoding":273},[26867],{"type":155,"value":5417},{"type":150,"tag":181,"props":26869,"children":26871},{"className":26870,"ariaHidden":223},[280],[26872],{"type":150,"tag":181,"props":26873,"children":26875},{"className":26874},[285],[26876,26880],{"type":150,"tag":181,"props":26877,"children":26879},{"className":26878,"style":5680},[290],[],{"type":150,"tag":181,"props":26881,"children":26883},{"className":26882,"style":5606},[296,301],[26884],{"type":155,"value":5417},{"type":150,"tag":22185,"props":26886,"children":26887},{},[26888],{"type":155,"value":26889},"算术强度",{"type":150,"tag":22185,"props":26891,"children":26892},{},[26893],{"type":155,"value":26894},"内存耗时",{"type":150,"tag":22185,"props":26896,"children":26897},{},[26898],{"type":155,"value":26899},"计算耗时",{"type":150,"tag":22185,"props":26901,"children":26902},{},[26903],{"type":155,"value":26904},"瓶颈",{"type":150,"tag":22199,"props":26906,"children":26907},{},[26908,26936,26963,26990],{"type":150,"tag":22181,"props":26909,"children":26910},{},[26911,26916,26921,26926,26931],{"type":150,"tag":22206,"props":26912,"children":26913},{},[26914],{"type":155,"value":26915},"4k",{"type":150,"tag":22206,"props":26917,"children":26918},{},[26919],{"type":155,"value":26920},"2.6",{"type":150,"tag":22206,"props":26922,"children":26923},{},[26924],{"type":155,"value":26925},"12.6 ms",{"type":150,"tag":22206,"props":26927,"children":26928},{},[26929],{"type":155,"value":26930},"0.11 ms",{"type":150,"tag":22206,"props":26932,"children":26933},{},[26934],{"type":155,"value":26935},"memory",{"type":150,"tag":22181,"props":26937,"children":26938},{},[26939,26944,26949,26954,26959],{"type":150,"tag":22206,"props":26940,"children":26941},{},[26942],{"type":155,"value":26943},"32k",{"type":150,"tag":22206,"props":26945,"children":26946},{},[26947],{"type":155,"value":26948},"13.3",{"type":150,"tag":22206,"props":26950,"children":26951},{},[26952],{"type":155,"value":26953},"13.2 ms",{"type":150,"tag":22206,"props":26955,"children":26956},{},[26957],{"type":155,"value":26958},"0.60 ms",{"type":150,"tag":22206,"props":26960,"children":26961},{},[26962],{"type":155,"value":26935},{"type":150,"tag":22181,"props":26964,"children":26965},{},[26966,26971,26976,26981,26986],{"type":150,"tag":22206,"props":26967,"children":26968},{},[26969],{"type":155,"value":26970},"128k",{"type":150,"tag":22206,"props":26972,"children":26973},{},[26974],{"type":155,"value":26975},"43.7",{"type":150,"tag":22206,"props":26977,"children":26978},{},[26979],{"type":155,"value":26980},"15.2 ms",{"type":150,"tag":22206,"props":26982,"children":26983},{},[26984],{"type":155,"value":26985},"2.3 ms",{"type":150,"tag":22206,"props":26987,"children":26988},{},[26989],{"type":155,"value":26935},{"type":150,"tag":22181,"props":26991,"children":26992},{},[26993,26998,27003,27008,27013],{"type":150,"tag":22206,"props":26994,"children":26995},{},[26996],{"type":155,"value":26997},"1M",{"type":150,"tag":22206,"props":26999,"children":27000},{},[27001],{"type":155,"value":27002},"153",{"type":150,"tag":22206,"props":27004,"children":27005},{},[27006],{"type":155,"value":27007},"34.2 ms",{"type":150,"tag":22206,"props":27009,"children":27010},{},[27011],{"type":155,"value":27012},"17.8 ms",{"type":150,"tag":22206,"props":27014,"children":27015},{},[27016],{"type":155,"value":26935},{"type":150,"tag":151,"props":27018,"children":27019},{},[27020],{"type":155,"value":27021},"到 1M 的时候计算耗时（17.8 ms）已经追到内存耗时（34.2 ms）的一半了，但还没越过去。",{"type":150,"tag":151,"props":27023,"children":27024},{},[27025],{"type":155,"value":27026},"不过这里只算了 batch size 等于 1。在线服务几乎不会单条跑，batch 是提高算术强度的另一条路子，而且它会改变结论。",{"type":150,"tag":151,"props":27028,"children":27029},{},[27030,27032,27075],{"type":155,"value":27031},"原因很直接：权重每一步只需要读一次，然后同时喂给 batch 里所有序列；KV 读取却是一条序列一份。把 ",{"type":150,"tag":181,"props":27033,"children":27035},{"className":27034},[184],[27036,27057],{"type":150,"tag":181,"props":27037,"children":27039},{"className":27038},[189],[27040],{"type":150,"tag":192,"props":27041,"children":27042},{"xmlns":194},[27043],{"type":150,"tag":197,"props":27044,"children":27045},{},[27046,27053],{"type":150,"tag":201,"props":27047,"children":27048},{},[27049],{"type":150,"tag":209,"props":27050,"children":27051},{},[27052],{"type":155,"value":23188},{"type":150,"tag":271,"props":27054,"children":27055},{"encoding":273},[27056],{"type":155,"value":23188},{"type":150,"tag":181,"props":27058,"children":27060},{"className":27059,"ariaHidden":223},[280],[27061],{"type":150,"tag":181,"props":27062,"children":27064},{"className":27063},[285],[27065,27069],{"type":150,"tag":181,"props":27066,"children":27068},{"className":27067,"style":5680},[290],[],{"type":150,"tag":181,"props":27070,"children":27073},{"className":27071,"style":27072},[296,301],"margin-right:0.0502em;",[27074],{"type":155,"value":23188},{"type":155,"value":27076}," 写进去：",{"type":150,"tag":181,"props":27078,"children":27080},{"className":27079},[1003],[27081],{"type":150,"tag":181,"props":27082,"children":27084},{"className":27083},[184],[27085,27221],{"type":150,"tag":181,"props":27086,"children":27088},{"className":27087},[189],[27089],{"type":150,"tag":192,"props":27090,"children":27091},{"xmlns":194,"display":1015},[27092],{"type":150,"tag":197,"props":27093,"children":27094},{},[27095,27216],{"type":150,"tag":201,"props":27096,"children":27097},{},[27098,27102,27106,27110,27114,27118,27122,27126],{"type":150,"tag":209,"props":27099,"children":27100},{},[27101],{"type":155,"value":24937},{"type":150,"tag":221,"props":27103,"children":27104},{"stretchy":789},[27105],{"type":155,"value":792},{"type":150,"tag":209,"props":27107,"children":27108},{},[27109],{"type":155,"value":23188},{"type":150,"tag":221,"props":27111,"children":27112},{"separator":223},[27113],{"type":155,"value":226},{"type":150,"tag":209,"props":27115,"children":27116},{},[27117],{"type":155,"value":5417},{"type":150,"tag":221,"props":27119,"children":27120},{"stretchy":789},[27121],{"type":155,"value":802},{"type":150,"tag":221,"props":27123,"children":27124},{},[27125],{"type":155,"value":865},{"type":150,"tag":1774,"props":27127,"children":27128},{},[27129,27178],{"type":150,"tag":201,"props":27130,"children":27131},{},[27132,27136],{"type":150,"tag":209,"props":27133,"children":27134},{},[27135],{"type":155,"value":23188},{"type":150,"tag":201,"props":27137,"children":27138},{},[27139,27143,27147,27151,27155,27166,27170,27174],{"type":150,"tag":221,"props":27140,"children":27141},{"fence":223},[27142],{"type":155,"value":792},{"type":150,"tag":215,"props":27144,"children":27145},{},[27146],{"type":155,"value":238},{"type":150,"tag":209,"props":27148,"children":27149},{},[27150],{"type":155,"value":25017},{"type":150,"tag":221,"props":27152,"children":27153},{},[27154],{"type":155,"value":4962},{"type":150,"tag":205,"props":27156,"children":27157},{},[27158,27162],{"type":150,"tag":209,"props":27159,"children":27160},{},[27161],{"type":155,"value":9859},{"type":150,"tag":249,"props":27163,"children":27164},{},[27165],{"type":155,"value":24228},{"type":150,"tag":209,"props":27167,"children":27168},{},[27169],{"type":155,"value":5385},{"type":150,"tag":209,"props":27171,"children":27172},{},[27173],{"type":155,"value":5417},{"type":150,"tag":221,"props":27175,"children":27176},{"fence":223},[27177],{"type":155,"value":802},{"type":150,"tag":201,"props":27179,"children":27180},{},[27181,27185,27189,27193,27197,27208,27212],{"type":150,"tag":209,"props":27182,"children":27183},{},[27184],{"type":155,"value":3265},{"type":150,"tag":221,"props":27186,"children":27187},{},[27188],{"type":155,"value":4962},{"type":150,"tag":209,"props":27190,"children":27191},{},[27192],{"type":155,"value":23188},{"type":150,"tag":221,"props":27194,"children":27195},{},[27196],{"type":155,"value":25037},{"type":150,"tag":205,"props":27198,"children":27199},{},[27200,27204],{"type":150,"tag":209,"props":27201,"children":27202},{},[27203],{"type":155,"value":9859},{"type":150,"tag":249,"props":27205,"children":27206},{},[27207],{"type":155,"value":23796},{"type":150,"tag":209,"props":27209,"children":27210},{},[27211],{"type":155,"value":5385},{"type":150,"tag":209,"props":27213,"children":27214},{},[27215],{"type":155,"value":5417},{"type":150,"tag":271,"props":27217,"children":27218},{"encoding":273},[27219],{"type":155,"value":27220},"I(B, N) = \\frac{B\\left(2P + c_{\\text{flop}} L N\\right)}{W + B \\cdot c_{\\text{kv}} L N}",{"type":150,"tag":181,"props":27222,"children":27224},{"className":27223,"ariaHidden":223},[280],[27225,27280],{"type":150,"tag":181,"props":27226,"children":27228},{"className":27227},[285],[27229,27233,27238,27243,27248,27253,27257,27262,27267,27271,27276],{"type":150,"tag":181,"props":27230,"children":27232},{"className":27231,"style":819},[290],[],{"type":150,"tag":181,"props":27234,"children":27236},{"className":27235,"style":24957},[296,301],[27237],{"type":155,"value":24937},{"type":150,"tag":181,"props":27239,"children":27241},{"className":27240},[824],[27242],{"type":155,"value":792},{"type":150,"tag":181,"props":27244,"children":27246},{"className":27245,"style":27072},[296,301],[27247],{"type":155,"value":23188},{"type":150,"tag":181,"props":27249,"children":27251},{"className":27250},[371],[27252],{"type":155,"value":226},{"type":150,"tag":181,"props":27254,"children":27256},{"className":27255,"style":378},[377],[],{"type":150,"tag":181,"props":27258,"children":27260},{"className":27259,"style":5606},[296,301],[27261],{"type":155,"value":5417},{"type":150,"tag":181,"props":27263,"children":27265},{"className":27264},[835],[27266],{"type":155,"value":802},{"type":150,"tag":181,"props":27268,"children":27270},{"className":27269,"style":658},[377],[],{"type":150,"tag":181,"props":27272,"children":27274},{"className":27273},[663],[27275],{"type":155,"value":865},{"type":150,"tag":181,"props":27277,"children":27279},{"className":27278,"style":658},[377],[],{"type":150,"tag":181,"props":27281,"children":27283},{"className":27282},[285],[27284,27289],{"type":150,"tag":181,"props":27285,"children":27288},{"className":27286,"style":27287},[290],"height:2.263em;vertical-align:-0.836em;",[],{"type":150,"tag":181,"props":27290,"children":27292},{"className":27291},[296],[27293,27297,27595],{"type":150,"tag":181,"props":27294,"children":27296},{"className":27295},[824,2482],[],{"type":150,"tag":181,"props":27298,"children":27300},{"className":27299},[1774],[27301],{"type":150,"tag":181,"props":27302,"children":27304},{"className":27303},[312,313],[27305,27584],{"type":150,"tag":181,"props":27306,"children":27308},{"className":27307},[318],[27309,27579],{"type":150,"tag":181,"props":27310,"children":27313},{"className":27311,"style":27312},[323],"height:1.427em;",[27314,27436,27447],{"type":150,"tag":181,"props":27315,"children":27316},{"style":8235},[27317,27321],{"type":150,"tag":181,"props":27318,"children":27320},{"className":27319,"style":8240},[333],[],{"type":150,"tag":181,"props":27322,"children":27324},{"className":27323},[296],[27325,27330,27334,27339,27343,27348,27352,27357,27361,27426,27431],{"type":150,"tag":181,"props":27326,"children":27328},{"className":27327,"style":3691},[296,301],[27329],{"type":155,"value":3265},{"type":150,"tag":181,"props":27331,"children":27333},{"className":27332,"style":4993},[377],[],{"type":150,"tag":181,"props":27335,"children":27337},{"className":27336},[4998],[27338],{"type":155,"value":4962},{"type":150,"tag":181,"props":27340,"children":27342},{"className":27341,"style":4993},[377],[],{"type":150,"tag":181,"props":27344,"children":27346},{"className":27345,"style":27072},[296,301],[27347],{"type":155,"value":23188},{"type":150,"tag":181,"props":27349,"children":27351},{"className":27350,"style":4993},[377],[],{"type":150,"tag":181,"props":27353,"children":27355},{"className":27354},[4998],[27356],{"type":155,"value":25037},{"type":150,"tag":181,"props":27358,"children":27360},{"className":27359,"style":4993},[377],[],{"type":150,"tag":181,"props":27362,"children":27364},{"className":27363},[296],[27365,27370],{"type":150,"tag":181,"props":27366,"children":27368},{"className":27367},[296,301],[27369],{"type":155,"value":9859},{"type":150,"tag":181,"props":27371,"children":27373},{"className":27372},[307],[27374],{"type":150,"tag":181,"props":27375,"children":27377},{"className":27376},[312,313],[27378,27415],{"type":150,"tag":181,"props":27379,"children":27381},{"className":27380},[318],[27382,27410],{"type":150,"tag":181,"props":27383,"children":27385},{"className":27384,"style":493},[323],[27386],{"type":150,"tag":181,"props":27387,"children":27388},{"style":328},[27389,27393],{"type":150,"tag":181,"props":27390,"children":27392},{"className":27391,"style":334},[333],[],{"type":150,"tag":181,"props":27394,"children":27396},{"className":27395},[339,340,341,342],[27397],{"type":150,"tag":181,"props":27398,"children":27400},{"className":27399},[296,342],[27401],{"type":150,"tag":181,"props":27402,"children":27404},{"className":27403},[296,155,342],[27405],{"type":150,"tag":181,"props":27406,"children":27408},{"className":27407},[296,342],[27409],{"type":155,"value":23796},{"type":150,"tag":181,"props":27411,"children":27413},{"className":27412},[352],[27414],{"type":155,"value":355},{"type":150,"tag":181,"props":27416,"children":27418},{"className":27417},[318],[27419],{"type":150,"tag":181,"props":27420,"children":27422},{"className":27421,"style":363},[323],[27423],{"type":150,"tag":181,"props":27424,"children":27425},{},[],{"type":150,"tag":181,"props":27427,"children":27429},{"className":27428},[296,301],[27430],{"type":155,"value":5385},{"type":150,"tag":181,"props":27432,"children":27434},{"className":27433,"style":5606},[296,301],[27435],{"type":155,"value":5417},{"type":150,"tag":181,"props":27437,"children":27438},{"style":8253},[27439,27443],{"type":150,"tag":181,"props":27440,"children":27442},{"className":27441,"style":8240},[333],[],{"type":150,"tag":181,"props":27444,"children":27446},{"className":27445,"style":2804},[2803],[],{"type":150,"tag":181,"props":27448,"children":27449},{"style":8265},[27450,27454],{"type":150,"tag":181,"props":27451,"children":27453},{"className":27452,"style":8240},[333],[],{"type":150,"tag":181,"props":27455,"children":27457},{"className":27456},[296],[27458,27463,27467],{"type":150,"tag":181,"props":27459,"children":27461},{"className":27460,"style":27072},[296,301],[27462],{"type":155,"value":23188},{"type":150,"tag":181,"props":27464,"children":27466},{"className":27465,"style":378},[377],[],{"type":150,"tag":181,"props":27468,"children":27470},{"className":27469},[449],[27471,27476,27481,27486,27490,27495,27499,27564,27569,27574],{"type":150,"tag":181,"props":27472,"children":27474},{"className":27473,"style":8199},[824,8198],[27475],{"type":155,"value":792},{"type":150,"tag":181,"props":27477,"children":27479},{"className":27478},[296],[27480],{"type":155,"value":238},{"type":150,"tag":181,"props":27482,"children":27484},{"className":27483,"style":3691},[296,301],[27485],{"type":155,"value":25017},{"type":150,"tag":181,"props":27487,"children":27489},{"className":27488,"style":4993},[377],[],{"type":150,"tag":181,"props":27491,"children":27493},{"className":27492},[4998],[27494],{"type":155,"value":4962},{"type":150,"tag":181,"props":27496,"children":27498},{"className":27497,"style":4993},[377],[],{"type":150,"tag":181,"props":27500,"children":27502},{"className":27501},[296],[27503,27508],{"type":150,"tag":181,"props":27504,"children":27506},{"className":27505},[296,301],[27507],{"type":155,"value":9859},{"type":150,"tag":181,"props":27509,"children":27511},{"className":27510},[307],[27512],{"type":150,"tag":181,"props":27513,"children":27515},{"className":27514},[312,313],[27516,27553],{"type":150,"tag":181,"props":27517,"children":27519},{"className":27518},[318],[27520,27548],{"type":150,"tag":181,"props":27521,"children":27523},{"className":27522,"style":493},[323],[27524],{"type":150,"tag":181,"props":27525,"children":27526},{"style":328},[27527,27531],{"type":150,"tag":181,"props":27528,"children":27530},{"className":27529,"style":334},[333],[],{"type":150,"tag":181,"props":27532,"children":27534},{"className":27533},[339,340,341,342],[27535],{"type":150,"tag":181,"props":27536,"children":27538},{"className":27537},[296,342],[27539],{"type":150,"tag":181,"props":27540,"children":27542},{"className":27541},[296,155,342],[27543],{"type":150,"tag":181,"props":27544,"children":27546},{"className":27545},[296,342],[27547],{"type":155,"value":24228},{"type":150,"tag":181,"props":27549,"children":27551},{"className":27550},[352],[27552],{"type":155,"value":355},{"type":150,"tag":181,"props":27554,"children":27556},{"className":27555},[318],[27557],{"type":150,"tag":181,"props":27558,"children":27560},{"className":27559,"style":6518},[323],[27561],{"type":150,"tag":181,"props":27562,"children":27563},{},[],{"type":150,"tag":181,"props":27565,"children":27567},{"className":27566},[296,301],[27568],{"type":155,"value":5385},{"type":150,"tag":181,"props":27570,"children":27572},{"className":27571,"style":5606},[296,301],[27573],{"type":155,"value":5417},{"type":150,"tag":181,"props":27575,"children":27577},{"className":27576,"style":8199},[835,8198],[27578],{"type":155,"value":802},{"type":150,"tag":181,"props":27580,"children":27582},{"className":27581},[352],[27583],{"type":155,"value":355},{"type":150,"tag":181,"props":27585,"children":27587},{"className":27586},[318],[27588],{"type":150,"tag":181,"props":27589,"children":27591},{"className":27590,"style":25459},[323],[27592],{"type":150,"tag":181,"props":27593,"children":27594},{},[],{"type":150,"tag":181,"props":27596,"children":27598},{"className":27597},[835,2482],[],{"type":150,"tag":151,"props":27600,"children":27601},{},[27602,27644,27646,27688],{"type":150,"tag":181,"props":27603,"children":27605},{"className":27604},[184],[27606,27627],{"type":150,"tag":181,"props":27607,"children":27609},{"className":27608},[189],[27610],{"type":150,"tag":192,"props":27611,"children":27612},{"xmlns":194},[27613],{"type":150,"tag":197,"props":27614,"children":27615},{},[27616,27623],{"type":150,"tag":201,"props":27617,"children":27618},{},[27619],{"type":150,"tag":209,"props":27620,"children":27621},{},[27622],{"type":155,"value":23188},{"type":150,"tag":271,"props":27624,"children":27625},{"encoding":273},[27626],{"type":155,"value":23188},{"type":150,"tag":181,"props":27628,"children":27630},{"className":27629,"ariaHidden":223},[280],[27631],{"type":150,"tag":181,"props":27632,"children":27634},{"className":27633},[285],[27635,27639],{"type":150,"tag":181,"props":27636,"children":27638},{"className":27637,"style":5680},[290],[],{"type":150,"tag":181,"props":27640,"children":27642},{"className":27641,"style":27072},[296,301],[27643],{"type":155,"value":23188},{"type":155,"value":27645}," 一大，分母里的 ",{"type":150,"tag":181,"props":27647,"children":27649},{"className":27648},[184],[27650,27671],{"type":150,"tag":181,"props":27651,"children":27653},{"className":27652},[189],[27654],{"type":150,"tag":192,"props":27655,"children":27656},{"xmlns":194},[27657],{"type":150,"tag":197,"props":27658,"children":27659},{},[27660,27667],{"type":150,"tag":201,"props":27661,"children":27662},{},[27663],{"type":150,"tag":209,"props":27664,"children":27665},{},[27666],{"type":155,"value":3265},{"type":150,"tag":271,"props":27668,"children":27669},{"encoding":273},[27670],{"type":155,"value":3265},{"type":150,"tag":181,"props":27672,"children":27674},{"className":27673,"ariaHidden":223},[280],[27675],{"type":150,"tag":181,"props":27676,"children":27678},{"className":27677},[285],[27679,27683],{"type":150,"tag":181,"props":27680,"children":27682},{"className":27681,"style":5680},[290],[],{"type":150,"tag":181,"props":27684,"children":27686},{"className":27685,"style":3691},[296,301],[27687],{"type":155,"value":3265},{"type":155,"value":27689}," 就被摊薄，算术强度趋近于：",{"type":150,"tag":181,"props":27691,"children":27693},{"className":27692},[1003],[27694],{"type":150,"tag":181,"props":27695,"children":27697},{"className":27696},[184],[27698,27803],{"type":150,"tag":181,"props":27699,"children":27701},{"className":27700},[189],[27702],{"type":150,"tag":192,"props":27703,"children":27704},{"xmlns":194,"display":1015},[27705],{"type":150,"tag":197,"props":27706,"children":27707},{},[27708,27798],{"type":150,"tag":201,"props":27709,"children":27710},{},[27711,27715,27719,27723,27727,27731,27735,27739],{"type":150,"tag":209,"props":27712,"children":27713},{},[27714],{"type":155,"value":24937},{"type":150,"tag":221,"props":27716,"children":27717},{"stretchy":789},[27718],{"type":155,"value":792},{"type":150,"tag":209,"props":27720,"children":27721},{"mathvariant":1618},[27722],{"type":155,"value":26184},{"type":150,"tag":221,"props":27724,"children":27725},{"separator":223},[27726],{"type":155,"value":226},{"type":150,"tag":209,"props":27728,"children":27729},{},[27730],{"type":155,"value":5417},{"type":150,"tag":221,"props":27732,"children":27733},{"stretchy":789},[27734],{"type":155,"value":802},{"type":150,"tag":221,"props":27736,"children":27737},{},[27738],{"type":155,"value":865},{"type":150,"tag":1774,"props":27740,"children":27741},{},[27742,27776],{"type":150,"tag":201,"props":27743,"children":27744},{},[27745,27749,27753,27757,27768,27772],{"type":150,"tag":215,"props":27746,"children":27747},{},[27748],{"type":155,"value":238},{"type":150,"tag":209,"props":27750,"children":27751},{},[27752],{"type":155,"value":25017},{"type":150,"tag":221,"props":27754,"children":27755},{},[27756],{"type":155,"value":4962},{"type":150,"tag":205,"props":27758,"children":27759},{},[27760,27764],{"type":150,"tag":209,"props":27761,"children":27762},{},[27763],{"type":155,"value":9859},{"type":150,"tag":249,"props":27765,"children":27766},{},[27767],{"type":155,"value":24228},{"type":150,"tag":209,"props":27769,"children":27770},{},[27771],{"type":155,"value":5385},{"type":150,"tag":209,"props":27773,"children":27774},{},[27775],{"type":155,"value":5417},{"type":150,"tag":201,"props":27777,"children":27778},{},[27779,27790,27794],{"type":150,"tag":205,"props":27780,"children":27781},{},[27782,27786],{"type":150,"tag":209,"props":27783,"children":27784},{},[27785],{"type":155,"value":9859},{"type":150,"tag":249,"props":27787,"children":27788},{},[27789],{"type":155,"value":23796},{"type":150,"tag":209,"props":27791,"children":27792},{},[27793],{"type":155,"value":5385},{"type":150,"tag":209,"props":27795,"children":27796},{},[27797],{"type":155,"value":5417},{"type":150,"tag":271,"props":27799,"children":27800},{"encoding":273},[27801],{"type":155,"value":27802},"I(\\infty, N) = \\frac{2P + c_{\\text{flop}} L N}{c_{\\text{kv}} L N}",{"type":150,"tag":181,"props":27804,"children":27806},{"className":27805,"ariaHidden":223},[280],[27807,27862],{"type":150,"tag":181,"props":27808,"children":27810},{"className":27809},[285],[27811,27815,27820,27825,27830,27835,27839,27844,27849,27853,27858],{"type":150,"tag":181,"props":27812,"children":27814},{"className":27813,"style":819},[290],[],{"type":150,"tag":181,"props":27816,"children":27818},{"className":27817,"style":24957},[296,301],[27819],{"type":155,"value":24937},{"type":150,"tag":181,"props":27821,"children":27823},{"className":27822},[824],[27824],{"type":155,"value":792},{"type":150,"tag":181,"props":27826,"children":27828},{"className":27827},[296],[27829],{"type":155,"value":26184},{"type":150,"tag":181,"props":27831,"children":27833},{"className":27832},[371],[27834],{"type":155,"value":226},{"type":150,"tag":181,"props":27836,"children":27838},{"className":27837,"style":378},[377],[],{"type":150,"tag":181,"props":27840,"children":27842},{"className":27841,"style":5606},[296,301],[27843],{"type":155,"value":5417},{"type":150,"tag":181,"props":27845,"children":27847},{"className":27846},[835],[27848],{"type":155,"value":802},{"type":150,"tag":181,"props":27850,"children":27852},{"className":27851,"style":658},[377],[],{"type":150,"tag":181,"props":27854,"children":27856},{"className":27855},[663],[27857],{"type":155,"value":865},{"type":150,"tag":181,"props":27859,"children":27861},{"className":27860,"style":658},[377],[],{"type":150,"tag":181,"props":27863,"children":27865},{"className":27864},[285],[27866,27870],{"type":150,"tag":181,"props":27867,"children":27869},{"className":27868,"style":25145},[290],[],{"type":150,"tag":181,"props":27871,"children":27873},{"className":27872},[296],[27874,27878,28116],{"type":150,"tag":181,"props":27875,"children":27877},{"className":27876},[824,2482],[],{"type":150,"tag":181,"props":27879,"children":27881},{"className":27880},[1774],[27882],{"type":150,"tag":181,"props":27883,"children":27885},{"className":27884},[312,313],[27886,28105],{"type":150,"tag":181,"props":27887,"children":27889},{"className":27888},[318],[27890,28100],{"type":150,"tag":181,"props":27891,"children":27893},{"className":27892,"style":23490},[323],[27894,27980,27991],{"type":150,"tag":181,"props":27895,"children":27896},{"style":8235},[27897,27901],{"type":150,"tag":181,"props":27898,"children":27900},{"className":27899,"style":8240},[333],[],{"type":150,"tag":181,"props":27902,"children":27904},{"className":27903},[296],[27905,27970,27975],{"type":150,"tag":181,"props":27906,"children":27908},{"className":27907},[296],[27909,27914],{"type":150,"tag":181,"props":27910,"children":27912},{"className":27911},[296,301],[27913],{"type":155,"value":9859},{"type":150,"tag":181,"props":27915,"children":27917},{"className":27916},[307],[27918],{"type":150,"tag":181,"props":27919,"children":27921},{"className":27920},[312,313],[27922,27959],{"type":150,"tag":181,"props":27923,"children":27925},{"className":27924},[318],[27926,27954],{"type":150,"tag":181,"props":27927,"children":27929},{"className":27928,"style":493},[323],[27930],{"type":150,"tag":181,"props":27931,"children":27932},{"style":328},[27933,27937],{"type":150,"tag":181,"props":27934,"children":27936},{"className":27935,"style":334},[333],[],{"type":150,"tag":181,"props":27938,"children":27940},{"className":27939},[339,340,341,342],[27941],{"type":150,"tag":181,"props":27942,"children":27944},{"className":27943},[296,342],[27945],{"type":150,"tag":181,"props":27946,"children":27948},{"className":27947},[296,155,342],[27949],{"type":150,"tag":181,"props":27950,"children":27952},{"className":27951},[296,342],[27953],{"type":155,"value":23796},{"type":150,"tag":181,"props":27955,"children":27957},{"className":27956},[352],[27958],{"type":155,"value":355},{"type":150,"tag":181,"props":27960,"children":27962},{"className":27961},[318],[27963],{"type":150,"tag":181,"props":27964,"children":27966},{"className":27965,"style":363},[323],[27967],{"type":150,"tag":181,"props":27968,"children":27969},{},[],{"type":150,"tag":181,"props":27971,"children":27973},{"className":27972},[296,301],[27974],{"type":155,"value":5385},{"type":150,"tag":181,"props":27976,"children":27978},{"className":27977,"style":5606},[296,301],[27979],{"type":155,"value":5417},{"type":150,"tag":181,"props":27981,"children":27982},{"style":8253},[27983,27987],{"type":150,"tag":181,"props":27984,"children":27986},{"className":27985,"style":8240},[333],[],{"type":150,"tag":181,"props":27988,"children":27990},{"className":27989,"style":2804},[2803],[],{"type":150,"tag":181,"props":27992,"children":27993},{"style":8265},[27994,27998],{"type":150,"tag":181,"props":27995,"children":27997},{"className":27996,"style":8240},[333],[],{"type":150,"tag":181,"props":27999,"children":28001},{"className":28000},[296],[28002,28007,28012,28016,28021,28025,28090,28095],{"type":150,"tag":181,"props":28003,"children":28005},{"className":28004},[296],[28006],{"type":155,"value":238},{"type":150,"tag":181,"props":28008,"children":28010},{"className":28009,"style":3691},[296,301],[28011],{"type":155,"value":25017},{"type":150,"tag":181,"props":28013,"children":28015},{"className":28014,"style":4993},[377],[],{"type":150,"tag":181,"props":28017,"children":28019},{"className":28018},[4998],[28020],{"type":155,"value":4962},{"type":150,"tag":181,"props":28022,"children":28024},{"className":28023,"style":4993},[377],[],{"type":150,"tag":181,"props":28026,"children":28028},{"className":28027},[296],[28029,28034],{"type":150,"tag":181,"props":28030,"children":28032},{"className":28031},[296,301],[28033],{"type":155,"value":9859},{"type":150,"tag":181,"props":28035,"children":28037},{"className":28036},[307],[28038],{"type":150,"tag":181,"props":28039,"children":28041},{"className":28040},[312,313],[28042,28079],{"type":150,"tag":181,"props":28043,"children":28045},{"className":28044},[318],[28046,28074],{"type":150,"tag":181,"props":28047,"children":28049},{"className":28048,"style":493},[323],[28050],{"type":150,"tag":181,"props":28051,"children":28052},{"style":328},[28053,28057],{"type":150,"tag":181,"props":28054,"children":28056},{"className":28055,"style":334},[333],[],{"type":150,"tag":181,"props":28058,"children":28060},{"className":28059},[339,340,341,342],[28061],{"type":150,"tag":181,"props":28062,"children":28064},{"className":28063},[296,342],[28065],{"type":150,"tag":181,"props":28066,"children":28068},{"className":28067},[296,155,342],[28069],{"type":150,"tag":181,"props":28070,"children":28072},{"className":28071},[296,342],[28073],{"type":155,"value":24228},{"type":150,"tag":181,"props":28075,"children":28077},{"className":28076},[352],[28078],{"type":155,"value":355},{"type":150,"tag":181,"props":28080,"children":28082},{"className":28081},[318],[28083],{"type":150,"tag":181,"props":28084,"children":28086},{"className":28085,"style":6518},[323],[28087],{"type":150,"tag":181,"props":28088,"children":28089},{},[],{"type":150,"tag":181,"props":28091,"children":28093},{"className":28092},[296,301],[28094],{"type":155,"value":5385},{"type":150,"tag":181,"props":28096,"children":28098},{"className":28097,"style":5606},[296,301],[28099],{"type":155,"value":5417},{"type":150,"tag":181,"props":28101,"children":28103},{"className":28102},[352],[28104],{"type":155,"value":355},{"type":150,"tag":181,"props":28106,"children":28108},{"className":28107},[318],[28109],{"type":150,"tag":181,"props":28110,"children":28112},{"className":28111,"style":25459},[323],[28113],{"type":150,"tag":181,"props":28114,"children":28115},{},[],{"type":150,"tag":181,"props":28117,"children":28119},{"className":28118},[835,2482],[],{"type":150,"tag":151,"props":28121,"children":28122},{},[28123],{"type":155,"value":23094},{"type":150,"tag":22173,"props":28125,"children":28126},{},[28127,28578],{"type":150,"tag":22177,"props":28128,"children":28129},{},[28130],{"type":150,"tag":22181,"props":28131,"children":28132},{},[28133,28178,28258,28338,28418,28498],{"type":150,"tag":22185,"props":28134,"children":28135},{},[28136],{"type":150,"tag":181,"props":28137,"children":28139},{"className":28138},[184],[28140,28161],{"type":150,"tag":181,"props":28141,"children":28143},{"className":28142},[189],[28144],{"type":150,"tag":192,"props":28145,"children":28146},{"xmlns":194},[28147],{"type":150,"tag":197,"props":28148,"children":28149},{},[28150,28157],{"type":150,"tag":201,"props":28151,"children":28152},{},[28153],{"type":150,"tag":209,"props":28154,"children":28155},{},[28156],{"type":155,"value":5417},{"type":150,"tag":271,"props":28158,"children":28159},{"encoding":273},[28160],{"type":155,"value":5417},{"type":150,"tag":181,"props":28162,"children":28164},{"className":28163,"ariaHidden":223},[280],[28165],{"type":150,"tag":181,"props":28166,"children":28168},{"className":28167},[285],[28169,28173],{"type":150,"tag":181,"props":28170,"children":28172},{"className":28171,"style":5680},[290],[],{"type":150,"tag":181,"props":28174,"children":28176},{"className":28175,"style":5606},[296,301],[28177],{"type":155,"value":5417},{"type":150,"tag":22185,"props":28179,"children":28180},{},[28181],{"type":150,"tag":181,"props":28182,"children":28184},{"className":28183},[184],[28185,28215],{"type":150,"tag":181,"props":28186,"children":28188},{"className":28187},[189],[28189],{"type":150,"tag":192,"props":28190,"children":28191},{"xmlns":194},[28192],{"type":150,"tag":197,"props":28193,"children":28194},{},[28195,28210],{"type":150,"tag":201,"props":28196,"children":28197},{},[28198,28202,28206],{"type":150,"tag":209,"props":28199,"children":28200},{},[28201],{"type":155,"value":23188},{"type":150,"tag":221,"props":28203,"children":28204},{},[28205],{"type":155,"value":865},{"type":150,"tag":215,"props":28207,"children":28208},{},[28209],{"type":155,"value":219},{"type":150,"tag":271,"props":28211,"children":28212},{"encoding":273},[28213],{"type":155,"value":28214},"B=1",{"type":150,"tag":181,"props":28216,"children":28218},{"className":28217,"ariaHidden":223},[280],[28219,28245],{"type":150,"tag":181,"props":28220,"children":28222},{"className":28221},[285],[28223,28227,28232,28236,28241],{"type":150,"tag":181,"props":28224,"children":28226},{"className":28225,"style":5680},[290],[],{"type":150,"tag":181,"props":28228,"children":28230},{"className":28229,"style":27072},[296,301],[28231],{"type":155,"value":23188},{"type":150,"tag":181,"props":28233,"children":28235},{"className":28234,"style":658},[377],[],{"type":150,"tag":181,"props":28237,"children":28239},{"className":28238},[663],[28240],{"type":155,"value":865},{"type":150,"tag":181,"props":28242,"children":28244},{"className":28243,"style":658},[377],[],{"type":150,"tag":181,"props":28246,"children":28248},{"className":28247},[285],[28249,28253],{"type":150,"tag":181,"props":28250,"children":28252},{"className":28251,"style":5012},[290],[],{"type":150,"tag":181,"props":28254,"children":28256},{"className":28255},[296],[28257],{"type":155,"value":219},{"type":150,"tag":22185,"props":28259,"children":28260},{},[28261],{"type":150,"tag":181,"props":28262,"children":28264},{"className":28263},[184],[28265,28295],{"type":150,"tag":181,"props":28266,"children":28268},{"className":28267},[189],[28269],{"type":150,"tag":192,"props":28270,"children":28271},{"xmlns":194},[28272],{"type":150,"tag":197,"props":28273,"children":28274},{},[28275,28290],{"type":150,"tag":201,"props":28276,"children":28277},{},[28278,28282,28286],{"type":150,"tag":209,"props":28279,"children":28280},{},[28281],{"type":155,"value":23188},{"type":150,"tag":221,"props":28283,"children":28284},{},[28285],{"type":155,"value":865},{"type":150,"tag":215,"props":28287,"children":28288},{},[28289],{"type":155,"value":9202},{"type":150,"tag":271,"props":28291,"children":28292},{"encoding":273},[28293],{"type":155,"value":28294},"B=8",{"type":150,"tag":181,"props":28296,"children":28298},{"className":28297,"ariaHidden":223},[280],[28299,28325],{"type":150,"tag":181,"props":28300,"children":28302},{"className":28301},[285],[28303,28307,28312,28316,28321],{"type":150,"tag":181,"props":28304,"children":28306},{"className":28305,"style":5680},[290],[],{"type":150,"tag":181,"props":28308,"children":28310},{"className":28309,"style":27072},[296,301],[28311],{"type":155,"value":23188},{"type":150,"tag":181,"props":28313,"children":28315},{"className":28314,"style":658},[377],[],{"type":150,"tag":181,"props":28317,"children":28319},{"className":28318},[663],[28320],{"type":155,"value":865},{"type":150,"tag":181,"props":28322,"children":28324},{"className":28323,"style":658},[377],[],{"type":150,"tag":181,"props":28326,"children":28328},{"className":28327},[285],[28329,28333],{"type":150,"tag":181,"props":28330,"children":28332},{"className":28331,"style":5012},[290],[],{"type":150,"tag":181,"props":28334,"children":28336},{"className":28335},[296],[28337],{"type":155,"value":9202},{"type":150,"tag":22185,"props":28339,"children":28340},{},[28341],{"type":150,"tag":181,"props":28342,"children":28344},{"className":28343},[184],[28345,28375],{"type":150,"tag":181,"props":28346,"children":28348},{"className":28347},[189],[28349],{"type":150,"tag":192,"props":28350,"children":28351},{"xmlns":194},[28352],{"type":150,"tag":197,"props":28353,"children":28354},{},[28355,28370],{"type":150,"tag":201,"props":28356,"children":28357},{},[28358,28362,28366],{"type":150,"tag":209,"props":28359,"children":28360},{},[28361],{"type":155,"value":23188},{"type":150,"tag":221,"props":28363,"children":28364},{},[28365],{"type":155,"value":865},{"type":150,"tag":215,"props":28367,"children":28368},{},[28369],{"type":155,"value":20641},{"type":150,"tag":271,"props":28371,"children":28372},{"encoding":273},[28373],{"type":155,"value":28374},"B=64",{"type":150,"tag":181,"props":28376,"children":28378},{"className":28377,"ariaHidden":223},[280],[28379,28405],{"type":150,"tag":181,"props":28380,"children":28382},{"className":28381},[285],[28383,28387,28392,28396,28401],{"type":150,"tag":181,"props":28384,"children":28386},{"className":28385,"style":5680},[290],[],{"type":150,"tag":181,"props":28388,"children":28390},{"className":28389,"style":27072},[296,301],[28391],{"type":155,"value":23188},{"type":150,"tag":181,"props":28393,"children":28395},{"className":28394,"style":658},[377],[],{"type":150,"tag":181,"props":28397,"children":28399},{"className":28398},[663],[28400],{"type":155,"value":865},{"type":150,"tag":181,"props":28402,"children":28404},{"className":28403,"style":658},[377],[],{"type":150,"tag":181,"props":28406,"children":28408},{"className":28407},[285],[28409,28413],{"type":150,"tag":181,"props":28410,"children":28412},{"className":28411,"style":5012},[290],[],{"type":150,"tag":181,"props":28414,"children":28416},{"className":28415},[296],[28417],{"type":155,"value":20641},{"type":150,"tag":22185,"props":28419,"children":28420},{},[28421],{"type":150,"tag":181,"props":28422,"children":28424},{"className":28423},[184],[28425,28455],{"type":150,"tag":181,"props":28426,"children":28428},{"className":28427},[189],[28429],{"type":150,"tag":192,"props":28430,"children":28431},{"xmlns":194},[28432],{"type":150,"tag":197,"props":28433,"children":28434},{},[28435,28450],{"type":150,"tag":201,"props":28436,"children":28437},{},[28438,28442,28446],{"type":150,"tag":209,"props":28439,"children":28440},{},[28441],{"type":155,"value":23188},{"type":150,"tag":221,"props":28443,"children":28444},{},[28445],{"type":155,"value":865},{"type":150,"tag":215,"props":28447,"children":28448},{},[28449],{"type":155,"value":20786},{"type":150,"tag":271,"props":28451,"children":28452},{"encoding":273},[28453],{"type":155,"value":28454},"B=512",{"type":150,"tag":181,"props":28456,"children":28458},{"className":28457,"ariaHidden":223},[280],[28459,28485],{"type":150,"tag":181,"props":28460,"children":28462},{"className":28461},[285],[28463,28467,28472,28476,28481],{"type":150,"tag":181,"props":28464,"children":28466},{"className":28465,"style":5680},[290],[],{"type":150,"tag":181,"props":28468,"children":28470},{"className":28469,"style":27072},[296,301],[28471],{"type":155,"value":23188},{"type":150,"tag":181,"props":28473,"children":28475},{"className":28474,"style":658},[377],[],{"type":150,"tag":181,"props":28477,"children":28479},{"className":28478},[663],[28480],{"type":155,"value":865},{"type":150,"tag":181,"props":28482,"children":28484},{"className":28483,"style":658},[377],[],{"type":150,"tag":181,"props":28486,"children":28488},{"className":28487},[285],[28489,28493],{"type":150,"tag":181,"props":28490,"children":28492},{"className":28491,"style":5012},[290],[],{"type":150,"tag":181,"props":28494,"children":28496},{"className":28495},[296],[28497],{"type":155,"value":20786},{"type":150,"tag":22185,"props":28499,"children":28500},{},[28501],{"type":150,"tag":181,"props":28502,"children":28504},{"className":28503},[184],[28505,28535],{"type":150,"tag":181,"props":28506,"children":28508},{"className":28507},[189],[28509],{"type":150,"tag":192,"props":28510,"children":28511},{"xmlns":194},[28512],{"type":150,"tag":197,"props":28513,"children":28514},{},[28515,28530],{"type":150,"tag":201,"props":28516,"children":28517},{},[28518,28522,28526],{"type":150,"tag":209,"props":28519,"children":28520},{},[28521],{"type":155,"value":23188},{"type":150,"tag":221,"props":28523,"children":28524},{},[28525],{"type":155,"value":12767},{"type":150,"tag":209,"props":28527,"children":28528},{"mathvariant":1618},[28529],{"type":155,"value":26184},{"type":150,"tag":271,"props":28531,"children":28532},{"encoding":273},[28533],{"type":155,"value":28534},"B \\to \\infty",{"type":150,"tag":181,"props":28536,"children":28538},{"className":28537,"ariaHidden":223},[280],[28539,28565],{"type":150,"tag":181,"props":28540,"children":28542},{"className":28541},[285],[28543,28547,28552,28556,28561],{"type":150,"tag":181,"props":28544,"children":28546},{"className":28545,"style":5680},[290],[],{"type":150,"tag":181,"props":28548,"children":28550},{"className":28549,"style":27072},[296,301],[28551],{"type":155,"value":23188},{"type":150,"tag":181,"props":28553,"children":28555},{"className":28554,"style":658},[377],[],{"type":150,"tag":181,"props":28557,"children":28559},{"className":28558},[663],[28560],{"type":155,"value":12767},{"type":150,"tag":181,"props":28562,"children":28564},{"className":28563,"style":658},[377],[],{"type":150,"tag":181,"props":28566,"children":28568},{"className":28567},[285],[28569,28573],{"type":150,"tag":181,"props":28570,"children":28572},{"className":28571,"style":914},[290],[],{"type":150,"tag":181,"props":28574,"children":28576},{"className":28575},[296],[28577],{"type":155,"value":26184},{"type":150,"tag":22199,"props":28579,"children":28580},{},[28581,28618,28649,28680],{"type":150,"tag":22181,"props":28582,"children":28583},{},[28584,28588,28592,28597,28602,28610],{"type":150,"tag":22206,"props":28585,"children":28586},{},[28587],{"type":155,"value":26915},{"type":150,"tag":22206,"props":28589,"children":28590},{},[28591],{"type":155,"value":26920},{"type":150,"tag":22206,"props":28593,"children":28594},{},[28595],{"type":155,"value":28596},"20.0",{"type":150,"tag":22206,"props":28598,"children":28599},{},[28600],{"type":155,"value":28601},"118",{"type":150,"tag":22206,"props":28603,"children":28604},{},[28605],{"type":150,"tag":158,"props":28606,"children":28607},{},[28608],{"type":155,"value":28609},"303",{"type":150,"tag":22206,"props":28611,"children":28612},{},[28613],{"type":150,"tag":158,"props":28614,"children":28615},{},[28616],{"type":155,"value":28617},"390",{"type":150,"tag":22181,"props":28619,"children":28620},{},[28621,28625,28629,28634,28639,28644],{"type":150,"tag":22206,"props":28622,"children":28623},{},[28624],{"type":155,"value":26943},{"type":150,"tag":22206,"props":28626,"children":28627},{},[28628],{"type":155,"value":26948},{"type":150,"tag":22206,"props":28630,"children":28631},{},[28632],{"type":155,"value":28633},"78.5",{"type":150,"tag":22206,"props":28635,"children":28636},{},[28637],{"type":155,"value":28638},"202",{"type":150,"tag":22206,"props":28640,"children":28641},{},[28642],{"type":155,"value":28643},"251",{"type":150,"tag":22206,"props":28645,"children":28646},{},[28647],{"type":155,"value":28648},"260",{"type":150,"tag":22181,"props":28650,"children":28651},{},[28652,28656,28660,28665,28670,28675],{"type":150,"tag":22206,"props":28653,"children":28654},{},[28655],{"type":155,"value":26970},{"type":150,"tag":22206,"props":28657,"children":28658},{},[28659],{"type":155,"value":26975},{"type":150,"tag":22206,"props":28661,"children":28662},{},[28663],{"type":155,"value":28664},"156",{"type":150,"tag":22206,"props":28666,"children":28667},{},[28668],{"type":155,"value":28669},"230",{"type":150,"tag":22206,"props":28671,"children":28672},{},[28673],{"type":155,"value":28674},"244",{"type":150,"tag":22206,"props":28676,"children":28677},{},[28678],{"type":155,"value":28679},"246",{"type":150,"tag":22181,"props":28681,"children":28682},{},[28683,28687,28691,28696,28701,28705],{"type":150,"tag":22206,"props":28684,"children":28685},{},[28686],{"type":155,"value":26997},{"type":150,"tag":22206,"props":28688,"children":28689},{},[28690],{"type":155,"value":27002},{"type":150,"tag":22206,"props":28692,"children":28693},{},[28694],{"type":155,"value":28695},"226",{"type":150,"tag":22206,"props":28697,"children":28698},{},[28699],{"type":155,"value":28700},"240",{"type":150,"tag":22206,"props":28702,"children":28703},{},[28704],{"type":155,"value":26253},{"type":150,"tag":22206,"props":28706,"children":28707},{},[28708],{"type":155,"value":26253},{"type":150,"tag":151,"props":28710,"children":28711},{},[28712],{"type":155,"value":28713},"（加粗的是越过 295 的格子。）",{"type":150,"tag":151,"props":28715,"children":28716},{},[28717],{"type":155,"value":28718},"于是结论要分区间说：",{"type":150,"tag":23633,"props":28720,"children":28721},{},[28722,28732],{"type":150,"tag":23637,"props":28723,"children":28724},{},[28725,28730],{"type":150,"tag":158,"props":28726,"children":28727},{},[28728],{"type":155,"value":28729},"短 context（4k 这一档）",{"type":155,"value":28731},"：batch 开得够大，decode 会翻到 compute bound。这时候那 4.25 倍的计算量是真的要还的。",{"type":150,"tag":23637,"props":28733,"children":28734},{},[28735,28740],{"type":150,"tag":158,"props":28736,"children":28737},{},[28738],{"type":155,"value":28739},"32k 及以上",{"type":155,"value":28741},"：即使 batch 开到无穷，算术强度也够不到 295。decode 始终 memory bound，那 4.25 倍照样白加。",{"type":150,"tag":151,"props":28743,"children":28744},{},[28745,28747,28789,28791,28919,28921,28972,28974,29204],{"type":155,"value":28746},"原因在渐近值里写得清楚。",{"type":150,"tag":181,"props":28748,"children":28750},{"className":28749},[184],[28751,28772],{"type":150,"tag":181,"props":28752,"children":28754},{"className":28753},[189],[28755],{"type":150,"tag":192,"props":28756,"children":28757},{"xmlns":194},[28758],{"type":150,"tag":197,"props":28759,"children":28760},{},[28761,28768],{"type":150,"tag":201,"props":28762,"children":28763},{},[28764],{"type":150,"tag":209,"props":28765,"children":28766},{},[28767],{"type":155,"value":5417},{"type":150,"tag":271,"props":28769,"children":28770},{"encoding":273},[28771],{"type":155,"value":5417},{"type":150,"tag":181,"props":28773,"children":28775},{"className":28774,"ariaHidden":223},[280],[28776],{"type":150,"tag":181,"props":28777,"children":28779},{"className":28778},[285],[28780,28784],{"type":150,"tag":181,"props":28781,"children":28783},{"className":28782,"style":5680},[290],[],{"type":150,"tag":181,"props":28785,"children":28787},{"className":28786,"style":5606},[296,301],[28788],{"type":155,"value":5417},{"type":155,"value":28790}," 一大，",{"type":150,"tag":181,"props":28792,"children":28794},{"className":28793},[184],[28795,28832],{"type":150,"tag":181,"props":28796,"children":28798},{"className":28797},[189],[28799],{"type":150,"tag":192,"props":28800,"children":28801},{"xmlns":194},[28802],{"type":150,"tag":197,"props":28803,"children":28804},{},[28805,28827],{"type":150,"tag":201,"props":28806,"children":28807},{},[28808,28819,28823],{"type":150,"tag":205,"props":28809,"children":28810},{},[28811,28815],{"type":150,"tag":209,"props":28812,"children":28813},{},[28814],{"type":155,"value":9859},{"type":150,"tag":249,"props":28816,"children":28817},{},[28818],{"type":155,"value":24228},{"type":150,"tag":209,"props":28820,"children":28821},{},[28822],{"type":155,"value":5385},{"type":150,"tag":209,"props":28824,"children":28825},{},[28826],{"type":155,"value":5417},{"type":150,"tag":271,"props":28828,"children":28829},{"encoding":273},[28830],{"type":155,"value":28831},"c_{\\text{flop}} L N",{"type":150,"tag":181,"props":28833,"children":28835},{"className":28834,"ariaHidden":223},[280],[28836],{"type":150,"tag":181,"props":28837,"children":28839},{"className":28838},[285],[28840,28844,28909,28914],{"type":150,"tag":181,"props":28841,"children":28843},{"className":28842,"style":6455},[290],[],{"type":150,"tag":181,"props":28845,"children":28847},{"className":28846},[296],[28848,28853],{"type":150,"tag":181,"props":28849,"children":28851},{"className":28850},[296,301],[28852],{"type":155,"value":9859},{"type":150,"tag":181,"props":28854,"children":28856},{"className":28855},[307],[28857],{"type":150,"tag":181,"props":28858,"children":28860},{"className":28859},[312,313],[28861,28898],{"type":150,"tag":181,"props":28862,"children":28864},{"className":28863},[318],[28865,28893],{"type":150,"tag":181,"props":28866,"children":28868},{"className":28867,"style":493},[323],[28869],{"type":150,"tag":181,"props":28870,"children":28871},{"style":328},[28872,28876],{"type":150,"tag":181,"props":28873,"children":28875},{"className":28874,"style":334},[333],[],{"type":150,"tag":181,"props":28877,"children":28879},{"className":28878},[339,340,341,342],[28880],{"type":150,"tag":181,"props":28881,"children":28883},{"className":28882},[296,342],[28884],{"type":150,"tag":181,"props":28885,"children":28887},{"className":28886},[296,155,342],[28888],{"type":150,"tag":181,"props":28889,"children":28891},{"className":28890},[296,342],[28892],{"type":155,"value":24228},{"type":150,"tag":181,"props":28894,"children":28896},{"className":28895},[352],[28897],{"type":155,"value":355},{"type":150,"tag":181,"props":28899,"children":28901},{"className":28900},[318],[28902],{"type":150,"tag":181,"props":28903,"children":28905},{"className":28904,"style":6518},[323],[28906],{"type":150,"tag":181,"props":28907,"children":28908},{},[],{"type":150,"tag":181,"props":28910,"children":28912},{"className":28911},[296,301],[28913],{"type":155,"value":5385},{"type":150,"tag":181,"props":28915,"children":28917},{"className":28916,"style":5606},[296,301],[28918],{"type":155,"value":5417},{"type":155,"value":28920}," 就压过 ",{"type":150,"tag":181,"props":28922,"children":28924},{"className":28923},[184],[28925,28950],{"type":150,"tag":181,"props":28926,"children":28928},{"className":28927},[189],[28929],{"type":150,"tag":192,"props":28930,"children":28931},{"xmlns":194},[28932],{"type":150,"tag":197,"props":28933,"children":28934},{},[28935,28946],{"type":150,"tag":201,"props":28936,"children":28937},{},[28938,28942],{"type":150,"tag":215,"props":28939,"children":28940},{},[28941],{"type":155,"value":238},{"type":150,"tag":209,"props":28943,"children":28944},{},[28945],{"type":155,"value":25017},{"type":150,"tag":271,"props":28947,"children":28948},{"encoding":273},[28949],{"type":155,"value":25502},{"type":150,"tag":181,"props":28951,"children":28953},{"className":28952,"ariaHidden":223},[280],[28954],{"type":150,"tag":181,"props":28955,"children":28957},{"className":28956},[285],[28958,28962,28967],{"type":150,"tag":181,"props":28959,"children":28961},{"className":28960,"style":5680},[290],[],{"type":150,"tag":181,"props":28963,"children":28965},{"className":28964},[296],[28966],{"type":155,"value":238},{"type":150,"tag":181,"props":28968,"children":28970},{"className":28969,"style":3691},[296,301],[28971],{"type":155,"value":25017},{"type":155,"value":28973},"，权重项被稀释掉，剩下的比值就是 ",{"type":150,"tag":181,"props":28975,"children":28977},{"className":28976},[184],[28978,29030],{"type":150,"tag":181,"props":28979,"children":28981},{"className":28980},[189],[28982],{"type":150,"tag":192,"props":28983,"children":28984},{"xmlns":194},[28985],{"type":150,"tag":197,"props":28986,"children":28987},{},[28988,29025],{"type":150,"tag":201,"props":28989,"children":28990},{},[28991,29002,29006,29017,29021],{"type":150,"tag":205,"props":28992,"children":28993},{},[28994,28998],{"type":150,"tag":209,"props":28995,"children":28996},{},[28997],{"type":155,"value":9859},{"type":150,"tag":249,"props":28999,"children":29000},{},[29001],{"type":155,"value":24228},{"type":150,"tag":209,"props":29003,"children":29004},{"mathvariant":1618},[29005],{"type":155,"value":4740},{"type":150,"tag":205,"props":29007,"children":29008},{},[29009,29013],{"type":150,"tag":209,"props":29010,"children":29011},{},[29012],{"type":155,"value":9859},{"type":150,"tag":249,"props":29014,"children":29015},{},[29016],{"type":155,"value":23796},{"type":150,"tag":221,"props":29018,"children":29019},{},[29020],{"type":155,"value":865},{"type":150,"tag":215,"props":29022,"children":29023},{},[29024],{"type":155,"value":26253},{"type":150,"tag":271,"props":29026,"children":29027},{"encoding":273},[29028],{"type":155,"value":29029},"c_{\\text{flop}}\u002Fc_{\\text{kv}} = 242",{"type":150,"tag":181,"props":29031,"children":29033},{"className":29032,"ariaHidden":223},[280],[29034,29191],{"type":150,"tag":181,"props":29035,"children":29037},{"className":29036},[285],[29038,29043,29108,29113,29178,29182,29187],{"type":150,"tag":181,"props":29039,"children":29042},{"className":29040,"style":29041},[290],"height:1.0361em;vertical-align:-0.2861em;",[],{"type":150,"tag":181,"props":29044,"children":29046},{"className":29045},[296],[29047,29052],{"type":150,"tag":181,"props":29048,"children":29050},{"className":29049},[296,301],[29051],{"type":155,"value":9859},{"type":150,"tag":181,"props":29053,"children":29055},{"className":29054},[307],[29056],{"type":150,"tag":181,"props":29057,"children":29059},{"className":29058},[312,313],[29060,29097],{"type":150,"tag":181,"props":29061,"children":29063},{"className":29062},[318],[29064,29092],{"type":150,"tag":181,"props":29065,"children":29067},{"className":29066,"style":493},[323],[29068],{"type":150,"tag":181,"props":29069,"children":29070},{"style":328},[29071,29075],{"type":150,"tag":181,"props":29072,"children":29074},{"className":29073,"style":334},[333],[],{"type":150,"tag":181,"props":29076,"children":29078},{"className":29077},[339,340,341,342],[29079],{"type":150,"tag":181,"props":29080,"children":29082},{"className":29081},[296,342],[29083],{"type":150,"tag":181,"props":29084,"children":29086},{"className":29085},[296,155,342],[29087],{"type":150,"tag":181,"props":29088,"children":29090},{"className":29089},[296,342],[29091],{"type":155,"value":24228},{"type":150,"tag":181,"props":29093,"children":29095},{"className":29094},[352],[29096],{"type":155,"value":355},{"type":150,"tag":181,"props":29098,"children":29100},{"className":29099},[318],[29101],{"type":150,"tag":181,"props":29102,"children":29104},{"className":29103,"style":6518},[323],[29105],{"type":150,"tag":181,"props":29106,"children":29107},{},[],{"type":150,"tag":181,"props":29109,"children":29111},{"className":29110},[296],[29112],{"type":155,"value":4740},{"type":150,"tag":181,"props":29114,"children":29116},{"className":29115},[296],[29117,29122],{"type":150,"tag":181,"props":29118,"children":29120},{"className":29119},[296,301],[29121],{"type":155,"value":9859},{"type":150,"tag":181,"props":29123,"children":29125},{"className":29124},[307],[29126],{"type":150,"tag":181,"props":29127,"children":29129},{"className":29128},[312,313],[29130,29167],{"type":150,"tag":181,"props":29131,"children":29133},{"className":29132},[318],[29134,29162],{"type":150,"tag":181,"props":29135,"children":29137},{"className":29136,"style":493},[323],[29138],{"type":150,"tag":181,"props":29139,"children":29140},{"style":328},[29141,29145],{"type":150,"tag":181,"props":29142,"children":29144},{"className":29143,"style":334},[333],[],{"type":150,"tag":181,"props":29146,"children":29148},{"className":29147},[339,340,341,342],[29149],{"type":150,"tag":181,"props":29150,"children":29152},{"className":29151},[296,342],[29153],{"type":150,"tag":181,"props":29154,"children":29156},{"className":29155},[296,155,342],[29157],{"type":150,"tag":181,"props":29158,"children":29160},{"className":29159},[296,342],[29161],{"type":155,"value":23796},{"type":150,"tag":181,"props":29163,"children":29165},{"className":29164},[352],[29166],{"type":155,"value":355},{"type":150,"tag":181,"props":29168,"children":29170},{"className":29169},[318],[29171],{"type":150,"tag":181,"props":29172,"children":29174},{"className":29173,"style":363},[323],[29175],{"type":150,"tag":181,"props":29176,"children":29177},{},[],{"type":150,"tag":181,"props":29179,"children":29181},{"className":29180,"style":658},[377],[],{"type":150,"tag":181,"props":29183,"children":29185},{"className":29184},[663],[29186],{"type":155,"value":865},{"type":150,"tag":181,"props":29188,"children":29190},{"className":29189,"style":658},[377],[],{"type":150,"tag":181,"props":29192,"children":29194},{"className":29193},[285],[29195,29199],{"type":150,"tag":181,"props":29196,"children":29198},{"className":29197,"style":5012},[290],[],{"type":150,"tag":181,"props":29200,"children":29202},{"className":29201},[296],[29203],{"type":155,"value":26253},{"type":155,"value":29205},"，跟 batch 无关。",{"type":150,"tag":151,"props":29207,"children":29208},{},[29209,29211],{"type":155,"value":29210},"所以 batch 只在短 context 上有效，而短 context 恰恰是 MLA 收益最小的区间。前面那张加速比表里，4k 只有 1.4×。两件事碰在一起，这个取舍就更清楚了：",{"type":150,"tag":158,"props":29212,"children":29213},{},[29214],{"type":155,"value":29215},"MLA 在自己最有价值的长 context 区间永远是 memory bound，那笔计算量从头到尾都不用还。",{"type":150,"tag":151,"props":29217,"children":29218},{},[29219,29221,29226],{"type":155,"value":29220},"长 context 下 batch 也开不大，还有另一个原因：KV Cache 装不下。MLA 在 128k 时每条序列 9.06 GB，32 条就是 290 GB，早就超出单卡。而同样条件下的 MHA 是 515 GB，",{"type":150,"tag":158,"props":29222,"children":29223},{},[29224],{"type":155,"value":29225},"一条序列都放不下",{"type":155,"value":723},{"type":150,"tag":151,"props":29228,"children":29229},{},[29230],{"type":155,"value":29231},"所以 MHA 在长 context 下跑不起来，谈不上快慢。",{"type":150,"tag":151,"props":29233,"children":29234},{},[29235],{"type":155,"value":29236},"那\"划算\"的刻度在哪里？MLA 省的是字节，而字节由权重和 KV 两部分组成。KV 读取追平权重的位置：",{"type":150,"tag":181,"props":29238,"children":29240},{"className":29239},[1003],[29241],{"type":150,"tag":181,"props":29242,"children":29244},{"className":29243},[184],[29245,29322],{"type":150,"tag":181,"props":29246,"children":29248},{"className":29247},[189],[29249],{"type":150,"tag":192,"props":29250,"children":29251},{"xmlns":194,"display":1015},[29252],{"type":150,"tag":197,"props":29253,"children":29254},{},[29255,29317],{"type":150,"tag":201,"props":29256,"children":29257},{},[29258,29263,29268,29273,29277,29281,29285,29289,29294,29299,29303,29307,29312],{"type":150,"tag":249,"props":29259,"children":29260},{},[29261],{"type":155,"value":29262},"MHA： ",{"type":150,"tag":215,"props":29264,"children":29265},{},[29266],{"type":155,"value":29267},"3.93",{"type":150,"tag":249,"props":29269,"children":29270},{},[29271],{"type":155,"value":29272}," MB",{"type":150,"tag":221,"props":29274,"children":29275},{},[29276],{"type":155,"value":5380},{"type":150,"tag":209,"props":29278,"children":29279},{},[29280],{"type":155,"value":5417},{"type":150,"tag":221,"props":29282,"children":29283},{},[29284],{"type":155,"value":865},{"type":150,"tag":215,"props":29286,"children":29287},{},[29288],{"type":155,"value":24739},{"type":150,"tag":249,"props":29290,"children":29291},{},[29292],{"type":155,"value":29293}," GB",{"type":150,"tag":221,"props":29295,"children":29296},{},[29297],{"type":155,"value":29298},"⇒",{"type":150,"tag":209,"props":29300,"children":29301},{},[29302],{"type":155,"value":5417},{"type":150,"tag":221,"props":29304,"children":29305},{},[29306],{"type":155,"value":5876},{"type":150,"tag":215,"props":29308,"children":29309},{},[29310],{"type":155,"value":29311},"1.1",{"type":150,"tag":249,"props":29313,"children":29314},{},[29315],{"type":155,"value":29316}," 万",{"type":150,"tag":271,"props":29318,"children":29319},{"encoding":273},[29320],{"type":155,"value":29321},"\\text{MHA：}\\ 3.93\\ \\text{MB} \\times N = 42\\ \\text{GB} \\Rightarrow N \\approx 1.1\\ \\text{万}",{"type":150,"tag":181,"props":29323,"children":29325},{"className":29324,"ariaHidden":223},[280],[29326,29386,29412,29453,29479],{"type":150,"tag":181,"props":29327,"children":29329},{"className":29328},[285],[29330,29334,29348,29353,29358,29363,29373,29377,29382],{"type":150,"tag":181,"props":29331,"children":29333},{"className":29332,"style":5470},[290],[],{"type":150,"tag":181,"props":29335,"children":29337},{"className":29336},[296,155],[29338,29343],{"type":150,"tag":181,"props":29339,"children":29341},{"className":29340},[296],[29342],{"type":155,"value":22210},{"type":150,"tag":181,"props":29344,"children":29346},{"className":29345},[296,24835],[29347],{"type":155,"value":11152},{"type":150,"tag":181,"props":29349,"children":29351},{"className":29350},[377],[29352],{"type":155,"value":8576},{"type":150,"tag":181,"props":29354,"children":29356},{"className":29355},[296],[29357],{"type":155,"value":29267},{"type":150,"tag":181,"props":29359,"children":29361},{"className":29360},[377],[29362],{"type":155,"value":8576},{"type":150,"tag":181,"props":29364,"children":29366},{"className":29365},[296,155],[29367],{"type":150,"tag":181,"props":29368,"children":29370},{"className":29369},[296],[29371],{"type":155,"value":29372},"MB",{"type":150,"tag":181,"props":29374,"children":29376},{"className":29375,"style":4993},[377],[],{"type":150,"tag":181,"props":29378,"children":29380},{"className":29379},[4998],[29381],{"type":155,"value":5380},{"type":150,"tag":181,"props":29383,"children":29385},{"className":29384,"style":4993},[377],[],{"type":150,"tag":181,"props":29387,"children":29389},{"className":29388},[285],[29390,29394,29399,29403,29408],{"type":150,"tag":181,"props":29391,"children":29393},{"className":29392,"style":5680},[290],[],{"type":150,"tag":181,"props":29395,"children":29397},{"className":29396,"style":5606},[296,301],[29398],{"type":155,"value":5417},{"type":150,"tag":181,"props":29400,"children":29402},{"className":29401,"style":658},[377],[],{"type":150,"tag":181,"props":29404,"children":29406},{"className":29405},[663],[29407],{"type":155,"value":865},{"type":150,"tag":181,"props":29409,"children":29411},{"className":29410,"style":658},[377],[],{"type":150,"tag":181,"props":29413,"children":29415},{"className":29414},[285],[29416,29420,29425,29430,29440,29444,29449],{"type":150,"tag":181,"props":29417,"children":29419},{"className":29418,"style":5680},[290],[],{"type":150,"tag":181,"props":29421,"children":29423},{"className":29422},[296],[29424],{"type":155,"value":24739},{"type":150,"tag":181,"props":29426,"children":29428},{"className":29427},[377],[29429],{"type":155,"value":8576},{"type":150,"tag":181,"props":29431,"children":29433},{"className":29432},[296,155],[29434],{"type":150,"tag":181,"props":29435,"children":29437},{"className":29436},[296],[29438],{"type":155,"value":29439},"GB",{"type":150,"tag":181,"props":29441,"children":29443},{"className":29442,"style":658},[377],[],{"type":150,"tag":181,"props":29445,"children":29447},{"className":29446},[663],[29448],{"type":155,"value":29298},{"type":150,"tag":181,"props":29450,"children":29452},{"className":29451,"style":658},[377],[],{"type":150,"tag":181,"props":29454,"children":29456},{"className":29455},[285],[29457,29461,29466,29470,29475],{"type":150,"tag":181,"props":29458,"children":29460},{"className":29459,"style":5680},[290],[],{"type":150,"tag":181,"props":29462,"children":29464},{"className":29463,"style":5606},[296,301],[29465],{"type":155,"value":5417},{"type":150,"tag":181,"props":29467,"children":29469},{"className":29468,"style":658},[377],[],{"type":150,"tag":181,"props":29471,"children":29473},{"className":29472},[663],[29474],{"type":155,"value":5876},{"type":150,"tag":181,"props":29476,"children":29478},{"className":29477,"style":658},[377],[],{"type":150,"tag":181,"props":29480,"children":29482},{"className":29481},[285],[29483,29487,29492,29497],{"type":150,"tag":181,"props":29484,"children":29486},{"className":29485,"style":5680},[290],[],{"type":150,"tag":181,"props":29488,"children":29490},{"className":29489},[296],[29491],{"type":155,"value":29311},{"type":150,"tag":181,"props":29493,"children":29495},{"className":29494},[377],[29496],{"type":155,"value":8576},{"type":150,"tag":181,"props":29498,"children":29500},{"className":29499},[296,155],[29501],{"type":150,"tag":181,"props":29502,"children":29504},{"className":29503},[296,24835],[29505],{"type":155,"value":29506},"万",{"type":150,"tag":181,"props":29508,"children":29510},{"className":29509},[1003],[29511],{"type":150,"tag":181,"props":29512,"children":29514},{"className":29513},[184],[29515,29589],{"type":150,"tag":181,"props":29516,"children":29518},{"className":29517},[189],[29519],{"type":150,"tag":192,"props":29520,"children":29521},{"xmlns":194,"display":1015},[29522],{"type":150,"tag":197,"props":29523,"children":29524},{},[29525,29584],{"type":150,"tag":201,"props":29526,"children":29527},{},[29528,29533,29538,29543,29547,29551,29555,29559,29563,29567,29571,29575,29580],{"type":150,"tag":249,"props":29529,"children":29530},{},[29531],{"type":155,"value":29532},"MLA： ",{"type":150,"tag":215,"props":29534,"children":29535},{},[29536],{"type":155,"value":29537},"69.1",{"type":150,"tag":249,"props":29539,"children":29540},{},[29541],{"type":155,"value":29542}," KB",{"type":150,"tag":221,"props":29544,"children":29545},{},[29546],{"type":155,"value":5380},{"type":150,"tag":209,"props":29548,"children":29549},{},[29550],{"type":155,"value":5417},{"type":150,"tag":221,"props":29552,"children":29553},{},[29554],{"type":155,"value":865},{"type":150,"tag":215,"props":29556,"children":29557},{},[29558],{"type":155,"value":24739},{"type":150,"tag":249,"props":29560,"children":29561},{},[29562],{"type":155,"value":29293},{"type":150,"tag":221,"props":29564,"children":29565},{},[29566],{"type":155,"value":29298},{"type":150,"tag":209,"props":29568,"children":29569},{},[29570],{"type":155,"value":5417},{"type":150,"tag":221,"props":29572,"children":29573},{},[29574],{"type":155,"value":5876},{"type":150,"tag":215,"props":29576,"children":29577},{},[29578],{"type":155,"value":29579},"61",{"type":150,"tag":249,"props":29581,"children":29582},{},[29583],{"type":155,"value":29316},{"type":150,"tag":271,"props":29585,"children":29586},{"encoding":273},[29587],{"type":155,"value":29588},"\\text{MLA：}\\ 69.1\\ \\text{KB} \\times N = 42\\ \\text{GB} \\Rightarrow N \\approx 61\\ \\text{万}",{"type":150,"tag":181,"props":29590,"children":29592},{"className":29591,"ariaHidden":223},[280],[29593,29653,29679,29719,29745],{"type":150,"tag":181,"props":29594,"children":29596},{"className":29595},[285],[29597,29601,29615,29620,29625,29630,29640,29644,29649],{"type":150,"tag":181,"props":29598,"children":29600},{"className":29599,"style":5470},[290],[],{"type":150,"tag":181,"props":29602,"children":29604},{"className":29603},[296,155],[29605,29610],{"type":150,"tag":181,"props":29606,"children":29608},{"className":29607},[296],[29609],{"type":155,"value":13},{"type":150,"tag":181,"props":29611,"children":29613},{"className":29612},[296,24835],[29614],{"type":155,"value":11152},{"type":150,"tag":181,"props":29616,"children":29618},{"className":29617},[377],[29619],{"type":155,"value":8576},{"type":150,"tag":181,"props":29621,"children":29623},{"className":29622},[296],[29624],{"type":155,"value":29537},{"type":150,"tag":181,"props":29626,"children":29628},{"className":29627},[377],[29629],{"type":155,"value":8576},{"type":150,"tag":181,"props":29631,"children":29633},{"className":29632},[296,155],[29634],{"type":150,"tag":181,"props":29635,"children":29637},{"className":29636},[296],[29638],{"type":155,"value":29639},"KB",{"type":150,"tag":181,"props":29641,"children":29643},{"className":29642,"style":4993},[377],[],{"type":150,"tag":181,"props":29645,"children":29647},{"className":29646},[4998],[29648],{"type":155,"value":5380},{"type":150,"tag":181,"props":29650,"children":29652},{"className":29651,"style":4993},[377],[],{"type":150,"tag":181,"props":29654,"children":29656},{"className":29655},[285],[29657,29661,29666,29670,29675],{"type":150,"tag":181,"props":29658,"children":29660},{"className":29659,"style":5680},[290],[],{"type":150,"tag":181,"props":29662,"children":29664},{"className":29663,"style":5606},[296,301],[29665],{"type":155,"value":5417},{"type":150,"tag":181,"props":29667,"children":29669},{"className":29668,"style":658},[377],[],{"type":150,"tag":181,"props":29671,"children":29673},{"className":29672},[663],[29674],{"type":155,"value":865},{"type":150,"tag":181,"props":29676,"children":29678},{"className":29677,"style":658},[377],[],{"type":150,"tag":181,"props":29680,"children":29682},{"className":29681},[285],[29683,29687,29692,29697,29706,29710,29715],{"type":150,"tag":181,"props":29684,"children":29686},{"className":29685,"style":5680},[290],[],{"type":150,"tag":181,"props":29688,"children":29690},{"className":29689},[296],[29691],{"type":155,"value":24739},{"type":150,"tag":181,"props":29693,"children":29695},{"className":29694},[377],[29696],{"type":155,"value":8576},{"type":150,"tag":181,"props":29698,"children":29700},{"className":29699},[296,155],[29701],{"type":150,"tag":181,"props":29702,"children":29704},{"className":29703},[296],[29705],{"type":155,"value":29439},{"type":150,"tag":181,"props":29707,"children":29709},{"className":29708,"style":658},[377],[],{"type":150,"tag":181,"props":29711,"children":29713},{"className":29712},[663],[29714],{"type":155,"value":29298},{"type":150,"tag":181,"props":29716,"children":29718},{"className":29717,"style":658},[377],[],{"type":150,"tag":181,"props":29720,"children":29722},{"className":29721},[285],[29723,29727,29732,29736,29741],{"type":150,"tag":181,"props":29724,"children":29726},{"className":29725,"style":5680},[290],[],{"type":150,"tag":181,"props":29728,"children":29730},{"className":29729,"style":5606},[296,301],[29731],{"type":155,"value":5417},{"type":150,"tag":181,"props":29733,"children":29735},{"className":29734,"style":658},[377],[],{"type":150,"tag":181,"props":29737,"children":29739},{"className":29738},[663],[29740],{"type":155,"value":5876},{"type":150,"tag":181,"props":29742,"children":29744},{"className":29743,"style":658},[377],[],{"type":150,"tag":181,"props":29746,"children":29748},{"className":29747},[285],[29749,29753,29758,29763],{"type":150,"tag":181,"props":29750,"children":29752},{"className":29751,"style":5680},[290],[],{"type":150,"tag":181,"props":29754,"children":29756},{"className":29755},[296],[29757],{"type":155,"value":29579},{"type":150,"tag":181,"props":29759,"children":29761},{"className":29760},[377],[29762],{"type":155,"value":8576},{"type":150,"tag":181,"props":29764,"children":29766},{"className":29765},[296,155],[29767],{"type":150,"tag":181,"props":29768,"children":29770},{"className":29769},[296,24835],[29771],{"type":155,"value":29506},{"type":150,"tag":151,"props":29773,"children":29774},{},[29775],{"type":155,"value":29776},"既然两边都是 memory bound，加速比就等于字节数之比：",{"type":150,"tag":22173,"props":29778,"children":29779},{},[29780,29836],{"type":150,"tag":22177,"props":29781,"children":29782},{},[29783],{"type":150,"tag":22181,"props":29784,"children":29785},{},[29786,29831],{"type":150,"tag":22185,"props":29787,"children":29788},{},[29789],{"type":150,"tag":181,"props":29790,"children":29792},{"className":29791},[184],[29793,29814],{"type":150,"tag":181,"props":29794,"children":29796},{"className":29795},[189],[29797],{"type":150,"tag":192,"props":29798,"children":29799},{"xmlns":194},[29800],{"type":150,"tag":197,"props":29801,"children":29802},{},[29803,29810],{"type":150,"tag":201,"props":29804,"children":29805},{},[29806],{"type":150,"tag":209,"props":29807,"children":29808},{},[29809],{"type":155,"value":5417},{"type":150,"tag":271,"props":29811,"children":29812},{"encoding":273},[29813],{"type":155,"value":5417},{"type":150,"tag":181,"props":29815,"children":29817},{"className":29816,"ariaHidden":223},[280],[29818],{"type":150,"tag":181,"props":29819,"children":29821},{"className":29820},[285],[29822,29826],{"type":150,"tag":181,"props":29823,"children":29825},{"className":29824,"style":5680},[290],[],{"type":150,"tag":181,"props":29827,"children":29829},{"className":29828,"style":5606},[296,301],[29830],{"type":155,"value":5417},{"type":150,"tag":22185,"props":29832,"children":29833},{},[29834],{"type":155,"value":29835},"MLA 相对 MHA 的每 token 加速比",{"type":150,"tag":22199,"props":29837,"children":29838},{},[29839,29851,29863,29875],{"type":150,"tag":22181,"props":29840,"children":29841},{},[29842,29846],{"type":150,"tag":22206,"props":29843,"children":29844},{},[29845],{"type":155,"value":26915},{"type":150,"tag":22206,"props":29847,"children":29848},{},[29849],{"type":155,"value":29850},"1.4×",{"type":150,"tag":22181,"props":29852,"children":29853},{},[29854,29858],{"type":150,"tag":22206,"props":29855,"children":29856},{},[29857],{"type":155,"value":26943},{"type":150,"tag":22206,"props":29859,"children":29860},{},[29861],{"type":155,"value":29862},"3.9×",{"type":150,"tag":22181,"props":29864,"children":29865},{},[29866,29870],{"type":150,"tag":22206,"props":29867,"children":29868},{},[29869],{"type":155,"value":26970},{"type":150,"tag":22206,"props":29871,"children":29872},{},[29873],{"type":155,"value":29874},"10.9×",{"type":150,"tag":22181,"props":29876,"children":29877},{},[29878,29882],{"type":150,"tag":22206,"props":29879,"children":29880},{},[29881],{"type":155,"value":26997},{"type":150,"tag":22206,"props":29883,"children":29884},{},[29885],{"type":155,"value":29886},"36.4×",{"type":150,"tag":151,"props":29888,"children":29889},{},[29890,29892],{"type":155,"value":29891},"所以答案是一条具体的曲线：",{"type":150,"tag":158,"props":29893,"children":29894},{},[29895],{"type":155,"value":29896},"4k 时几乎白干（1.4×），32k 开始有意义（3.9×），128k 是 11 倍，1M 是 36 倍。",{"type":150,"tag":151,"props":29898,"children":29899},{},[29900],{"type":155,"value":29901},"这条曲线顺带解释了 MLA 的时间线。它 2024 年出来的时候，主流 context 还在 4k 到 32k 之间，那一段的收益并不惊人；让它值回票价的是后面 128k、1M 的军备竞赛。",{"type":150,"tag":151,"props":29903,"children":29904},{},[29905,29907,30077,30079,30328],{"type":155,"value":29906},"另外还有一件事能说明这个设计有多自觉。一般 LLM 架构满足 ",{"type":150,"tag":181,"props":29908,"children":29910},{"className":29909},[184],[29911,29956],{"type":150,"tag":181,"props":29912,"children":29914},{"className":29913},[189],[29915],{"type":150,"tag":192,"props":29916,"children":29917},{"xmlns":194},[29918],{"type":150,"tag":197,"props":29919,"children":29920},{},[29921,29951],{"type":150,"tag":201,"props":29922,"children":29923},{},[29924,29928,29932,29943,29947],{"type":150,"tag":209,"props":29925,"children":29926},{},[29927],{"type":155,"value":745},{"type":150,"tag":221,"props":29929,"children":29930},{},[29931],{"type":155,"value":5380},{"type":150,"tag":205,"props":29933,"children":29934},{},[29935,29939],{"type":150,"tag":209,"props":29936,"children":29937},{},[29938],{"type":155,"value":578},{"type":150,"tag":209,"props":29940,"children":29941},{},[29942],{"type":155,"value":1698},{"type":150,"tag":221,"props":29944,"children":29945},{},[29946],{"type":155,"value":865},{"type":150,"tag":209,"props":29948,"children":29949},{},[29950],{"type":155,"value":578},{"type":150,"tag":271,"props":29952,"children":29953},{"encoding":273},[29954],{"type":155,"value":29955},"h \\times d_k = d",{"type":150,"tag":181,"props":29957,"children":29959},{"className":29958,"ariaHidden":223},[280],[29960,29986,30064],{"type":150,"tag":181,"props":29961,"children":29963},{"className":29962},[285],[29964,29968,29973,29977,29982],{"type":150,"tag":181,"props":29965,"children":29967},{"className":29966,"style":5497},[290],[],{"type":150,"tag":181,"props":29969,"children":29971},{"className":29970},[296,301],[29972],{"type":155,"value":745},{"type":150,"tag":181,"props":29974,"children":29976},{"className":29975,"style":4993},[377],[],{"type":150,"tag":181,"props":29978,"children":29980},{"className":29979},[4998],[29981],{"type":155,"value":5380},{"type":150,"tag":181,"props":29983,"children":29985},{"className":29984,"style":4993},[377],[],{"type":150,"tag":181,"props":29987,"children":29989},{"className":29988},[285],[29990,29994,30051,30055,30060],{"type":150,"tag":181,"props":29991,"children":29993},{"className":29992,"style":4761},[290],[],{"type":150,"tag":181,"props":29995,"children":29997},{"className":29996},[296],[29998,30003],{"type":150,"tag":181,"props":29999,"children":30001},{"className":30000},[296,301],[30002],{"type":155,"value":578},{"type":150,"tag":181,"props":30004,"children":30006},{"className":30005},[307],[30007],{"type":150,"tag":181,"props":30008,"children":30010},{"className":30009},[312,313],[30011,30040],{"type":150,"tag":181,"props":30012,"children":30014},{"className":30013},[318],[30015,30035],{"type":150,"tag":181,"props":30016,"children":30018},{"className":30017,"style":493},[323],[30019],{"type":150,"tag":181,"props":30020,"children":30021},{"style":328},[30022,30026],{"type":150,"tag":181,"props":30023,"children":30025},{"className":30024,"style":334},[333],[],{"type":150,"tag":181,"props":30027,"children":30029},{"className":30028},[339,340,341,342],[30030],{"type":150,"tag":181,"props":30031,"children":30033},{"className":30032,"style":2249},[296,301,342],[30034],{"type":155,"value":1698},{"type":150,"tag":181,"props":30036,"children":30038},{"className":30037},[352],[30039],{"type":155,"value":355},{"type":150,"tag":181,"props":30041,"children":30043},{"className":30042},[318],[30044],{"type":150,"tag":181,"props":30045,"children":30047},{"className":30046,"style":363},[323],[30048],{"type":150,"tag":181,"props":30049,"children":30050},{},[],{"type":150,"tag":181,"props":30052,"children":30054},{"className":30053,"style":658},[377],[],{"type":150,"tag":181,"props":30056,"children":30058},{"className":30057},[663],[30059],{"type":155,"value":865},{"type":150,"tag":181,"props":30061,"children":30063},{"className":30062,"style":658},[377],[],{"type":150,"tag":181,"props":30065,"children":30067},{"className":30066},[285],[30068,30072],{"type":150,"tag":181,"props":30069,"children":30071},{"className":30070,"style":761},[290],[],{"type":150,"tag":181,"props":30073,"children":30075},{"className":30074},[296,301],[30076],{"type":155,"value":578},{"type":155,"value":30078},"，但 DeepSeek-V2 是 ",{"type":150,"tag":181,"props":30080,"children":30082},{"className":30081},[184],[30083,30153],{"type":150,"tag":181,"props":30084,"children":30086},{"className":30085},[189],[30087],{"type":150,"tag":192,"props":30088,"children":30089},{"xmlns":194},[30090],{"type":150,"tag":197,"props":30091,"children":30092},{},[30093,30148],{"type":150,"tag":201,"props":30094,"children":30095},{},[30096,30107,30111,30115,30119,30123,30127,30132,30136,30140,30144],{"type":150,"tag":205,"props":30097,"children":30098},{},[30099,30103],{"type":150,"tag":209,"props":30100,"children":30101},{},[30102],{"type":155,"value":578},{"type":150,"tag":209,"props":30104,"children":30105},{},[30106],{"type":155,"value":1698},{"type":150,"tag":221,"props":30108,"children":30109},{},[30110],{"type":155,"value":865},{"type":150,"tag":215,"props":30112,"children":30113},{},[30114],{"type":155,"value":5871},{"type":150,"tag":221,"props":30116,"children":30117},{"separator":223},[30118],{"type":155,"value":226},{"type":150,"tag":209,"props":30120,"children":30121},{},[30122],{"type":155,"value":578},{"type":150,"tag":221,"props":30124,"children":30125},{},[30126],{"type":155,"value":865},{"type":150,"tag":215,"props":30128,"children":30129},{},[30130],{"type":155,"value":30131},"5120",{"type":150,"tag":221,"props":30133,"children":30134},{"separator":223},[30135],{"type":155,"value":226},{"type":150,"tag":209,"props":30137,"children":30138},{},[30139],{"type":155,"value":745},{"type":150,"tag":221,"props":30141,"children":30142},{},[30143],{"type":155,"value":865},{"type":150,"tag":215,"props":30145,"children":30146},{},[30147],{"type":155,"value":5871},{"type":150,"tag":271,"props":30149,"children":30150},{"encoding":273},[30151],{"type":155,"value":30152},"d_k = 128, d = 5120, h = 128",{"type":150,"tag":181,"props":30154,"children":30156},{"className":30155,"ariaHidden":223},[280],[30157,30235,30275,30315],{"type":150,"tag":181,"props":30158,"children":30160},{"className":30159},[285],[30161,30165,30222,30226,30231],{"type":150,"tag":181,"props":30162,"children":30164},{"className":30163,"style":4761},[290],[],{"type":150,"tag":181,"props":30166,"children":30168},{"className":30167},[296],[30169,30174],{"type":150,"tag":181,"props":30170,"children":30172},{"className":30171},[296,301],[30173],{"type":155,"value":578},{"type":150,"tag":181,"props":30175,"children":30177},{"className":30176},[307],[30178],{"type":150,"tag":181,"props":30179,"children":30181},{"className":30180},[312,313],[30182,30211],{"type":150,"tag":181,"props":30183,"children":30185},{"className":30184},[318],[30186,30206],{"type":150,"tag":181,"props":30187,"children":30189},{"className":30188,"style":493},[323],[30190],{"type":150,"tag":181,"props":30191,"children":30192},{"style":328},[30193,30197],{"type":150,"tag":181,"props":30194,"children":30196},{"className":30195,"style":334},[333],[],{"type":150,"tag":181,"props":30198,"children":30200},{"className":30199},[339,340,341,342],[30201],{"type":150,"tag":181,"props":30202,"children":30204},{"className":30203,"style":2249},[296,301,342],[30205],{"type":155,"value":1698},{"type":150,"tag":181,"props":30207,"children":30209},{"className":30208},[352],[30210],{"type":155,"value":355},{"type":150,"tag":181,"props":30212,"children":30214},{"className":30213},[318],[30215],{"type":150,"tag":181,"props":30216,"children":30218},{"className":30217,"style":363},[323],[30219],{"type":150,"tag":181,"props":30220,"children":30221},{},[],{"type":150,"tag":181,"props":30223,"children":30225},{"className":30224,"style":658},[377],[],{"type":150,"tag":181,"props":30227,"children":30229},{"className":30228},[663],[30230],{"type":155,"value":865},{"type":150,"tag":181,"props":30232,"children":30234},{"className":30233,"style":658},[377],[],{"type":150,"tag":181,"props":30236,"children":30238},{"className":30237},[285],[30239,30243,30248,30253,30257,30262,30266,30271],{"type":150,"tag":181,"props":30240,"children":30242},{"className":30241,"style":941},[290],[],{"type":150,"tag":181,"props":30244,"children":30246},{"className":30245},[296],[30247],{"type":155,"value":5871},{"type":150,"tag":181,"props":30249,"children":30251},{"className":30250},[371],[30252],{"type":155,"value":226},{"type":150,"tag":181,"props":30254,"children":30256},{"className":30255,"style":378},[377],[],{"type":150,"tag":181,"props":30258,"children":30260},{"className":30259},[296,301],[30261],{"type":155,"value":578},{"type":150,"tag":181,"props":30263,"children":30265},{"className":30264,"style":658},[377],[],{"type":150,"tag":181,"props":30267,"children":30269},{"className":30268},[663],[30270],{"type":155,"value":865},{"type":150,"tag":181,"props":30272,"children":30274},{"className":30273,"style":658},[377],[],{"type":150,"tag":181,"props":30276,"children":30278},{"className":30277},[285],[30279,30283,30288,30293,30297,30302,30306,30311],{"type":150,"tag":181,"props":30280,"children":30282},{"className":30281,"style":941},[290],[],{"type":150,"tag":181,"props":30284,"children":30286},{"className":30285},[296],[30287],{"type":155,"value":30131},{"type":150,"tag":181,"props":30289,"children":30291},{"className":30290},[371],[30292],{"type":155,"value":226},{"type":150,"tag":181,"props":30294,"children":30296},{"className":30295,"style":378},[377],[],{"type":150,"tag":181,"props":30298,"children":30300},{"className":30299},[296,301],[30301],{"type":155,"value":745},{"type":150,"tag":181,"props":30303,"children":30305},{"className":30304,"style":658},[377],[],{"type":150,"tag":181,"props":30307,"children":30309},{"className":30308},[663],[30310],{"type":155,"value":865},{"type":150,"tag":181,"props":30312,"children":30314},{"className":30313,"style":658},[377],[],{"type":150,"tag":181,"props":30316,"children":30318},{"className":30317},[285],[30319,30323],{"type":150,"tag":181,"props":30320,"children":30322},{"className":30321,"style":5012},[290],[],{"type":150,"tag":181,"props":30324,"children":30326},{"className":30325},[296],[30327],{"type":155,"value":5871},{"type":155,"value":30329},"，head 数是一般设置的 3 倍。",{"type":150,"tag":151,"props":30331,"children":30332},{},[30333,30335,30377,30379,30421,30423,30428],{"type":155,"value":30334},"因为 MLA 的 KV Cache 大小跟 ",{"type":150,"tag":181,"props":30336,"children":30338},{"className":30337},[184],[30339,30360],{"type":150,"tag":181,"props":30340,"children":30342},{"className":30341},[189],[30343],{"type":150,"tag":192,"props":30344,"children":30345},{"xmlns":194},[30346],{"type":150,"tag":197,"props":30347,"children":30348},{},[30349,30356],{"type":150,"tag":201,"props":30350,"children":30351},{},[30352],{"type":150,"tag":209,"props":30353,"children":30354},{},[30355],{"type":155,"value":745},{"type":150,"tag":271,"props":30357,"children":30358},{"encoding":273},[30359],{"type":155,"value":745},{"type":150,"tag":181,"props":30361,"children":30363},{"className":30362,"ariaHidden":223},[280],[30364],{"type":150,"tag":181,"props":30365,"children":30367},{"className":30366},[285],[30368,30372],{"type":150,"tag":181,"props":30369,"children":30371},{"className":30370,"style":761},[290],[],{"type":150,"tag":181,"props":30373,"children":30375},{"className":30374},[296,301],[30376],{"type":155,"value":745},{"type":155,"value":30378}," 无关。增大 ",{"type":150,"tag":181,"props":30380,"children":30382},{"className":30381},[184],[30383,30404],{"type":150,"tag":181,"props":30384,"children":30386},{"className":30385},[189],[30387],{"type":150,"tag":192,"props":30388,"children":30389},{"xmlns":194},[30390],{"type":150,"tag":197,"props":30391,"children":30392},{},[30393,30400],{"type":150,"tag":201,"props":30394,"children":30395},{},[30396],{"type":150,"tag":209,"props":30397,"children":30398},{},[30399],{"type":155,"value":745},{"type":150,"tag":271,"props":30401,"children":30402},{"encoding":273},[30403],{"type":155,"value":745},{"type":150,"tag":181,"props":30405,"children":30407},{"className":30406,"ariaHidden":223},[280],[30408],{"type":150,"tag":181,"props":30409,"children":30411},{"className":30410},[285],[30412,30416],{"type":150,"tag":181,"props":30413,"children":30415},{"className":30414,"style":761},[290],[],{"type":150,"tag":181,"props":30417,"children":30419},{"className":30418},[296,301],[30420],{"type":155,"value":745},{"type":155,"value":30422}," 只增加计算量和提升模型能力，不增加 KV Cache，所以不会带来速度瓶颈。MLA 完成的思想升级：",{"type":150,"tag":158,"props":30424,"children":30425},{},[30426],{"type":155,"value":30427},"从离散的 head sharing，到连续的 representation compression",{"type":155,"value":723},{"type":150,"tag":169,"props":30430,"children":30432},{"id":30431},"csa-与-hca",[30433],{"type":155,"value":30434},"CSA 与 HCA",{"type":150,"tag":30436,"props":30437,"children":30439},"h3",{"id":30438},"压-sequence-轴",[30440],{"type":155,"value":30441},"压 Sequence 轴",{"type":150,"tag":151,"props":30443,"children":30444},{},[30445,30447,30746,30748,30790,30792,30869],{"type":155,"value":30446},"MLA 压小的是每个 token 存的东西。但 ",{"type":150,"tag":181,"props":30448,"children":30450},{"className":30449},[184],[30451,30522],{"type":150,"tag":181,"props":30452,"children":30454},{"className":30453},[189],[30455],{"type":150,"tag":192,"props":30456,"children":30457},{"xmlns":194},[30458],{"type":150,"tag":197,"props":30459,"children":30460},{},[30461,30517],{"type":150,"tag":201,"props":30462,"children":30463},{},[30464,30475,30479,30490,30494,30498,30502,30506],{"type":150,"tag":205,"props":30465,"children":30466},{},[30467,30471],{"type":150,"tag":209,"props":30468,"children":30469},{},[30470],{"type":155,"value":9859},{"type":150,"tag":215,"props":30472,"children":30473},{},[30474],{"type":155,"value":219},{"type":150,"tag":221,"props":30476,"children":30477},{"separator":223},[30478],{"type":155,"value":226},{"type":150,"tag":205,"props":30480,"children":30481},{},[30482,30486],{"type":150,"tag":209,"props":30483,"children":30484},{},[30485],{"type":155,"value":9859},{"type":150,"tag":215,"props":30487,"children":30488},{},[30489],{"type":155,"value":238},{"type":150,"tag":221,"props":30491,"children":30492},{"separator":223},[30493],{"type":155,"value":226},{"type":150,"tag":221,"props":30495,"children":30496},{},[30497],{"type":155,"value":247},{"type":150,"tag":249,"props":30499,"children":30500},{},[30501],{"type":155,"value":253},{"type":150,"tag":221,"props":30503,"children":30504},{"separator":223},[30505],{"type":155,"value":226},{"type":150,"tag":205,"props":30507,"children":30508},{},[30509,30513],{"type":150,"tag":209,"props":30510,"children":30511},{},[30512],{"type":155,"value":9859},{"type":150,"tag":209,"props":30514,"children":30515},{},[30516],{"type":155,"value":5417},{"type":150,"tag":271,"props":30518,"children":30519},{"encoding":273},[30520],{"type":155,"value":30521},"c_1, c_2, \\cdots, c_N",{"type":150,"tag":181,"props":30523,"children":30525},{"className":30524,"ariaHidden":223},[280],[30526],{"type":150,"tag":181,"props":30527,"children":30529},{"className":30528},[285],[30530,30534,30591,30596,30600,30657,30662,30666,30671,30675,30679,30684,30688],{"type":150,"tag":181,"props":30531,"children":30533},{"className":30532,"style":291},[290],[],{"type":150,"tag":181,"props":30535,"children":30537},{"className":30536},[296],[30538,30543],{"type":150,"tag":181,"props":30539,"children":30541},{"className":30540},[296,301],[30542],{"type":155,"value":9859},{"type":150,"tag":181,"props":30544,"children":30546},{"className":30545},[307],[30547],{"type":150,"tag":181,"props":30548,"children":30550},{"className":30549},[312,313],[30551,30580],{"type":150,"tag":181,"props":30552,"children":30554},{"className":30553},[318],[30555,30575],{"type":150,"tag":181,"props":30556,"children":30558},{"className":30557,"style":324},[323],[30559],{"type":150,"tag":181,"props":30560,"children":30561},{"style":328},[30562,30566],{"type":150,"tag":181,"props":30563,"children":30565},{"className":30564,"style":334},[333],[],{"type":150,"tag":181,"props":30567,"children":30569},{"className":30568},[339,340,341,342],[30570],{"type":150,"tag":181,"props":30571,"children":30573},{"className":30572},[296,342],[30574],{"type":155,"value":219},{"type":150,"tag":181,"props":30576,"children":30578},{"className":30577},[352],[30579],{"type":155,"value":355},{"type":150,"tag":181,"props":30581,"children":30583},{"className":30582},[318],[30584],{"type":150,"tag":181,"props":30585,"children":30587},{"className":30586,"style":363},[323],[30588],{"type":150,"tag":181,"props":30589,"children":30590},{},[],{"type":150,"tag":181,"props":30592,"children":30594},{"className":30593},[371],[30595],{"type":155,"value":226},{"type":150,"tag":181,"props":30597,"children":30599},{"className":30598,"style":378},[377],[],{"type":150,"tag":181,"props":30601,"children":30603},{"className":30602},[296],[30604,30609],{"type":150,"tag":181,"props":30605,"children":30607},{"className":30606},[296,301],[30608],{"type":155,"value":9859},{"type":150,"tag":181,"props":30610,"children":30612},{"className":30611},[307],[30613],{"type":150,"tag":181,"props":30614,"children":30616},{"className":30615},[312,313],[30617,30646],{"type":150,"tag":181,"props":30618,"children":30620},{"className":30619},[318],[30621,30641],{"type":150,"tag":181,"props":30622,"children":30624},{"className":30623,"style":324},[323],[30625],{"type":150,"tag":181,"props":30626,"children":30627},{"style":328},[30628,30632],{"type":150,"tag":181,"props":30629,"children":30631},{"className":30630,"style":334},[333],[],{"type":150,"tag":181,"props":30633,"children":30635},{"className":30634},[339,340,341,342],[30636],{"type":150,"tag":181,"props":30637,"children":30639},{"className":30638},[296,342],[30640],{"type":155,"value":238},{"type":150,"tag":181,"props":30642,"children":30644},{"className":30643},[352],[30645],{"type":155,"value":355},{"type":150,"tag":181,"props":30647,"children":30649},{"className":30648},[318],[30650],{"type":150,"tag":181,"props":30651,"children":30653},{"className":30652,"style":363},[323],[30654],{"type":150,"tag":181,"props":30655,"children":30656},{},[],{"type":150,"tag":181,"props":30658,"children":30660},{"className":30659},[371],[30661],{"type":155,"value":226},{"type":150,"tag":181,"props":30663,"children":30665},{"className":30664,"style":378},[377],[],{"type":150,"tag":181,"props":30667,"children":30669},{"className":30668},[449],[30670],{"type":155,"value":247},{"type":150,"tag":181,"props":30672,"children":30674},{"className":30673,"style":378},[377],[],{"type":150,"tag":181,"props":30676,"children":30678},{"className":30677,"style":378},[377],[],{"type":150,"tag":181,"props":30680,"children":30682},{"className":30681},[371],[30683],{"type":155,"value":226},{"type":150,"tag":181,"props":30685,"children":30687},{"className":30686,"style":378},[377],[],{"type":150,"tag":181,"props":30689,"children":30691},{"className":30690},[296],[30692,30697],{"type":150,"tag":181,"props":30693,"children":30695},{"className":30694},[296,301],[30696],{"type":155,"value":9859},{"type":150,"tag":181,"props":30698,"children":30700},{"className":30699},[307],[30701],{"type":150,"tag":181,"props":30702,"children":30704},{"className":30703},[312,313],[30705,30735],{"type":150,"tag":181,"props":30706,"children":30708},{"className":30707},[318],[30709,30730],{"type":150,"tag":181,"props":30710,"children":30713},{"className":30711,"style":30712},[323],"height:0.3283em;",[30714],{"type":150,"tag":181,"props":30715,"children":30716},{"style":328},[30717,30721],{"type":150,"tag":181,"props":30718,"children":30720},{"className":30719,"style":334},[333],[],{"type":150,"tag":181,"props":30722,"children":30724},{"className":30723},[339,340,341,342],[30725],{"type":150,"tag":181,"props":30726,"children":30728},{"className":30727,"style":5606},[296,301,342],[30729],{"type":155,"value":5417},{"type":150,"tag":181,"props":30731,"children":30733},{"className":30732},[352],[30734],{"type":155,"value":355},{"type":150,"tag":181,"props":30736,"children":30738},{"className":30737},[318],[30739],{"type":150,"tag":181,"props":30740,"children":30742},{"className":30741,"style":363},[323],[30743],{"type":150,"tag":181,"props":30744,"children":30745},{},[],{"type":155,"value":30747}," 仍然有 ",{"type":150,"tag":181,"props":30749,"children":30751},{"className":30750},[184],[30752,30773],{"type":150,"tag":181,"props":30753,"children":30755},{"className":30754},[189],[30756],{"type":150,"tag":192,"props":30757,"children":30758},{"xmlns":194},[30759],{"type":150,"tag":197,"props":30760,"children":30761},{},[30762,30769],{"type":150,"tag":201,"props":30763,"children":30764},{},[30765],{"type":150,"tag":209,"props":30766,"children":30767},{},[30768],{"type":155,"value":5417},{"type":150,"tag":271,"props":30770,"children":30771},{"encoding":273},[30772],{"type":155,"value":5417},{"type":150,"tag":181,"props":30774,"children":30776},{"className":30775,"ariaHidden":223},[280],[30777],{"type":150,"tag":181,"props":30778,"children":30780},{"className":30779},[285],[30781,30785],{"type":150,"tag":181,"props":30782,"children":30784},{"className":30783,"style":5680},[290],[],{"type":150,"tag":181,"props":30786,"children":30788},{"className":30787,"style":5606},[296,301],[30789],{"type":155,"value":5417},{"type":155,"value":30791}," 个。如果每个 query 还是要看全部历史，那 ",{"type":150,"tag":181,"props":30793,"children":30795},{"className":30794},[184],[30796,30826],{"type":150,"tag":181,"props":30797,"children":30799},{"className":30798},[189],[30800],{"type":150,"tag":192,"props":30801,"children":30802},{"xmlns":194},[30803],{"type":150,"tag":197,"props":30804,"children":30805},{},[30806,30821],{"type":150,"tag":201,"props":30807,"children":30808},{},[30809,30813,30817],{"type":150,"tag":209,"props":30810,"children":30811},{},[30812],{"type":155,"value":5417},{"type":150,"tag":221,"props":30814,"children":30815},{},[30816],{"type":155,"value":5380},{"type":150,"tag":209,"props":30818,"children":30819},{},[30820],{"type":155,"value":5417},{"type":150,"tag":271,"props":30822,"children":30823},{"encoding":273},[30824],{"type":155,"value":30825},"N \\times N",{"type":150,"tag":181,"props":30827,"children":30829},{"className":30828,"ariaHidden":223},[280],[30830,30856],{"type":150,"tag":181,"props":30831,"children":30833},{"className":30832},[285],[30834,30838,30843,30847,30852],{"type":150,"tag":181,"props":30835,"children":30837},{"className":30836,"style":5470},[290],[],{"type":150,"tag":181,"props":30839,"children":30841},{"className":30840,"style":5606},[296,301],[30842],{"type":155,"value":5417},{"type":150,"tag":181,"props":30844,"children":30846},{"className":30845,"style":4993},[377],[],{"type":150,"tag":181,"props":30848,"children":30850},{"className":30849},[4998],[30851],{"type":155,"value":5380},{"type":150,"tag":181,"props":30853,"children":30855},{"className":30854,"style":4993},[377],[],{"type":150,"tag":181,"props":30857,"children":30859},{"className":30858},[285],[30860,30864],{"type":150,"tag":181,"props":30861,"children":30863},{"className":30862,"style":5680},[290],[],{"type":150,"tag":181,"props":30865,"children":30867},{"className":30866,"style":5606},[296,301],[30868],{"type":155,"value":5417},{"type":155,"value":30870}," 还在。",{"type":150,"tag":151,"props":30872,"children":30873},{},[30874],{"type":150,"tag":158,"props":30875,"children":30876},{},[30877],{"type":155,"value":30878},"MLA 解决的是 KV 的宽度，没解决历史的长度。",{"type":150,"tag":151,"props":30880,"children":30881},{},[30882],{"type":155,"value":30883},"把 KV Cache 想成一个二维矩阵：",{"type":150,"tag":181,"props":30885,"children":30887},{"className":30886},[1003],[30888],{"type":150,"tag":181,"props":30889,"children":30891},{"className":30890},[184],[30892,30924],{"type":150,"tag":181,"props":30893,"children":30895},{"className":30894},[189],[30896],{"type":150,"tag":192,"props":30897,"children":30898},{"xmlns":194,"display":1015},[30899],{"type":150,"tag":197,"props":30900,"children":30901},{},[30902,30919],{"type":150,"tag":201,"props":30903,"children":30904},{},[30905,30910,30914],{"type":150,"tag":249,"props":30906,"children":30907},{},[30908],{"type":155,"value":30909},"Sequence",{"type":150,"tag":221,"props":30911,"children":30912},{},[30913],{"type":155,"value":5380},{"type":150,"tag":249,"props":30915,"children":30916},{},[30917],{"type":155,"value":30918},"Feature",{"type":150,"tag":271,"props":30920,"children":30921},{"encoding":273},[30922],{"type":155,"value":30923},"\\text{Sequence} \\times \\text{Feature}",{"type":150,"tag":181,"props":30925,"children":30927},{"className":30926,"ariaHidden":223},[280],[30928,30958],{"type":150,"tag":181,"props":30929,"children":30931},{"className":30930},[285],[30932,30936,30945,30949,30954],{"type":150,"tag":181,"props":30933,"children":30935},{"className":30934,"style":11536},[290],[],{"type":150,"tag":181,"props":30937,"children":30939},{"className":30938},[296,155],[30940],{"type":150,"tag":181,"props":30941,"children":30943},{"className":30942},[296],[30944],{"type":155,"value":30909},{"type":150,"tag":181,"props":30946,"children":30948},{"className":30947,"style":4993},[377],[],{"type":150,"tag":181,"props":30950,"children":30952},{"className":30951},[4998],[30953],{"type":155,"value":5380},{"type":150,"tag":181,"props":30955,"children":30957},{"className":30956,"style":4993},[377],[],{"type":150,"tag":181,"props":30959,"children":30961},{"className":30960},[285],[30962,30966],{"type":150,"tag":181,"props":30963,"children":30965},{"className":30964,"style":5680},[290],[],{"type":150,"tag":181,"props":30967,"children":30969},{"className":30968},[296,155],[30970],{"type":150,"tag":181,"props":30971,"children":30973},{"className":30972},[296],[30974],{"type":155,"value":30918},{"type":150,"tag":151,"props":30976,"children":30977},{},[30978],{"type":155,"value":30979},"MQA、GQA、MLA 压的都是 Feature 轴。V4 开始压 Sequence 轴。V4 改成了三类 attention：",{"type":150,"tag":23633,"props":30981,"children":30982},{},[30983,30993,31003],{"type":150,"tag":23637,"props":30984,"children":30985},{},[30986,30991],{"type":150,"tag":158,"props":30987,"children":30988},{},[30989],{"type":155,"value":30990},"CSA",{"type":155,"value":30992},"：先按大约 4:1 沿序列压缩，再用 Lightning Indexer 给每个 query 挑少量 top-k compressed blocks",{"type":150,"tag":23637,"props":30994,"children":30995},{},[30996,31001],{"type":150,"tag":158,"props":30997,"children":30998},{},[30999],{"type":155,"value":31000},"HCA",{"type":155,"value":31002},"：用更激进的序列压缩（约 128 倍），让 query 对压缩后的全局历史做 attention",{"type":150,"tag":23637,"props":31004,"children":31005},{},[31006,31011],{"type":150,"tag":158,"props":31007,"children":31008},{},[31009],{"type":155,"value":31010},"Sliding Window",{"type":155,"value":31012},"：保留最近的精细上下文",{"type":150,"tag":151,"props":31014,"children":31015},{},[31016],{"type":155,"value":31017},"这两个压缩率不是估的。Hugging Face 对 V4 的实现文档给出了默认值：CSA 是 4，HCA 是 128。为什么非要留一个 sliding window？因为压缩是有损的。CSA 压到 1\u002F4，HCA 压到 1\u002F128，最近的几个 token 一旦被压进 block，细节就再也找不回来了。而语言模型对紧接着的上下文又格外敏感。代词指谁、缩进几个空格、刚才那个变量叫什么名字、上一个 tool call 返回了什么。",{"type":150,"tag":151,"props":31019,"children":31020},{},[31021],{"type":155,"value":31022},"这些信息全都发生在最后几百个 token 里。",{"type":150,"tag":151,"props":31024,"children":31025},{},[31026,31028,31104,31106,31148,31150,31192,31194,31236],{"type":155,"value":31027},"所以最近的一段必须原样保留。三类 attention 是三段分工，不能互相替换：最近的原样看，中间挑着看，最远的粗着看。这个分工其实很符合直觉。人读一篇长文档也是这样：最后一段逐字读，中间扫小标题，开头的细节早忘了，只记得大意。于是 attention 不再是简单的 ",{"type":150,"tag":181,"props":31029,"children":31031},{"className":31030},[184],[31032,31061],{"type":150,"tag":181,"props":31033,"children":31035},{"className":31034},[189],[31036],{"type":150,"tag":192,"props":31037,"children":31038},{"xmlns":194},[31039],{"type":150,"tag":197,"props":31040,"children":31041},{},[31042,31057],{"type":150,"tag":201,"props":31043,"children":31044},{},[31045,31049,31053],{"type":150,"tag":209,"props":31046,"children":31047},{},[31048],{"type":155,"value":5417},{"type":150,"tag":221,"props":31050,"children":31051},{},[31052],{"type":155,"value":5380},{"type":150,"tag":209,"props":31054,"children":31055},{},[31056],{"type":155,"value":5417},{"type":150,"tag":271,"props":31058,"children":31059},{"encoding":273},[31060],{"type":155,"value":30825},{"type":150,"tag":181,"props":31062,"children":31064},{"className":31063,"ariaHidden":223},[280],[31065,31091],{"type":150,"tag":181,"props":31066,"children":31068},{"className":31067},[285],[31069,31073,31078,31082,31087],{"type":150,"tag":181,"props":31070,"children":31072},{"className":31071,"style":5470},[290],[],{"type":150,"tag":181,"props":31074,"children":31076},{"className":31075,"style":5606},[296,301],[31077],{"type":155,"value":5417},{"type":150,"tag":181,"props":31079,"children":31081},{"className":31080,"style":4993},[377],[],{"type":150,"tag":181,"props":31083,"children":31085},{"className":31084},[4998],[31086],{"type":155,"value":5380},{"type":150,"tag":181,"props":31088,"children":31090},{"className":31089,"style":4993},[377],[],{"type":150,"tag":181,"props":31092,"children":31094},{"className":31093},[285],[31095,31099],{"type":150,"tag":181,"props":31096,"children":31098},{"className":31097,"style":5680},[290],[],{"type":150,"tag":181,"props":31100,"children":31102},{"className":31101,"style":5606},[296,301],[31103],{"type":155,"value":5417},{"type":155,"value":31105},"，而是三种东西的叠加（记局部窗口大小为 ",{"type":150,"tag":181,"props":31107,"children":31109},{"className":31108},[184],[31110,31131],{"type":150,"tag":181,"props":31111,"children":31113},{"className":31112},[189],[31114],{"type":150,"tag":192,"props":31115,"children":31116},{"xmlns":194},[31117],{"type":150,"tag":197,"props":31118,"children":31119},{},[31120,31127],{"type":150,"tag":201,"props":31121,"children":31122},{},[31123],{"type":150,"tag":209,"props":31124,"children":31125},{},[31126],{"type":155,"value":3265},{"type":150,"tag":271,"props":31128,"children":31129},{"encoding":273},[31130],{"type":155,"value":3265},{"type":150,"tag":181,"props":31132,"children":31134},{"className":31133,"ariaHidden":223},[280],[31135],{"type":150,"tag":181,"props":31136,"children":31138},{"className":31137},[285],[31139,31143],{"type":150,"tag":181,"props":31140,"children":31142},{"className":31141,"style":5680},[290],[],{"type":150,"tag":181,"props":31144,"children":31146},{"className":31145,"style":3691},[296,301],[31147],{"type":155,"value":3265},{"type":155,"value":31149},"，每个 query 选中的 block 数为 ",{"type":150,"tag":181,"props":31151,"children":31153},{"className":31152},[184],[31154,31175],{"type":150,"tag":181,"props":31155,"children":31157},{"className":31156},[189],[31158],{"type":150,"tag":192,"props":31159,"children":31160},{"xmlns":194},[31161],{"type":150,"tag":197,"props":31162,"children":31163},{},[31164,31171],{"type":150,"tag":201,"props":31165,"children":31166},{},[31167],{"type":150,"tag":209,"props":31168,"children":31169},{},[31170],{"type":155,"value":23188},{"type":150,"tag":271,"props":31172,"children":31173},{"encoding":273},[31174],{"type":155,"value":23188},{"type":150,"tag":181,"props":31176,"children":31178},{"className":31177,"ariaHidden":223},[280],[31179],{"type":150,"tag":181,"props":31180,"children":31182},{"className":31181},[285],[31183,31187],{"type":150,"tag":181,"props":31184,"children":31186},{"className":31185,"style":5680},[290],[],{"type":150,"tag":181,"props":31188,"children":31190},{"className":31189,"style":27072},[296,301],[31191],{"type":155,"value":23188},{"type":155,"value":31193},"，压缩率为 ",{"type":150,"tag":181,"props":31195,"children":31197},{"className":31196},[184],[31198,31219],{"type":150,"tag":181,"props":31199,"children":31201},{"className":31200},[189],[31202],{"type":150,"tag":192,"props":31203,"children":31204},{"xmlns":194},[31205],{"type":150,"tag":197,"props":31206,"children":31207},{},[31208,31215],{"type":150,"tag":201,"props":31209,"children":31210},{},[31211],{"type":150,"tag":209,"props":31212,"children":31213},{},[31214],{"type":155,"value":20412},{"type":150,"tag":271,"props":31216,"children":31217},{"encoding":273},[31218],{"type":155,"value":20412},{"type":150,"tag":181,"props":31220,"children":31222},{"className":31221,"ariaHidden":223},[280],[31223],{"type":150,"tag":181,"props":31224,"children":31226},{"className":31225},[285],[31227,31231],{"type":150,"tag":181,"props":31228,"children":31230},{"className":31229,"style":914},[290],[],{"type":150,"tag":181,"props":31232,"children":31234},{"className":31233,"style":6794},[296,301],[31235],{"type":155,"value":20412},{"type":155,"value":31237},"）：",{"type":150,"tag":181,"props":31239,"children":31241},{"className":31240},[1003],[31242],{"type":150,"tag":181,"props":31243,"children":31245},{"className":31244},[184],[31246,31284],{"type":150,"tag":181,"props":31247,"children":31249},{"className":31248},[189],[31250],{"type":150,"tag":192,"props":31251,"children":31252},{"xmlns":194,"display":1015},[31253],{"type":150,"tag":197,"props":31254,"children":31255},{},[31256,31279],{"type":150,"tag":201,"props":31257,"children":31258},{},[31259,31263,31267,31271,31274],{"type":150,"tag":209,"props":31260,"children":31261},{},[31262],{"type":155,"value":5417},{"type":150,"tag":221,"props":31264,"children":31265},{},[31266],{"type":155,"value":5380},{"type":150,"tag":209,"props":31268,"children":31269},{},[31270],{"type":155,"value":3265},{"type":150,"tag":377,"props":31272,"children":31273},{"width":3313},[],{"type":150,"tag":249,"props":31275,"children":31276},{},[31277],{"type":155,"value":31278},"（局部窗口）",{"type":150,"tag":271,"props":31280,"children":31281},{"encoding":273},[31282],{"type":155,"value":31283},"N \\times W \\quad \\text{（局部窗口）}",{"type":150,"tag":181,"props":31285,"children":31287},{"className":31286,"ariaHidden":223},[280],[31288,31314],{"type":150,"tag":181,"props":31289,"children":31291},{"className":31290},[285],[31292,31296,31301,31305,31310],{"type":150,"tag":181,"props":31293,"children":31295},{"className":31294,"style":5470},[290],[],{"type":150,"tag":181,"props":31297,"children":31299},{"className":31298,"style":5606},[296,301],[31300],{"type":155,"value":5417},{"type":150,"tag":181,"props":31302,"children":31304},{"className":31303,"style":4993},[377],[],{"type":150,"tag":181,"props":31306,"children":31308},{"className":31307},[4998],[31309],{"type":155,"value":5380},{"type":150,"tag":181,"props":31311,"children":31313},{"className":31312,"style":4993},[377],[],{"type":150,"tag":181,"props":31315,"children":31317},{"className":31316},[285],[31318,31322,31327,31331],{"type":150,"tag":181,"props":31319,"children":31321},{"className":31320,"style":5680},[290],[],{"type":150,"tag":181,"props":31323,"children":31325},{"className":31324,"style":3691},[296,301],[31326],{"type":155,"value":3265},{"type":150,"tag":181,"props":31328,"children":31330},{"className":31329,"style":3909},[377],[],{"type":150,"tag":181,"props":31332,"children":31334},{"className":31333},[296,155],[31335],{"type":150,"tag":181,"props":31336,"children":31338},{"className":31337},[296,24835],[31339],{"type":155,"value":31278},{"type":150,"tag":181,"props":31341,"children":31343},{"className":31342},[1003],[31344],{"type":150,"tag":181,"props":31345,"children":31347},{"className":31346},[184],[31348,31386],{"type":150,"tag":181,"props":31349,"children":31351},{"className":31350},[189],[31352],{"type":150,"tag":192,"props":31353,"children":31354},{"xmlns":194,"display":1015},[31355],{"type":150,"tag":197,"props":31356,"children":31357},{},[31358,31381],{"type":150,"tag":201,"props":31359,"children":31360},{},[31361,31365,31369,31373,31376],{"type":150,"tag":209,"props":31362,"children":31363},{},[31364],{"type":155,"value":5417},{"type":150,"tag":221,"props":31366,"children":31367},{},[31368],{"type":155,"value":5380},{"type":150,"tag":209,"props":31370,"children":31371},{},[31372],{"type":155,"value":23188},{"type":150,"tag":377,"props":31374,"children":31375},{"width":3313},[],{"type":150,"tag":249,"props":31377,"children":31378},{},[31379],{"type":155,"value":31380},"（稀疏选中的 compressed blocks）",{"type":150,"tag":271,"props":31382,"children":31383},{"encoding":273},[31384],{"type":155,"value":31385},"N \\times B \\quad \\text{（稀疏选中的 compressed blocks）}",{"type":150,"tag":181,"props":31387,"children":31389},{"className":31388,"ariaHidden":223},[280],[31390,31416],{"type":150,"tag":181,"props":31391,"children":31393},{"className":31392},[285],[31394,31398,31403,31407,31412],{"type":150,"tag":181,"props":31395,"children":31397},{"className":31396,"style":5470},[290],[],{"type":150,"tag":181,"props":31399,"children":31401},{"className":31400,"style":5606},[296,301],[31402],{"type":155,"value":5417},{"type":150,"tag":181,"props":31404,"children":31406},{"className":31405,"style":4993},[377],[],{"type":150,"tag":181,"props":31408,"children":31410},{"className":31409},[4998],[31411],{"type":155,"value":5380},{"type":150,"tag":181,"props":31413,"children":31415},{"className":31414,"style":4993},[377],[],{"type":150,"tag":181,"props":31417,"children":31419},{"className":31418},[285],[31420,31424,31429,31433],{"type":150,"tag":181,"props":31421,"children":31423},{"className":31422,"style":941},[290],[],{"type":150,"tag":181,"props":31425,"children":31427},{"className":31426,"style":27072},[296,301],[31428],{"type":155,"value":23188},{"type":150,"tag":181,"props":31430,"children":31432},{"className":31431,"style":3909},[377],[],{"type":150,"tag":181,"props":31434,"children":31436},{"className":31435},[296,155],[31437,31443,31449],{"type":150,"tag":181,"props":31438,"children":31440},{"className":31439},[296,24835],[31441],{"type":155,"value":31442},"（稀疏选中的",{"type":150,"tag":181,"props":31444,"children":31446},{"className":31445},[296],[31447],{"type":155,"value":31448}," compressed blocks",{"type":150,"tag":181,"props":31450,"children":31452},{"className":31451},[296,24835],[31453],{"type":155,"value":31454},"）",{"type":150,"tag":181,"props":31456,"children":31458},{"className":31457},[1003],[31459],{"type":150,"tag":181,"props":31460,"children":31462},{"className":31461},[184],[31463,31508],{"type":150,"tag":181,"props":31464,"children":31466},{"className":31465},[189],[31467],{"type":150,"tag":192,"props":31468,"children":31469},{"xmlns":194,"display":1015},[31470],{"type":150,"tag":197,"props":31471,"children":31472},{},[31473,31503],{"type":150,"tag":201,"props":31474,"children":31475},{},[31476,31480,31484,31495,31498],{"type":150,"tag":209,"props":31477,"children":31478},{},[31479],{"type":155,"value":5417},{"type":150,"tag":221,"props":31481,"children":31482},{},[31483],{"type":155,"value":5380},{"type":150,"tag":1774,"props":31485,"children":31486},{},[31487,31491],{"type":150,"tag":209,"props":31488,"children":31489},{},[31490],{"type":155,"value":5417},{"type":150,"tag":209,"props":31492,"children":31493},{},[31494],{"type":155,"value":20412},{"type":150,"tag":377,"props":31496,"children":31497},{"width":3313},[],{"type":150,"tag":249,"props":31499,"children":31500},{},[31501],{"type":155,"value":31502},"（高度压缩的 global branch）",{"type":150,"tag":271,"props":31504,"children":31505},{"encoding":273},[31506],{"type":155,"value":31507},"N \\times \\frac{N}{r} \\quad \\text{（高度压缩的 global branch）}",{"type":150,"tag":181,"props":31509,"children":31511},{"className":31510,"ariaHidden":223},[280],[31512,31538],{"type":150,"tag":181,"props":31513,"children":31515},{"className":31514},[285],[31516,31520,31525,31529,31534],{"type":150,"tag":181,"props":31517,"children":31519},{"className":31518,"style":5470},[290],[],{"type":150,"tag":181,"props":31521,"children":31523},{"className":31522,"style":5606},[296,301],[31524],{"type":155,"value":5417},{"type":150,"tag":181,"props":31526,"children":31528},{"className":31527,"style":4993},[377],[],{"type":150,"tag":181,"props":31530,"children":31532},{"className":31531},[4998],[31533],{"type":155,"value":5380},{"type":150,"tag":181,"props":31535,"children":31537},{"className":31536,"style":4993},[377],[],{"type":150,"tag":181,"props":31539,"children":31541},{"className":31540},[285],[31542,31547,31634,31638],{"type":150,"tag":181,"props":31543,"children":31546},{"className":31544,"style":31545},[290],"height:2.0463em;vertical-align:-0.686em;",[],{"type":150,"tag":181,"props":31548,"children":31550},{"className":31549},[296],[31551,31555,31630],{"type":150,"tag":181,"props":31552,"children":31554},{"className":31553},[824,2482],[],{"type":150,"tag":181,"props":31556,"children":31558},{"className":31557},[1774],[31559],{"type":150,"tag":181,"props":31560,"children":31562},{"className":31561},[312,313],[31563,31619],{"type":150,"tag":181,"props":31564,"children":31566},{"className":31565},[318],[31567,31614],{"type":150,"tag":181,"props":31568,"children":31570},{"className":31569,"style":23490},[323],[31571,31587,31598],{"type":150,"tag":181,"props":31572,"children":31573},{"style":8235},[31574,31578],{"type":150,"tag":181,"props":31575,"children":31577},{"className":31576,"style":8240},[333],[],{"type":150,"tag":181,"props":31579,"children":31581},{"className":31580},[296],[31582],{"type":150,"tag":181,"props":31583,"children":31585},{"className":31584,"style":6794},[296,301],[31586],{"type":155,"value":20412},{"type":150,"tag":181,"props":31588,"children":31589},{"style":8253},[31590,31594],{"type":150,"tag":181,"props":31591,"children":31593},{"className":31592,"style":8240},[333],[],{"type":150,"tag":181,"props":31595,"children":31597},{"className":31596,"style":2804},[2803],[],{"type":150,"tag":181,"props":31599,"children":31600},{"style":8265},[31601,31605],{"type":150,"tag":181,"props":31602,"children":31604},{"className":31603,"style":8240},[333],[],{"type":150,"tag":181,"props":31606,"children":31608},{"className":31607},[296],[31609],{"type":150,"tag":181,"props":31610,"children":31612},{"className":31611,"style":5606},[296,301],[31613],{"type":155,"value":5417},{"type":150,"tag":181,"props":31615,"children":31617},{"className":31616},[352],[31618],{"type":155,"value":355},{"type":150,"tag":181,"props":31620,"children":31622},{"className":31621},[318],[31623],{"type":150,"tag":181,"props":31624,"children":31626},{"className":31625,"style":8297},[323],[31627],{"type":150,"tag":181,"props":31628,"children":31629},{},[],{"type":150,"tag":181,"props":31631,"children":31633},{"className":31632},[835,2482],[],{"type":150,"tag":181,"props":31635,"children":31637},{"className":31636,"style":3909},[377],[],{"type":150,"tag":181,"props":31639,"children":31641},{"className":31640},[296,155],[31642,31648,31654],{"type":150,"tag":181,"props":31643,"children":31645},{"className":31644},[296,24835],[31646],{"type":155,"value":31647},"（高度压缩的",{"type":150,"tag":181,"props":31649,"children":31651},{"className":31650},[296],[31652],{"type":155,"value":31653}," global branch",{"type":150,"tag":181,"props":31655,"children":31657},{"className":31656},[296,24835],[31658],{"type":155,"value":31454},{"type":150,"tag":30436,"props":31660,"children":31662},{"id":31661},"复杂度没有变成-on",[31663],{"type":155,"value":31664},"复杂度没有变成 O(N)",{"type":150,"tag":151,"props":31666,"children":31667},{},[31668,31670,31712,31714,31818,31820,31890],{"type":155,"value":31669},"如果 HCA 那一路的 ",{"type":150,"tag":181,"props":31671,"children":31673},{"className":31672},[184],[31674,31695],{"type":150,"tag":181,"props":31675,"children":31677},{"className":31676},[189],[31678],{"type":150,"tag":192,"props":31679,"children":31680},{"xmlns":194},[31681],{"type":150,"tag":197,"props":31682,"children":31683},{},[31684,31691],{"type":150,"tag":201,"props":31685,"children":31686},{},[31687],{"type":150,"tag":209,"props":31688,"children":31689},{},[31690],{"type":155,"value":20412},{"type":150,"tag":271,"props":31692,"children":31693},{"encoding":273},[31694],{"type":155,"value":20412},{"type":150,"tag":181,"props":31696,"children":31698},{"className":31697,"ariaHidden":223},[280],[31699],{"type":150,"tag":181,"props":31700,"children":31702},{"className":31701},[285],[31703,31707],{"type":150,"tag":181,"props":31704,"children":31706},{"className":31705,"style":914},[290],[],{"type":150,"tag":181,"props":31708,"children":31710},{"className":31709,"style":6794},[296,301],[31711],{"type":155,"value":20412},{"type":155,"value":31713}," 是个固定常数，那么它的理论渐进复杂度仍然带有 ",{"type":150,"tag":181,"props":31715,"children":31717},{"className":31716},[184],[31718,31755],{"type":150,"tag":181,"props":31719,"children":31721},{"className":31720},[189],[31722],{"type":150,"tag":192,"props":31723,"children":31724},{"xmlns":194},[31725],{"type":150,"tag":197,"props":31726,"children":31727},{},[31728,31750],{"type":150,"tag":201,"props":31729,"children":31730},{},[31731,31742,31746],{"type":150,"tag":565,"props":31732,"children":31733},{},[31734,31738],{"type":150,"tag":209,"props":31735,"children":31736},{},[31737],{"type":155,"value":5417},{"type":150,"tag":215,"props":31739,"children":31740},{},[31741],{"type":155,"value":238},{"type":150,"tag":209,"props":31743,"children":31744},{"mathvariant":1618},[31745],{"type":155,"value":4740},{"type":150,"tag":209,"props":31747,"children":31748},{},[31749],{"type":155,"value":20412},{"type":150,"tag":271,"props":31751,"children":31752},{"encoding":273},[31753],{"type":155,"value":31754},"N^2\u002Fr",{"type":150,"tag":181,"props":31756,"children":31758},{"className":31757,"ariaHidden":223},[280],[31759],{"type":150,"tag":181,"props":31760,"children":31762},{"className":31761},[285],[31763,31767,31808,31813],{"type":150,"tag":181,"props":31764,"children":31766},{"className":31765,"style":6789},[290],[],{"type":150,"tag":181,"props":31768,"children":31770},{"className":31769},[296],[31771,31776],{"type":150,"tag":181,"props":31772,"children":31774},{"className":31773,"style":5606},[296,301],[31775],{"type":155,"value":5417},{"type":150,"tag":181,"props":31777,"children":31779},{"className":31778},[307],[31780],{"type":150,"tag":181,"props":31781,"children":31783},{"className":31782},[312],[31784],{"type":150,"tag":181,"props":31785,"children":31787},{"className":31786},[318],[31788],{"type":150,"tag":181,"props":31789,"children":31791},{"className":31790,"style":6831},[323],[31792],{"type":150,"tag":181,"props":31793,"children":31794},{"style":707},[31795,31799],{"type":150,"tag":181,"props":31796,"children":31798},{"className":31797,"style":334},[333],[],{"type":150,"tag":181,"props":31800,"children":31802},{"className":31801},[339,340,341,342],[31803],{"type":150,"tag":181,"props":31804,"children":31806},{"className":31805},[296,342],[31807],{"type":155,"value":238},{"type":150,"tag":181,"props":31809,"children":31811},{"className":31810},[296],[31812],{"type":155,"value":4740},{"type":150,"tag":181,"props":31814,"children":31816},{"className":31815,"style":6794},[296,301],[31817],{"type":155,"value":20412},{"type":155,"value":31819},"，没有数学意义上变成 ",{"type":150,"tag":181,"props":31821,"children":31823},{"className":31822},[184],[31824,31858],{"type":150,"tag":181,"props":31825,"children":31827},{"className":31826},[189],[31828],{"type":150,"tag":192,"props":31829,"children":31830},{"xmlns":194},[31831],{"type":150,"tag":197,"props":31832,"children":31833},{},[31834,31853],{"type":150,"tag":201,"props":31835,"children":31836},{},[31837,31841,31845,31849],{"type":150,"tag":209,"props":31838,"children":31839},{},[31840],{"type":155,"value":6749},{"type":150,"tag":221,"props":31842,"children":31843},{"stretchy":789},[31844],{"type":155,"value":792},{"type":150,"tag":209,"props":31846,"children":31847},{},[31848],{"type":155,"value":5417},{"type":150,"tag":221,"props":31850,"children":31851},{"stretchy":789},[31852],{"type":155,"value":802},{"type":150,"tag":271,"props":31854,"children":31855},{"encoding":273},[31856],{"type":155,"value":31857},"O(N)",{"type":150,"tag":181,"props":31859,"children":31861},{"className":31860,"ariaHidden":223},[280],[31862],{"type":150,"tag":181,"props":31863,"children":31865},{"className":31864},[285],[31866,31870,31875,31880,31885],{"type":150,"tag":181,"props":31867,"children":31869},{"className":31868,"style":819},[290],[],{"type":150,"tag":181,"props":31871,"children":31873},{"className":31872,"style":6794},[296,301],[31874],{"type":155,"value":6749},{"type":150,"tag":181,"props":31876,"children":31878},{"className":31877},[824],[31879],{"type":155,"value":792},{"type":150,"tag":181,"props":31881,"children":31883},{"className":31882,"style":5606},[296,301],[31884],{"type":155,"value":5417},{"type":150,"tag":181,"props":31886,"children":31888},{"className":31887},[835],[31889],{"type":155,"value":802},{"type":155,"value":723},{"type":150,"tag":151,"props":31892,"children":31893},{},[31894],{"type":155,"value":31895},"但工程量级已经差很多。而 CSA 的 sparse branch 更接近：",{"type":150,"tag":181,"props":31897,"children":31899},{"className":31898},[1003],[31900],{"type":150,"tag":181,"props":31901,"children":31903},{"className":31902},[184],[31904,31962],{"type":150,"tag":181,"props":31905,"children":31907},{"className":31906},[189],[31908],{"type":150,"tag":192,"props":31909,"children":31910},{"xmlns":194,"display":1015},[31911],{"type":150,"tag":197,"props":31912,"children":31913},{},[31914,31957],{"type":150,"tag":201,"props":31915,"children":31916},{},[31917,31921,31925,31929,31933,31937,31941,31944,31948,31953],{"type":150,"tag":209,"props":31918,"children":31919},{},[31920],{"type":155,"value":6749},{"type":150,"tag":221,"props":31922,"children":31923},{"stretchy":789},[31924],{"type":155,"value":792},{"type":150,"tag":209,"props":31926,"children":31927},{},[31928],{"type":155,"value":5417},{"type":150,"tag":209,"props":31930,"children":31931},{},[31932],{"type":155,"value":23188},{"type":150,"tag":221,"props":31934,"children":31935},{"stretchy":789},[31936],{"type":155,"value":802},{"type":150,"tag":221,"props":31938,"children":31939},{"separator":223},[31940],{"type":155,"value":226},{"type":150,"tag":377,"props":31942,"children":31943},{"width":3313},[],{"type":150,"tag":209,"props":31945,"children":31946},{},[31947],{"type":155,"value":23188},{"type":150,"tag":221,"props":31949,"children":31950},{},[31951],{"type":155,"value":31952},"≪",{"type":150,"tag":209,"props":31954,"children":31955},{},[31956],{"type":155,"value":5417},{"type":150,"tag":271,"props":31958,"children":31959},{"encoding":273},[31960],{"type":155,"value":31961},"O(NB), \\quad B \\ll N",{"type":150,"tag":181,"props":31963,"children":31965},{"className":31964,"ariaHidden":223},[280],[31966,32030],{"type":150,"tag":181,"props":31967,"children":31969},{"className":31968},[285],[31970,31974,31979,31984,31989,31994,31999,32004,32008,32012,32017,32021,32026],{"type":150,"tag":181,"props":31971,"children":31973},{"className":31972,"style":819},[290],[],{"type":150,"tag":181,"props":31975,"children":31977},{"className":31976,"style":6794},[296,301],[31978],{"type":155,"value":6749},{"type":150,"tag":181,"props":31980,"children":31982},{"className":31981},[824],[31983],{"type":155,"value":792},{"type":150,"tag":181,"props":31985,"children":31987},{"className":31986,"style":5606},[296,301],[31988],{"type":155,"value":5417},{"type":150,"tag":181,"props":31990,"children":31992},{"className":31991,"style":27072},[296,301],[31993],{"type":155,"value":23188},{"type":150,"tag":181,"props":31995,"children":31997},{"className":31996},[835],[31998],{"type":155,"value":802},{"type":150,"tag":181,"props":32000,"children":32002},{"className":32001},[371],[32003],{"type":155,"value":226},{"type":150,"tag":181,"props":32005,"children":32007},{"className":32006,"style":3909},[377],[],{"type":150,"tag":181,"props":32009,"children":32011},{"className":32010,"style":378},[377],[],{"type":150,"tag":181,"props":32013,"children":32015},{"className":32014,"style":27072},[296,301],[32016],{"type":155,"value":23188},{"type":150,"tag":181,"props":32018,"children":32020},{"className":32019,"style":658},[377],[],{"type":150,"tag":181,"props":32022,"children":32024},{"className":32023},[663],[32025],{"type":155,"value":31952},{"type":150,"tag":181,"props":32027,"children":32029},{"className":32028,"style":658},[377],[],{"type":150,"tag":181,"props":32031,"children":32033},{"className":32032},[285],[32034,32038],{"type":150,"tag":181,"props":32035,"children":32037},{"className":32036,"style":5680},[290],[],{"type":150,"tag":181,"props":32039,"children":32041},{"className":32040,"style":5606},[296,301],[32042],{"type":155,"value":5417},{"type":150,"tag":151,"props":32044,"children":32045},{},[32046],{"type":155,"value":32047},"这是 V4 攻击 quadratic attention 的部分。所以\"V4 把 attention 变成线性了\"这种说法，把两件事混成了一件。严格结论和工程结论要分开讲。",{"type":150,"tag":169,"props":32049,"children":32051},{"id":32050},"linear-attention-与-ssm",[32052],{"type":155,"value":32053},"Linear Attention 与 SSM",{"type":150,"tag":151,"props":32055,"children":32056},{},[32057,32059,32064],{"type":155,"value":32058},"前面所有方案都建立在同一个前提上：KV 要保存下来。那换个问题。",{"type":150,"tag":158,"props":32060,"children":32061},{},[32062],{"type":155,"value":32063},"如果 KV 这么贵，为什么一定要保存它？",{"type":155,"value":32065}," RNN 的答案是最极端的：",{"type":150,"tag":181,"props":32067,"children":32069},{"className":32068},[1003],[32070],{"type":150,"tag":181,"props":32071,"children":32073},{"className":32072},[184],[32074,32191],{"type":150,"tag":181,"props":32075,"children":32077},{"className":32076},[189],[32078],{"type":150,"tag":192,"props":32079,"children":32080},{"xmlns":194,"display":1015},[32081],{"type":150,"tag":197,"props":32082,"children":32083},{},[32084,32186],{"type":150,"tag":201,"props":32085,"children":32086},{},[32087,32098,32102,32107,32111,32115,32126,32137,32141,32152,32174,32178,32182],{"type":150,"tag":205,"props":32088,"children":32089},{},[32090,32094],{"type":150,"tag":209,"props":32091,"children":32092},{},[32093],{"type":155,"value":745},{"type":150,"tag":209,"props":32095,"children":32096},{},[32097],{"type":155,"value":1035},{"type":150,"tag":221,"props":32099,"children":32100},{},[32101],{"type":155,"value":865},{"type":150,"tag":209,"props":32103,"children":32104},{},[32105],{"type":155,"value":32106},"tanh",{"type":150,"tag":221,"props":32108,"children":32109},{},[32110],{"type":155,"value":1813},{"type":150,"tag":221,"props":32112,"children":32113},{"stretchy":789},[32114],{"type":155,"value":792},{"type":150,"tag":205,"props":32116,"children":32117},{},[32118,32122],{"type":150,"tag":209,"props":32119,"children":32120},{},[32121],{"type":155,"value":3265},{"type":150,"tag":209,"props":32123,"children":32124},{},[32125],{"type":155,"value":213},{"type":150,"tag":205,"props":32127,"children":32128},{},[32129,32133],{"type":150,"tag":209,"props":32130,"children":32131},{},[32132],{"type":155,"value":213},{"type":150,"tag":209,"props":32134,"children":32135},{},[32136],{"type":155,"value":1035},{"type":150,"tag":221,"props":32138,"children":32139},{},[32140],{"type":155,"value":4962},{"type":150,"tag":205,"props":32142,"children":32143},{},[32144,32148],{"type":150,"tag":209,"props":32145,"children":32146},{},[32147],{"type":155,"value":3265},{"type":150,"tag":209,"props":32149,"children":32150},{},[32151],{"type":155,"value":745},{"type":150,"tag":205,"props":32153,"children":32154},{},[32155,32159],{"type":150,"tag":209,"props":32156,"children":32157},{},[32158],{"type":155,"value":745},{"type":150,"tag":201,"props":32160,"children":32161},{},[32162,32166,32170],{"type":150,"tag":209,"props":32163,"children":32164},{},[32165],{"type":155,"value":1035},{"type":150,"tag":221,"props":32167,"children":32168},{},[32169],{"type":155,"value":18794},{"type":150,"tag":215,"props":32171,"children":32172},{},[32173],{"type":155,"value":219},{"type":150,"tag":221,"props":32175,"children":32176},{},[32177],{"type":155,"value":4962},{"type":150,"tag":209,"props":32179,"children":32180},{},[32181],{"type":155,"value":22069},{"type":150,"tag":221,"props":32183,"children":32184},{"stretchy":789},[32185],{"type":155,"value":802},{"type":150,"tag":271,"props":32187,"children":32188},{"encoding":273},[32189],{"type":155,"value":32190},"h_t = \\tanh(W_x x_t + W_h h_{t-1} + b)",{"type":150,"tag":181,"props":32192,"children":32194},{"className":32193,"ariaHidden":223},[280],[32195,32273,32418,32568],{"type":150,"tag":181,"props":32196,"children":32198},{"className":32197},[285],[32199,32203,32260,32264,32269],{"type":150,"tag":181,"props":32200,"children":32202},{"className":32201,"style":4761},[290],[],{"type":150,"tag":181,"props":32204,"children":32206},{"className":32205},[296],[32207,32212],{"type":150,"tag":181,"props":32208,"children":32210},{"className":32209},[296,301],[32211],{"type":155,"value":745},{"type":150,"tag":181,"props":32213,"children":32215},{"className":32214},[307],[32216],{"type":150,"tag":181,"props":32217,"children":32219},{"className":32218},[312,313],[32220,32249],{"type":150,"tag":181,"props":32221,"children":32223},{"className":32222},[318],[32224,32244],{"type":150,"tag":181,"props":32225,"children":32227},{"className":32226,"style":1191},[323],[32228],{"type":150,"tag":181,"props":32229,"children":32230},{"style":328},[32231,32235],{"type":150,"tag":181,"props":32232,"children":32234},{"className":32233,"style":334},[333],[],{"type":150,"tag":181,"props":32236,"children":32238},{"className":32237},[339,340,341,342],[32239],{"type":150,"tag":181,"props":32240,"children":32242},{"className":32241},[296,301,342],[32243],{"type":155,"value":1035},{"type":150,"tag":181,"props":32245,"children":32247},{"className":32246},[352],[32248],{"type":155,"value":355},{"type":150,"tag":181,"props":32250,"children":32252},{"className":32251},[318],[32253],{"type":150,"tag":181,"props":32254,"children":32256},{"className":32255,"style":363},[323],[32257],{"type":150,"tag":181,"props":32258,"children":32259},{},[],{"type":150,"tag":181,"props":32261,"children":32263},{"className":32262,"style":658},[377],[],{"type":150,"tag":181,"props":32265,"children":32267},{"className":32266},[663],[32268],{"type":155,"value":865},{"type":150,"tag":181,"props":32270,"children":32272},{"className":32271,"style":658},[377],[],{"type":150,"tag":181,"props":32274,"children":32276},{"className":32275},[285],[32277,32281,32286,32291,32348,32405,32409,32414],{"type":150,"tag":181,"props":32278,"children":32280},{"className":32279,"style":819},[290],[],{"type":150,"tag":181,"props":32282,"children":32284},{"className":32283},[2517],[32285],{"type":155,"value":32106},{"type":150,"tag":181,"props":32287,"children":32289},{"className":32288},[824],[32290],{"type":155,"value":792},{"type":150,"tag":181,"props":32292,"children":32294},{"className":32293},[296],[32295,32300],{"type":150,"tag":181,"props":32296,"children":32298},{"className":32297,"style":3691},[296,301],[32299],{"type":155,"value":3265},{"type":150,"tag":181,"props":32301,"children":32303},{"className":32302},[307],[32304],{"type":150,"tag":181,"props":32305,"children":32307},{"className":32306},[312,313],[32308,32337],{"type":150,"tag":181,"props":32309,"children":32311},{"className":32310},[318],[32312,32332],{"type":150,"tag":181,"props":32313,"children":32315},{"className":32314,"style":4865},[323],[32316],{"type":150,"tag":181,"props":32317,"children":32318},{"style":11120},[32319,32323],{"type":150,"tag":181,"props":32320,"children":32322},{"className":32321,"style":334},[333],[],{"type":150,"tag":181,"props":32324,"children":32326},{"className":32325},[339,340,341,342],[32327],{"type":150,"tag":181,"props":32328,"children":32330},{"className":32329},[296,301,342],[32331],{"type":155,"value":213},{"type":150,"tag":181,"props":32333,"children":32335},{"className":32334},[352],[32336],{"type":155,"value":355},{"type":150,"tag":181,"props":32338,"children":32340},{"className":32339},[318],[32341],{"type":150,"tag":181,"props":32342,"children":32344},{"className":32343,"style":363},[323],[32345],{"type":150,"tag":181,"props":32346,"children":32347},{},[],{"type":150,"tag":181,"props":32349,"children":32351},{"className":32350},[296],[32352,32357],{"type":150,"tag":181,"props":32353,"children":32355},{"className":32354},[296,301],[32356],{"type":155,"value":213},{"type":150,"tag":181,"props":32358,"children":32360},{"className":32359},[307],[32361],{"type":150,"tag":181,"props":32362,"children":32364},{"className":32363},[312,313],[32365,32394],{"type":150,"tag":181,"props":32366,"children":32368},{"className":32367},[318],[32369,32389],{"type":150,"tag":181,"props":32370,"children":32372},{"className":32371,"style":1191},[323],[32373],{"type":150,"tag":181,"props":32374,"children":32375},{"style":328},[32376,32380],{"type":150,"tag":181,"props":32377,"children":32379},{"className":32378,"style":334},[333],[],{"type":150,"tag":181,"props":32381,"children":32383},{"className":32382},[339,340,341,342],[32384],{"type":150,"tag":181,"props":32385,"children":32387},{"className":32386},[296,301,342],[32388],{"type":155,"value":1035},{"type":150,"tag":181,"props":32390,"children":32392},{"className":32391},[352],[32393],{"type":155,"value":355},{"type":150,"tag":181,"props":32395,"children":32397},{"className":32396},[318],[32398],{"type":150,"tag":181,"props":32399,"children":32401},{"className":32400,"style":363},[323],[32402],{"type":150,"tag":181,"props":32403,"children":32404},{},[],{"type":150,"tag":181,"props":32406,"children":32408},{"className":32407,"style":4993},[377],[],{"type":150,"tag":181,"props":32410,"children":32412},{"className":32411},[4998],[32413],{"type":155,"value":4962},{"type":150,"tag":181,"props":32415,"children":32417},{"className":32416,"style":4993},[377],[],{"type":150,"tag":181,"props":32419,"children":32421},{"className":32420},[285],[32422,32427,32484,32555,32559,32564],{"type":150,"tag":181,"props":32423,"children":32426},{"className":32424,"style":32425},[290],"height:0.9028em;vertical-align:-0.2083em;",[],{"type":150,"tag":181,"props":32428,"children":32430},{"className":32429},[296],[32431,32436],{"type":150,"tag":181,"props":32432,"children":32434},{"className":32433,"style":3691},[296,301],[32435],{"type":155,"value":3265},{"type":150,"tag":181,"props":32437,"children":32439},{"className":32438},[307],[32440],{"type":150,"tag":181,"props":32441,"children":32443},{"className":32442},[312,313],[32444,32473],{"type":150,"tag":181,"props":32445,"children":32447},{"className":32446},[318],[32448,32468],{"type":150,"tag":181,"props":32449,"children":32451},{"className":32450,"style":493},[323],[32452],{"type":150,"tag":181,"props":32453,"children":32454},{"style":11120},[32455,32459],{"type":150,"tag":181,"props":32456,"children":32458},{"className":32457,"style":334},[333],[],{"type":150,"tag":181,"props":32460,"children":32462},{"className":32461},[339,340,341,342],[32463],{"type":150,"tag":181,"props":32464,"children":32466},{"className":32465},[296,301,342],[32467],{"type":155,"value":745},{"type":150,"tag":181,"props":32469,"children":32471},{"className":32470},[352],[32472],{"type":155,"value":355},{"type":150,"tag":181,"props":32474,"children":32476},{"className":32475},[318],[32477],{"type":150,"tag":181,"props":32478,"children":32480},{"className":32479,"style":363},[323],[32481],{"type":150,"tag":181,"props":32482,"children":32483},{},[],{"type":150,"tag":181,"props":32485,"children":32487},{"className":32486},[296],[32488,32493],{"type":150,"tag":181,"props":32489,"children":32491},{"className":32490},[296,301],[32492],{"type":155,"value":745},{"type":150,"tag":181,"props":32494,"children":32496},{"className":32495},[307],[32497],{"type":150,"tag":181,"props":32498,"children":32500},{"className":32499},[312,313],[32501,32544],{"type":150,"tag":181,"props":32502,"children":32504},{"className":32503},[318],[32505,32539],{"type":150,"tag":181,"props":32506,"children":32508},{"className":32507,"style":324},[323],[32509],{"type":150,"tag":181,"props":32510,"children":32511},{"style":328},[32512,32516],{"type":150,"tag":181,"props":32513,"children":32515},{"className":32514,"style":334},[333],[],{"type":150,"tag":181,"props":32517,"children":32519},{"className":32518},[339,340,341,342],[32520],{"type":150,"tag":181,"props":32521,"children":32523},{"className":32522},[296,342],[32524,32529,32534],{"type":150,"tag":181,"props":32525,"children":32527},{"className":32526},[296,301,342],[32528],{"type":155,"value":1035},{"type":150,"tag":181,"props":32530,"children":32532},{"className":32531},[4998,342],[32533],{"type":155,"value":18794},{"type":150,"tag":181,"props":32535,"children":32537},{"className":32536},[296,342],[32538],{"type":155,"value":219},{"type":150,"tag":181,"props":32540,"children":32542},{"className":32541},[352],[32543],{"type":155,"value":355},{"type":150,"tag":181,"props":32545,"children":32547},{"className":32546},[318],[32548],{"type":150,"tag":181,"props":32549,"children":32551},{"className":32550,"style":19762},[323],[32552],{"type":150,"tag":181,"props":32553,"children":32554},{},[],{"type":150,"tag":181,"props":32556,"children":32558},{"className":32557,"style":4993},[377],[],{"type":150,"tag":181,"props":32560,"children":32562},{"className":32561},[4998],[32563],{"type":155,"value":4962},{"type":150,"tag":181,"props":32565,"children":32567},{"className":32566,"style":4993},[377],[],{"type":150,"tag":181,"props":32569,"children":32571},{"className":32570},[285],[32572,32576,32581],{"type":150,"tag":181,"props":32573,"children":32575},{"className":32574,"style":819},[290],[],{"type":150,"tag":181,"props":32577,"children":32579},{"className":32578},[296,301],[32580],{"type":155,"value":22069},{"type":150,"tag":181,"props":32582,"children":32584},{"className":32583},[835],[32585],{"type":155,"value":802},{"type":150,"tag":151,"props":32587,"children":32588},{},[32589],{"type":155,"value":32590},"过去全部信息压进一个固定大小的状态。不保存 KV，只保存状态。代价是长期记忆很差，而且不能并行。LSTM \u002F GRU 用门控改善了记忆，但没根治，也仍然不能并行。然后 Linear Attention 出现了：",{"type":150,"tag":181,"props":32592,"children":32594},{"className":32593},[1003],[32595],{"type":150,"tag":181,"props":32596,"children":32598},{"className":32597},[184],[32599,32698],{"type":150,"tag":181,"props":32600,"children":32602},{"className":32601},[189],[32603],{"type":150,"tag":192,"props":32604,"children":32605},{"xmlns":194,"display":1015},[32606],{"type":150,"tag":197,"props":32607,"children":32608},{},[32609,32693],{"type":150,"tag":201,"props":32610,"children":32611},{},[32612,32624,32628,32650,32654,32659,32663,32674,32678],{"type":150,"tag":205,"props":32613,"children":32614},{},[32615,32620],{"type":150,"tag":209,"props":32616,"children":32617},{},[32618],{"type":155,"value":32619},"S",{"type":150,"tag":209,"props":32621,"children":32622},{},[32623],{"type":155,"value":1035},{"type":150,"tag":221,"props":32625,"children":32626},{},[32627],{"type":155,"value":865},{"type":150,"tag":205,"props":32629,"children":32630},{},[32631,32635],{"type":150,"tag":209,"props":32632,"children":32633},{},[32634],{"type":155,"value":32619},{"type":150,"tag":201,"props":32636,"children":32637},{},[32638,32642,32646],{"type":150,"tag":209,"props":32639,"children":32640},{},[32641],{"type":155,"value":1035},{"type":150,"tag":221,"props":32643,"children":32644},{},[32645],{"type":155,"value":18794},{"type":150,"tag":215,"props":32647,"children":32648},{},[32649],{"type":155,"value":219},{"type":150,"tag":221,"props":32651,"children":32652},{},[32653],{"type":155,"value":4962},{"type":150,"tag":209,"props":32655,"children":32656},{},[32657],{"type":155,"value":32658},"ϕ",{"type":150,"tag":221,"props":32660,"children":32661},{"stretchy":789},[32662],{"type":155,"value":792},{"type":150,"tag":205,"props":32664,"children":32665},{},[32666,32670],{"type":150,"tag":209,"props":32667,"children":32668},{},[32669],{"type":155,"value":1698},{"type":150,"tag":209,"props":32671,"children":32672},{},[32673],{"type":155,"value":1035},{"type":150,"tag":221,"props":32675,"children":32676},{"stretchy":789},[32677],{"type":155,"value":802},{"type":150,"tag":1046,"props":32679,"children":32680},{},[32681,32685,32689],{"type":150,"tag":209,"props":32682,"children":32683},{},[32684],{"type":155,"value":1737},{"type":150,"tag":209,"props":32686,"children":32687},{},[32688],{"type":155,"value":1035},{"type":150,"tag":209,"props":32690,"children":32691},{"mathvariant":1618},[32692],{"type":155,"value":1874},{"type":150,"tag":271,"props":32694,"children":32695},{"encoding":273},[32696],{"type":155,"value":32697},"S_t = S_{t-1} + \\phi(k_t) v_t^\\top",{"type":150,"tag":181,"props":32699,"children":32701},{"className":32700,"ariaHidden":223},[280],[32702,32782,32875],{"type":150,"tag":181,"props":32703,"children":32705},{"className":32704},[285],[32706,32710,32769,32773,32778],{"type":150,"tag":181,"props":32707,"children":32709},{"className":32708,"style":6190},[290],[],{"type":150,"tag":181,"props":32711,"children":32713},{"className":32712},[296],[32714,32720],{"type":150,"tag":181,"props":32715,"children":32718},{"className":32716,"style":32717},[296,301],"margin-right:0.0576em;",[32719],{"type":155,"value":32619},{"type":150,"tag":181,"props":32721,"children":32723},{"className":32722},[307],[32724],{"type":150,"tag":181,"props":32725,"children":32727},{"className":32726},[312,313],[32728,32758],{"type":150,"tag":181,"props":32729,"children":32731},{"className":32730},[318],[32732,32753],{"type":150,"tag":181,"props":32733,"children":32735},{"className":32734,"style":1191},[323],[32736],{"type":150,"tag":181,"props":32737,"children":32739},{"style":32738},"top:-2.55em;margin-left:-0.0576em;margin-right:0.05em;",[32740,32744],{"type":150,"tag":181,"props":32741,"children":32743},{"className":32742,"style":334},[333],[],{"type":150,"tag":181,"props":32745,"children":32747},{"className":32746},[339,340,341,342],[32748],{"type":150,"tag":181,"props":32749,"children":32751},{"className":32750},[296,301,342],[32752],{"type":155,"value":1035},{"type":150,"tag":181,"props":32754,"children":32756},{"className":32755},[352],[32757],{"type":155,"value":355},{"type":150,"tag":181,"props":32759,"children":32761},{"className":32760},[318],[32762],{"type":150,"tag":181,"props":32763,"children":32765},{"className":32764,"style":363},[323],[32766],{"type":150,"tag":181,"props":32767,"children":32768},{},[],{"type":150,"tag":181,"props":32770,"children":32772},{"className":32771,"style":658},[377],[],{"type":150,"tag":181,"props":32774,"children":32776},{"className":32775},[663],[32777],{"type":155,"value":865},{"type":150,"tag":181,"props":32779,"children":32781},{"className":32780,"style":658},[377],[],{"type":150,"tag":181,"props":32783,"children":32785},{"className":32784},[285],[32786,32791,32862,32866,32871],{"type":150,"tag":181,"props":32787,"children":32790},{"className":32788,"style":32789},[290],"height:0.8917em;vertical-align:-0.2083em;",[],{"type":150,"tag":181,"props":32792,"children":32794},{"className":32793},[296],[32795,32800],{"type":150,"tag":181,"props":32796,"children":32798},{"className":32797,"style":32717},[296,301],[32799],{"type":155,"value":32619},{"type":150,"tag":181,"props":32801,"children":32803},{"className":32802},[307],[32804],{"type":150,"tag":181,"props":32805,"children":32807},{"className":32806},[312,313],[32808,32851],{"type":150,"tag":181,"props":32809,"children":32811},{"className":32810},[318],[32812,32846],{"type":150,"tag":181,"props":32813,"children":32815},{"className":32814,"style":324},[323],[32816],{"type":150,"tag":181,"props":32817,"children":32818},{"style":32738},[32819,32823],{"type":150,"tag":181,"props":32820,"children":32822},{"className":32821,"style":334},[333],[],{"type":150,"tag":181,"props":32824,"children":32826},{"className":32825},[339,340,341,342],[32827],{"type":150,"tag":181,"props":32828,"children":32830},{"className":32829},[296,342],[32831,32836,32841],{"type":150,"tag":181,"props":32832,"children":32834},{"className":32833},[296,301,342],[32835],{"type":155,"value":1035},{"type":150,"tag":181,"props":32837,"children":32839},{"className":32838},[4998,342],[32840],{"type":155,"value":18794},{"type":150,"tag":181,"props":32842,"children":32844},{"className":32843},[296,342],[32845],{"type":155,"value":219},{"type":150,"tag":181,"props":32847,"children":32849},{"className":32848},[352],[32850],{"type":155,"value":355},{"type":150,"tag":181,"props":32852,"children":32854},{"className":32853},[318],[32855],{"type":150,"tag":181,"props":32856,"children":32858},{"className":32857,"style":19762},[323],[32859],{"type":150,"tag":181,"props":32860,"children":32861},{},[],{"type":150,"tag":181,"props":32863,"children":32865},{"className":32864,"style":4993},[377],[],{"type":150,"tag":181,"props":32867,"children":32869},{"className":32868},[4998],[32870],{"type":155,"value":4962},{"type":150,"tag":181,"props":32872,"children":32874},{"className":32873,"style":4993},[377],[],{"type":150,"tag":181,"props":32876,"children":32878},{"className":32877},[285],[32879,32884,32889,32894,32951,32956],{"type":150,"tag":181,"props":32880,"children":32883},{"className":32881,"style":32882},[290],"height:1.1491em;vertical-align:-0.25em;",[],{"type":150,"tag":181,"props":32885,"children":32887},{"className":32886},[296,301],[32888],{"type":155,"value":32658},{"type":150,"tag":181,"props":32890,"children":32892},{"className":32891},[824],[32893],{"type":155,"value":792},{"type":150,"tag":181,"props":32895,"children":32897},{"className":32896},[296],[32898,32903],{"type":150,"tag":181,"props":32899,"children":32901},{"className":32900,"style":2249},[296,301],[32902],{"type":155,"value":1698},{"type":150,"tag":181,"props":32904,"children":32906},{"className":32905},[307],[32907],{"type":150,"tag":181,"props":32908,"children":32910},{"className":32909},[312,313],[32911,32940],{"type":150,"tag":181,"props":32912,"children":32914},{"className":32913},[318],[32915,32935],{"type":150,"tag":181,"props":32916,"children":32918},{"className":32917,"style":1191},[323],[32919],{"type":150,"tag":181,"props":32920,"children":32921},{"style":14163},[32922,32926],{"type":150,"tag":181,"props":32923,"children":32925},{"className":32924,"style":334},[333],[],{"type":150,"tag":181,"props":32927,"children":32929},{"className":32928},[339,340,341,342],[32930],{"type":150,"tag":181,"props":32931,"children":32933},{"className":32932},[296,301,342],[32934],{"type":155,"value":1035},{"type":150,"tag":181,"props":32936,"children":32938},{"className":32937},[352],[32939],{"type":155,"value":355},{"type":150,"tag":181,"props":32941,"children":32943},{"className":32942},[318],[32944],{"type":150,"tag":181,"props":32945,"children":32947},{"className":32946,"style":363},[323],[32948],{"type":150,"tag":181,"props":32949,"children":32950},{},[],{"type":150,"tag":181,"props":32952,"children":32954},{"className":32953},[835],[32955],{"type":155,"value":802},{"type":150,"tag":181,"props":32957,"children":32959},{"className":32958},[296],[32960,32965],{"type":150,"tag":181,"props":32961,"children":32963},{"className":32962,"style":2151},[296,301],[32964],{"type":155,"value":1737},{"type":150,"tag":181,"props":32966,"children":32968},{"className":32967},[307],[32969],{"type":150,"tag":181,"props":32970,"children":32972},{"className":32971},[312,313],[32973,33019],{"type":150,"tag":181,"props":32974,"children":32976},{"className":32975},[318],[32977,33014],{"type":150,"tag":181,"props":32978,"children":32980},{"className":32979,"style":3821},[323],[32981,32998],{"type":150,"tag":181,"props":32982,"children":32984},{"style":32983},"top:-2.453em;margin-left:-0.0359em;margin-right:0.05em;",[32985,32989],{"type":150,"tag":181,"props":32986,"children":32988},{"className":32987,"style":334},[333],[],{"type":150,"tag":181,"props":32990,"children":32992},{"className":32991},[339,340,341,342],[32993],{"type":150,"tag":181,"props":32994,"children":32996},{"className":32995},[296,301,342],[32997],{"type":155,"value":1035},{"type":150,"tag":181,"props":32999,"children":33000},{"style":3730},[33001,33005],{"type":150,"tag":181,"props":33002,"children":33004},{"className":33003,"style":334},[333],[],{"type":150,"tag":181,"props":33006,"children":33008},{"className":33007},[339,340,341,342],[33009],{"type":150,"tag":181,"props":33010,"children":33012},{"className":33011},[296,342],[33013],{"type":155,"value":1874},{"type":150,"tag":181,"props":33015,"children":33017},{"className":33016},[352],[33018],{"type":155,"value":355},{"type":150,"tag":181,"props":33020,"children":33022},{"className":33021},[318],[33023],{"type":150,"tag":181,"props":33024,"children":33026},{"className":33025,"style":4550},[323],[33027],{"type":150,"tag":181,"props":33028,"children":33029},{},[],{"type":150,"tag":181,"props":33031,"children":33033},{"className":33032},[1003],[33034],{"type":150,"tag":181,"props":33035,"children":33037},{"className":33036},[184],[33038,33112],{"type":150,"tag":181,"props":33039,"children":33041},{"className":33040},[189],[33042],{"type":150,"tag":192,"props":33043,"children":33044},{"xmlns":194,"display":1015},[33045],{"type":150,"tag":197,"props":33046,"children":33047},{},[33048,33107],{"type":150,"tag":201,"props":33049,"children":33050},{},[33051,33062,33066,33070,33074,33085,33096],{"type":150,"tag":205,"props":33052,"children":33053},{},[33054,33058],{"type":150,"tag":209,"props":33055,"children":33056},{},[33057],{"type":155,"value":1030},{"type":150,"tag":209,"props":33059,"children":33060},{},[33061],{"type":155,"value":1035},{"type":150,"tag":221,"props":33063,"children":33064},{},[33065],{"type":155,"value":865},{"type":150,"tag":209,"props":33067,"children":33068},{},[33069],{"type":155,"value":32658},{"type":150,"tag":221,"props":33071,"children":33072},{"stretchy":789},[33073],{"type":155,"value":792},{"type":150,"tag":205,"props":33075,"children":33076},{},[33077,33081],{"type":150,"tag":209,"props":33078,"children":33079},{},[33080],{"type":155,"value":1667},{"type":150,"tag":209,"props":33082,"children":33083},{},[33084],{"type":155,"value":1035},{"type":150,"tag":565,"props":33086,"children":33087},{},[33088,33092],{"type":150,"tag":221,"props":33089,"children":33090},{"stretchy":789},[33091],{"type":155,"value":802},{"type":150,"tag":209,"props":33093,"children":33094},{"mathvariant":1618},[33095],{"type":155,"value":1874},{"type":150,"tag":205,"props":33097,"children":33098},{},[33099,33103],{"type":150,"tag":209,"props":33100,"children":33101},{},[33102],{"type":155,"value":32619},{"type":150,"tag":209,"props":33104,"children":33105},{},[33106],{"type":155,"value":1035},{"type":150,"tag":271,"props":33108,"children":33109},{"encoding":273},[33110],{"type":155,"value":33111},"o_t = \\phi(q_t)^\\top S_t",{"type":150,"tag":181,"props":33113,"children":33115},{"className":33114,"ariaHidden":223},[280],[33116,33194],{"type":150,"tag":181,"props":33117,"children":33119},{"className":33118},[285],[33120,33124,33181,33185,33190],{"type":150,"tag":181,"props":33121,"children":33123},{"className":33122,"style":1165},[290],[],{"type":150,"tag":181,"props":33125,"children":33127},{"className":33126},[296],[33128,33133],{"type":150,"tag":181,"props":33129,"children":33131},{"className":33130},[296,301],[33132],{"type":155,"value":1030},{"type":150,"tag":181,"props":33134,"children":33136},{"className":33135},[307],[33137],{"type":150,"tag":181,"props":33138,"children":33140},{"className":33139},[312,313],[33141,33170],{"type":150,"tag":181,"props":33142,"children":33144},{"className":33143},[318],[33145,33165],{"type":150,"tag":181,"props":33146,"children":33148},{"className":33147,"style":1191},[323],[33149],{"type":150,"tag":181,"props":33150,"children":33151},{"style":328},[33152,33156],{"type":150,"tag":181,"props":33153,"children":33155},{"className":33154,"style":334},[333],[],{"type":150,"tag":181,"props":33157,"children":33159},{"className":33158},[339,340,341,342],[33160],{"type":150,"tag":181,"props":33161,"children":33163},{"className":33162},[296,301,342],[33164],{"type":155,"value":1035},{"type":150,"tag":181,"props":33166,"children":33168},{"className":33167},[352],[33169],{"type":155,"value":355},{"type":150,"tag":181,"props":33171,"children":33173},{"className":33172},[318],[33174],{"type":150,"tag":181,"props":33175,"children":33177},{"className":33176,"style":363},[323],[33178],{"type":150,"tag":181,"props":33179,"children":33180},{},[],{"type":150,"tag":181,"props":33182,"children":33184},{"className":33183,"style":658},[377],[],{"type":150,"tag":181,"props":33186,"children":33188},{"className":33187},[663],[33189],{"type":155,"value":865},{"type":150,"tag":181,"props":33191,"children":33193},{"className":33192,"style":658},[377],[],{"type":150,"tag":181,"props":33195,"children":33197},{"className":33196},[285],[33198,33202,33207,33212,33269,33310],{"type":150,"tag":181,"props":33199,"children":33201},{"className":33200,"style":32882},[290],[],{"type":150,"tag":181,"props":33203,"children":33205},{"className":33204},[296,301],[33206],{"type":155,"value":32658},{"type":150,"tag":181,"props":33208,"children":33210},{"className":33209},[824],[33211],{"type":155,"value":792},{"type":150,"tag":181,"props":33213,"children":33215},{"className":33214},[296],[33216,33221],{"type":150,"tag":181,"props":33217,"children":33219},{"className":33218,"style":2151},[296,301],[33220],{"type":155,"value":1667},{"type":150,"tag":181,"props":33222,"children":33224},{"className":33223},[307],[33225],{"type":150,"tag":181,"props":33226,"children":33228},{"className":33227},[312,313],[33229,33258],{"type":150,"tag":181,"props":33230,"children":33232},{"className":33231},[318],[33233,33253],{"type":150,"tag":181,"props":33234,"children":33236},{"className":33235,"style":1191},[323],[33237],{"type":150,"tag":181,"props":33238,"children":33239},{"style":14230},[33240,33244],{"type":150,"tag":181,"props":33241,"children":33243},{"className":33242,"style":334},[333],[],{"type":150,"tag":181,"props":33245,"children":33247},{"className":33246},[339,340,341,342],[33248],{"type":150,"tag":181,"props":33249,"children":33251},{"className":33250},[296,301,342],[33252],{"type":155,"value":1035},{"type":150,"tag":181,"props":33254,"children":33256},{"className":33255},[352],[33257],{"type":155,"value":355},{"type":150,"tag":181,"props":33259,"children":33261},{"className":33260},[318],[33262],{"type":150,"tag":181,"props":33263,"children":33265},{"className":33264,"style":363},[323],[33266],{"type":150,"tag":181,"props":33267,"children":33268},{},[],{"type":150,"tag":181,"props":33270,"children":33272},{"className":33271},[835],[33273,33278],{"type":150,"tag":181,"props":33274,"children":33276},{"className":33275},[835],[33277],{"type":155,"value":802},{"type":150,"tag":181,"props":33279,"children":33281},{"className":33280},[307],[33282],{"type":150,"tag":181,"props":33283,"children":33285},{"className":33284},[312],[33286],{"type":150,"tag":181,"props":33287,"children":33289},{"className":33288},[318],[33290],{"type":150,"tag":181,"props":33291,"children":33293},{"className":33292,"style":3821},[323],[33294],{"type":150,"tag":181,"props":33295,"children":33296},{"style":3730},[33297,33301],{"type":150,"tag":181,"props":33298,"children":33300},{"className":33299,"style":334},[333],[],{"type":150,"tag":181,"props":33302,"children":33304},{"className":33303},[339,340,341,342],[33305],{"type":150,"tag":181,"props":33306,"children":33308},{"className":33307},[296,342],[33309],{"type":155,"value":1874},{"type":150,"tag":181,"props":33311,"children":33313},{"className":33312},[296],[33314,33319],{"type":150,"tag":181,"props":33315,"children":33317},{"className":33316,"style":32717},[296,301],[33318],{"type":155,"value":32619},{"type":150,"tag":181,"props":33320,"children":33322},{"className":33321},[307],[33323],{"type":150,"tag":181,"props":33324,"children":33326},{"className":33325},[312,313],[33327,33356],{"type":150,"tag":181,"props":33328,"children":33330},{"className":33329},[318],[33331,33351],{"type":150,"tag":181,"props":33332,"children":33334},{"className":33333,"style":1191},[323],[33335],{"type":150,"tag":181,"props":33336,"children":33337},{"style":32738},[33338,33342],{"type":150,"tag":181,"props":33339,"children":33341},{"className":33340,"style":334},[333],[],{"type":150,"tag":181,"props":33343,"children":33345},{"className":33344},[339,340,341,342],[33346],{"type":150,"tag":181,"props":33347,"children":33349},{"className":33348},[296,301,342],[33350],{"type":155,"value":1035},{"type":150,"tag":181,"props":33352,"children":33354},{"className":33353},[352],[33355],{"type":155,"value":355},{"type":150,"tag":181,"props":33357,"children":33359},{"className":33358},[318],[33360],{"type":150,"tag":181,"props":33361,"children":33363},{"className":33362,"style":363},[323],[33364],{"type":150,"tag":181,"props":33365,"children":33366},{},[],{"type":150,"tag":151,"props":33368,"children":33369},{},[33370,33372,33441,33443,33448,33450,33455],{"type":155,"value":33371},"每步只更新一个 state，复杂度 ",{"type":150,"tag":181,"props":33373,"children":33375},{"className":33374},[184],[33376,33409],{"type":150,"tag":181,"props":33377,"children":33379},{"className":33378},[189],[33380],{"type":150,"tag":192,"props":33381,"children":33382},{"xmlns":194},[33383],{"type":150,"tag":197,"props":33384,"children":33385},{},[33386,33405],{"type":150,"tag":201,"props":33387,"children":33388},{},[33389,33393,33397,33401],{"type":150,"tag":209,"props":33390,"children":33391},{},[33392],{"type":155,"value":6749},{"type":150,"tag":221,"props":33394,"children":33395},{"stretchy":789},[33396],{"type":155,"value":792},{"type":150,"tag":209,"props":33398,"children":33399},{},[33400],{"type":155,"value":5417},{"type":150,"tag":221,"props":33402,"children":33403},{"stretchy":789},[33404],{"type":155,"value":802},{"type":150,"tag":271,"props":33406,"children":33407},{"encoding":273},[33408],{"type":155,"value":31857},{"type":150,"tag":181,"props":33410,"children":33412},{"className":33411,"ariaHidden":223},[280],[33413],{"type":150,"tag":181,"props":33414,"children":33416},{"className":33415},[285],[33417,33421,33426,33431,33436],{"type":150,"tag":181,"props":33418,"children":33420},{"className":33419,"style":819},[290],[],{"type":150,"tag":181,"props":33422,"children":33424},{"className":33423,"style":6794},[296,301],[33425],{"type":155,"value":6749},{"type":150,"tag":181,"props":33427,"children":33429},{"className":33428},[824],[33430],{"type":155,"value":792},{"type":150,"tag":181,"props":33432,"children":33434},{"className":33433,"style":5606},[296,301],[33435],{"type":155,"value":5417},{"type":150,"tag":181,"props":33437,"children":33439},{"className":33438},[835],[33440],{"type":155,"value":802},{"type":155,"value":33442},"。看到这个式子会觉得眼熟：它就是 ",{"type":150,"tag":158,"props":33444,"children":33445},{},[33446],{"type":155,"value":33447},"LSTM 式的 recurrent state",{"type":155,"value":33449}," 加 ",{"type":150,"tag":158,"props":33451,"children":33452},{},[33453],{"type":155,"value":33454},"Attention 式的 query-key retrieval",{"type":155,"value":723},{"type":150,"tag":151,"props":33457,"children":33458},{},[33459,33461,33466,33468,33570,33572,33767,33769,33811],{"type":155,"value":33460},"所以它像一个\"现代 LSTM\"是有道理的，不是公式恰巧长得像。它重新选择了",{"type":150,"tag":158,"props":33462,"children":33463},{},[33464],{"type":155,"value":33465},"压缩历史，而不是保存历史",{"type":155,"value":33467},"。代价是信息碰撞。不同的 key 的信息全挤进同一个 state，容量有限，互相干扰。state ",{"type":150,"tag":181,"props":33469,"children":33471},{"className":33470},[184],[33472,33501],{"type":150,"tag":181,"props":33473,"children":33475},{"className":33474},[189],[33476],{"type":150,"tag":192,"props":33477,"children":33478},{"xmlns":194},[33479],{"type":150,"tag":197,"props":33480,"children":33481},{},[33482,33496],{"type":150,"tag":201,"props":33483,"children":33484},{},[33485],{"type":150,"tag":205,"props":33486,"children":33487},{},[33488,33492],{"type":150,"tag":209,"props":33489,"children":33490},{},[33491],{"type":155,"value":32619},{"type":150,"tag":209,"props":33493,"children":33494},{},[33495],{"type":155,"value":1035},{"type":150,"tag":271,"props":33497,"children":33498},{"encoding":273},[33499],{"type":155,"value":33500},"S_t",{"type":150,"tag":181,"props":33502,"children":33504},{"className":33503,"ariaHidden":223},[280],[33505],{"type":150,"tag":181,"props":33506,"children":33508},{"className":33507},[285],[33509,33513],{"type":150,"tag":181,"props":33510,"children":33512},{"className":33511,"style":6190},[290],[],{"type":150,"tag":181,"props":33514,"children":33516},{"className":33515},[296],[33517,33522],{"type":150,"tag":181,"props":33518,"children":33520},{"className":33519,"style":32717},[296,301],[33521],{"type":155,"value":32619},{"type":150,"tag":181,"props":33523,"children":33525},{"className":33524},[307],[33526],{"type":150,"tag":181,"props":33527,"children":33529},{"className":33528},[312,313],[33530,33559],{"type":150,"tag":181,"props":33531,"children":33533},{"className":33532},[318],[33534,33554],{"type":150,"tag":181,"props":33535,"children":33537},{"className":33536,"style":1191},[323],[33538],{"type":150,"tag":181,"props":33539,"children":33540},{"style":32738},[33541,33545],{"type":150,"tag":181,"props":33542,"children":33544},{"className":33543,"style":334},[333],[],{"type":150,"tag":181,"props":33546,"children":33548},{"className":33547},[339,340,341,342],[33549],{"type":150,"tag":181,"props":33550,"children":33552},{"className":33551},[296,301,342],[33553],{"type":155,"value":1035},{"type":150,"tag":181,"props":33555,"children":33557},{"className":33556},[352],[33558],{"type":155,"value":355},{"type":150,"tag":181,"props":33560,"children":33562},{"className":33561},[318],[33563],{"type":150,"tag":181,"props":33564,"children":33566},{"className":33565,"style":363},[323],[33567],{"type":150,"tag":181,"props":33568,"children":33569},{},[],{"type":155,"value":33571}," 的大小是固定的，跟 ",{"type":150,"tag":181,"props":33573,"children":33575},{"className":33574},[184],[33576,33620],{"type":150,"tag":181,"props":33577,"children":33579},{"className":33578},[189],[33580],{"type":150,"tag":192,"props":33581,"children":33582},{"xmlns":194},[33583],{"type":150,"tag":197,"props":33584,"children":33585},{},[33586,33615],{"type":150,"tag":201,"props":33587,"children":33588},{},[33589,33600,33604],{"type":150,"tag":205,"props":33590,"children":33591},{},[33592,33596],{"type":150,"tag":209,"props":33593,"children":33594},{},[33595],{"type":155,"value":578},{"type":150,"tag":209,"props":33597,"children":33598},{},[33599],{"type":155,"value":1698},{"type":150,"tag":221,"props":33601,"children":33602},{},[33603],{"type":155,"value":5380},{"type":150,"tag":205,"props":33605,"children":33606},{},[33607,33611],{"type":150,"tag":209,"props":33608,"children":33609},{},[33610],{"type":155,"value":578},{"type":150,"tag":209,"props":33612,"children":33613},{},[33614],{"type":155,"value":1737},{"type":150,"tag":271,"props":33616,"children":33617},{"encoding":273},[33618],{"type":155,"value":33619},"d_k \\times d_v",{"type":150,"tag":181,"props":33621,"children":33623},{"className":33622,"ariaHidden":223},[280],[33624,33702],{"type":150,"tag":181,"props":33625,"children":33627},{"className":33626},[285],[33628,33632,33689,33693,33698],{"type":150,"tag":181,"props":33629,"children":33631},{"className":33630,"style":4761},[290],[],{"type":150,"tag":181,"props":33633,"children":33635},{"className":33634},[296],[33636,33641],{"type":150,"tag":181,"props":33637,"children":33639},{"className":33638},[296,301],[33640],{"type":155,"value":578},{"type":150,"tag":181,"props":33642,"children":33644},{"className":33643},[307],[33645],{"type":150,"tag":181,"props":33646,"children":33648},{"className":33647},[312,313],[33649,33678],{"type":150,"tag":181,"props":33650,"children":33652},{"className":33651},[318],[33653,33673],{"type":150,"tag":181,"props":33654,"children":33656},{"className":33655,"style":493},[323],[33657],{"type":150,"tag":181,"props":33658,"children":33659},{"style":328},[33660,33664],{"type":150,"tag":181,"props":33661,"children":33663},{"className":33662,"style":334},[333],[],{"type":150,"tag":181,"props":33665,"children":33667},{"className":33666},[339,340,341,342],[33668],{"type":150,"tag":181,"props":33669,"children":33671},{"className":33670,"style":2249},[296,301,342],[33672],{"type":155,"value":1698},{"type":150,"tag":181,"props":33674,"children":33676},{"className":33675},[352],[33677],{"type":155,"value":355},{"type":150,"tag":181,"props":33679,"children":33681},{"className":33680},[318],[33682],{"type":150,"tag":181,"props":33683,"children":33685},{"className":33684,"style":363},[323],[33686],{"type":150,"tag":181,"props":33687,"children":33688},{},[],{"type":150,"tag":181,"props":33690,"children":33692},{"className":33691,"style":4993},[377],[],{"type":150,"tag":181,"props":33694,"children":33696},{"className":33695},[4998],[33697],{"type":155,"value":5380},{"type":150,"tag":181,"props":33699,"children":33701},{"className":33700,"style":4993},[377],[],{"type":150,"tag":181,"props":33703,"children":33705},{"className":33704},[285],[33706,33710],{"type":150,"tag":181,"props":33707,"children":33709},{"className":33708,"style":4761},[290],[],{"type":150,"tag":181,"props":33711,"children":33713},{"className":33712},[296],[33714,33719],{"type":150,"tag":181,"props":33715,"children":33717},{"className":33716},[296,301],[33718],{"type":155,"value":578},{"type":150,"tag":181,"props":33720,"children":33722},{"className":33721},[307],[33723],{"type":150,"tag":181,"props":33724,"children":33726},{"className":33725},[312,313],[33727,33756],{"type":150,"tag":181,"props":33728,"children":33730},{"className":33729},[318],[33731,33751],{"type":150,"tag":181,"props":33732,"children":33734},{"className":33733,"style":4865},[323],[33735],{"type":150,"tag":181,"props":33736,"children":33737},{"style":328},[33738,33742],{"type":150,"tag":181,"props":33739,"children":33741},{"className":33740,"style":334},[333],[],{"type":150,"tag":181,"props":33743,"children":33745},{"className":33744},[339,340,341,342],[33746],{"type":150,"tag":181,"props":33747,"children":33749},{"className":33748,"style":2151},[296,301,342],[33750],{"type":155,"value":1737},{"type":150,"tag":181,"props":33752,"children":33754},{"className":33753},[352],[33755],{"type":155,"value":355},{"type":150,"tag":181,"props":33757,"children":33759},{"className":33758},[318],[33760],{"type":150,"tag":181,"props":33761,"children":33763},{"className":33762,"style":363},[323],[33764],{"type":150,"tag":181,"props":33765,"children":33766},{},[],{"type":155,"value":33768}," 同阶，不随 ",{"type":150,"tag":181,"props":33770,"children":33772},{"className":33771},[184],[33773,33794],{"type":150,"tag":181,"props":33774,"children":33776},{"className":33775},[189],[33777],{"type":150,"tag":192,"props":33778,"children":33779},{"xmlns":194},[33780],{"type":150,"tag":197,"props":33781,"children":33782},{},[33783,33790],{"type":150,"tag":201,"props":33784,"children":33785},{},[33786],{"type":150,"tag":209,"props":33787,"children":33788},{},[33789],{"type":155,"value":5417},{"type":150,"tag":271,"props":33791,"children":33792},{"encoding":273},[33793],{"type":155,"value":5417},{"type":150,"tag":181,"props":33795,"children":33797},{"className":33796,"ariaHidden":223},[280],[33798],{"type":150,"tag":181,"props":33799,"children":33801},{"className":33800},[285],[33802,33806],{"type":150,"tag":181,"props":33803,"children":33805},{"className":33804,"style":5680},[290],[],{"type":150,"tag":181,"props":33807,"children":33809},{"className":33808,"style":5606},[296,301],[33810],{"type":155,"value":5417},{"type":155,"value":33812}," 增长，而它要装的是全部历史的信息。序列越长，需要塞进去的东西越多，碰撞就越严重。具体表现是：想精确取回某一个早先 token 的信息时，取回来的其实是很多 token 叠加后的结果；full attention 能精确地只挑那一个，Linear Attention 做不到。门控能缓解遗忘，缓解不了容量上限造成的记混。",{"type":150,"tag":151,"props":33814,"children":33815},{},[33816,33818,33931,33933,34002],{"type":155,"value":33817},"SSM（State Space Model）是这条线的另一个分支，典型代表是 Mamba。它和 Linear Attention 的数学形式不同，但目标一样：用固定大小的状态代替完整历史，把 ",{"type":150,"tag":181,"props":33819,"children":33821},{"className":33820},[184],[33822,33863],{"type":150,"tag":181,"props":33823,"children":33825},{"className":33824},[189],[33826],{"type":150,"tag":192,"props":33827,"children":33828},{"xmlns":194},[33829],{"type":150,"tag":197,"props":33830,"children":33831},{},[33832,33858],{"type":150,"tag":201,"props":33833,"children":33834},{},[33835,33839,33843,33854],{"type":150,"tag":209,"props":33836,"children":33837},{},[33838],{"type":155,"value":6749},{"type":150,"tag":221,"props":33840,"children":33841},{"stretchy":789},[33842],{"type":155,"value":792},{"type":150,"tag":565,"props":33844,"children":33845},{},[33846,33850],{"type":150,"tag":209,"props":33847,"children":33848},{},[33849],{"type":155,"value":5417},{"type":150,"tag":215,"props":33851,"children":33852},{},[33853],{"type":155,"value":238},{"type":150,"tag":221,"props":33855,"children":33856},{"stretchy":789},[33857],{"type":155,"value":802},{"type":150,"tag":271,"props":33859,"children":33860},{"encoding":273},[33861],{"type":155,"value":33862},"O(N^2)",{"type":150,"tag":181,"props":33864,"children":33866},{"className":33865,"ariaHidden":223},[280],[33867],{"type":150,"tag":181,"props":33868,"children":33870},{"className":33869},[285],[33871,33875,33880,33885,33926],{"type":150,"tag":181,"props":33872,"children":33874},{"className":33873,"style":6789},[290],[],{"type":150,"tag":181,"props":33876,"children":33878},{"className":33877,"style":6794},[296,301],[33879],{"type":155,"value":6749},{"type":150,"tag":181,"props":33881,"children":33883},{"className":33882},[824],[33884],{"type":155,"value":792},{"type":150,"tag":181,"props":33886,"children":33888},{"className":33887},[296],[33889,33894],{"type":150,"tag":181,"props":33890,"children":33892},{"className":33891,"style":5606},[296,301],[33893],{"type":155,"value":5417},{"type":150,"tag":181,"props":33895,"children":33897},{"className":33896},[307],[33898],{"type":150,"tag":181,"props":33899,"children":33901},{"className":33900},[312],[33902],{"type":150,"tag":181,"props":33903,"children":33905},{"className":33904},[318],[33906],{"type":150,"tag":181,"props":33907,"children":33909},{"className":33908,"style":6831},[323],[33910],{"type":150,"tag":181,"props":33911,"children":33912},{"style":707},[33913,33917],{"type":150,"tag":181,"props":33914,"children":33916},{"className":33915,"style":334},[333],[],{"type":150,"tag":181,"props":33918,"children":33920},{"className":33919},[339,340,341,342],[33921],{"type":150,"tag":181,"props":33922,"children":33924},{"className":33923},[296,342],[33925],{"type":155,"value":238},{"type":150,"tag":181,"props":33927,"children":33929},{"className":33928},[835],[33930],{"type":155,"value":802},{"type":155,"value":33932}," 降到 ",{"type":150,"tag":181,"props":33934,"children":33936},{"className":33935},[184],[33937,33970],{"type":150,"tag":181,"props":33938,"children":33940},{"className":33939},[189],[33941],{"type":150,"tag":192,"props":33942,"children":33943},{"xmlns":194},[33944],{"type":150,"tag":197,"props":33945,"children":33946},{},[33947,33966],{"type":150,"tag":201,"props":33948,"children":33949},{},[33950,33954,33958,33962],{"type":150,"tag":209,"props":33951,"children":33952},{},[33953],{"type":155,"value":6749},{"type":150,"tag":221,"props":33955,"children":33956},{"stretchy":789},[33957],{"type":155,"value":792},{"type":150,"tag":209,"props":33959,"children":33960},{},[33961],{"type":155,"value":5417},{"type":150,"tag":221,"props":33963,"children":33964},{"stretchy":789},[33965],{"type":155,"value":802},{"type":150,"tag":271,"props":33967,"children":33968},{"encoding":273},[33969],{"type":155,"value":31857},{"type":150,"tag":181,"props":33971,"children":33973},{"className":33972,"ariaHidden":223},[280],[33974],{"type":150,"tag":181,"props":33975,"children":33977},{"className":33976},[285],[33978,33982,33987,33992,33997],{"type":150,"tag":181,"props":33979,"children":33981},{"className":33980,"style":819},[290],[],{"type":150,"tag":181,"props":33983,"children":33985},{"className":33984,"style":6794},[296,301],[33986],{"type":155,"value":6749},{"type":150,"tag":181,"props":33988,"children":33990},{"className":33989},[824],[33991],{"type":155,"value":792},{"type":150,"tag":181,"props":33993,"children":33995},{"className":33994,"style":5606},[296,301],[33996],{"type":155,"value":5417},{"type":150,"tag":181,"props":33998,"children":34000},{"className":33999},[835],[34001],{"type":155,"value":802},{"type":155,"value":34003},"。Mamba 的特点在状态更新的方式上。它让状态转移矩阵随输入变化，相当于给状态加了一层输入相关的门控。这和 LSTM 的门控思路一脉相承，只是换了个数学外衣。",{"type":150,"tag":151,"props":34005,"children":34006},{},[34007,34009,34014],{"type":155,"value":34008},"所以往上看，Mamba、RWKV、RetNet、DeltaNet、Gated DeltaNet、Kimi Linear，看起来像一堆论文名字动物园，其实都在回答同一个问题：",{"type":150,"tag":158,"props":34010,"children":34011},{},[34012],{"type":155,"value":34013},"怎么用固定大小的记忆，尽可能少地丢掉历史。",{"type":155,"value":34015}," 于是有了 Hybrid：大部分层用廉价的 state memory，少数几层保留对完整历史的随机访问。这不是\"RNN 又赢了\"：attention 的随机访问能力和 RNN 的固定状态成本各有好处，把两者揉到一起，比宗教战争式地二选一更有产出。",{"type":150,"tag":169,"props":34017,"children":34019},{"id":34018},"kimi-与-deepseek",[34020],{"type":155,"value":34021},"Kimi 与 DeepSeek",{"type":150,"tag":151,"props":34023,"children":34024},{},[34025,34027,34032],{"type":155,"value":34026},"Kimi 和 DeepSeek 都在压历史，但压完之后的处理方式不一样：",{"type":150,"tag":158,"props":34028,"children":34029},{},[34030],{"type":155,"value":34031},"Kimi 把历史压成 state，DeepSeek 压缩之后仍然保持可检索",{"type":155,"value":723},{"type":150,"tag":30436,"props":34034,"children":34036},{"id":34035},"state-machine-与-indexed-memory",[34037],{"type":155,"value":34038},"State Machine 与 Indexed Memory",{"type":150,"tag":151,"props":34040,"children":34041},{},[34042],{"type":155,"value":34043},"Kimi 的 Linear Attention 路线（KDA 那一支）是把历史递推成一个 state，查询时直接对 state 做操作：",{"type":150,"tag":34045,"props":34046,"children":34049},"pre",{"className":34047,"code":34048,"language":155,"meta":28,"style":28},"language-text shiki shiki-themes github-light github-dark","历史\n ↓\nrecurrent state\n ↓\nquery state\n",[34050],{"type":150,"tag":34051,"props":34052,"children":34053},"code",{"__ignoreMap":28},[34054,34064,34073,34082,34090],{"type":150,"tag":181,"props":34055,"children":34058},{"class":34056,"line":34057},"line",1,[34059],{"type":150,"tag":181,"props":34060,"children":34061},{},[34062],{"type":155,"value":34063},"历史\n",{"type":150,"tag":181,"props":34065,"children":34067},{"class":34056,"line":34066},2,[34068],{"type":150,"tag":181,"props":34069,"children":34070},{},[34071],{"type":155,"value":34072}," ↓\n",{"type":150,"tag":181,"props":34074,"children":34076},{"class":34056,"line":34075},3,[34077],{"type":150,"tag":181,"props":34078,"children":34079},{},[34080],{"type":155,"value":34081},"recurrent state\n",{"type":150,"tag":181,"props":34083,"children":34085},{"class":34056,"line":34084},4,[34086],{"type":150,"tag":181,"props":34087,"children":34088},{},[34089],{"type":155,"value":34072},{"type":150,"tag":181,"props":34091,"children":34093},{"class":34056,"line":34092},5,[34094],{"type":150,"tag":181,"props":34095,"children":34096},{},[34097],{"type":155,"value":34098},"query state\n",{"type":150,"tag":151,"props":34100,"children":34101},{},[34102],{"type":155,"value":34103},"DeepSeek 的路线是压缩，但压缩后的东西仍然带索引、可以被检索回来：",{"type":150,"tag":34045,"props":34105,"children":34107},{"className":34047,"code":34106,"language":155,"meta":28,"style":28},"历史\n ↓\ncompressed memory\n ↓\nindex \u002F retrieval\n ↓\nsparse attention\n",[34108],{"type":150,"tag":34051,"props":34109,"children":34110},{"__ignoreMap":28},[34111,34118,34125,34133,34140,34148,34156],{"type":150,"tag":181,"props":34112,"children":34113},{"class":34056,"line":34057},[34114],{"type":150,"tag":181,"props":34115,"children":34116},{},[34117],{"type":155,"value":34063},{"type":150,"tag":181,"props":34119,"children":34120},{"class":34056,"line":34066},[34121],{"type":150,"tag":181,"props":34122,"children":34123},{},[34124],{"type":155,"value":34072},{"type":150,"tag":181,"props":34126,"children":34127},{"class":34056,"line":34075},[34128],{"type":150,"tag":181,"props":34129,"children":34130},{},[34131],{"type":155,"value":34132},"compressed memory\n",{"type":150,"tag":181,"props":34134,"children":34135},{"class":34056,"line":34084},[34136],{"type":150,"tag":181,"props":34137,"children":34138},{},[34139],{"type":155,"value":34072},{"type":150,"tag":181,"props":34141,"children":34142},{"class":34056,"line":34092},[34143],{"type":150,"tag":181,"props":34144,"children":34145},{},[34146],{"type":155,"value":34147},"index \u002F retrieval\n",{"type":150,"tag":181,"props":34149,"children":34151},{"class":34056,"line":34150},6,[34152],{"type":150,"tag":181,"props":34153,"children":34154},{},[34155],{"type":155,"value":34072},{"type":150,"tag":181,"props":34157,"children":34159},{"class":34056,"line":34158},7,[34160],{"type":150,"tag":181,"props":34161,"children":34162},{},[34163],{"type":155,"value":34164},"sparse attention\n",{"type":150,"tag":151,"props":34166,"children":34167},{},[34168,34170,34175,34177,34182,34184,34189],{"type":155,"value":34169},"我把它们概括成 ",{"type":150,"tag":158,"props":34171,"children":34172},{},[34173],{"type":155,"value":34174},"State Machine vs. Indexed Memory",{"type":155,"value":34176},"。这比说\"Linear Attention vs Sparse Attention\"有解释力：后者的区别看起来只是稀疏程度不同，前者是两种根本不同的 memory 哲学，一个",{"type":150,"tag":158,"props":34178,"children":34179},{},[34180],{"type":155,"value":34181},"把历史变成状态",{"type":155,"value":34183},"，一个",{"type":150,"tag":158,"props":34185,"children":34186},{},[34187],{"type":155,"value":34188},"把历史变成可检索的索引",{"type":155,"value":34190},"。有一个细节能说明 Kimi 这边并不只是\"换了个 attention\"。",{"type":150,"tag":151,"props":34192,"children":34193},{},[34194],{"type":155,"value":34195},"K3 的技术报告把 refined training and data recipes、general\u002Fagentic\u002Fcoding RL、million-token agentic RL infrastructure、sandbox state，和 KDA、AttnRes、LatentMoE 并列成核心进展。注意这个列表的构成：一半是架构，一半是数据和基础设施。",{"type":150,"tag":151,"props":34197,"children":34198},{},[34199],{"type":155,"value":34200},"如果 KDA 真是能力跃迁的唯一来源，报告没必要把 RL infra 抬到同样的位置。",{"type":150,"tag":151,"props":34202,"children":34203},{},[34204,34206,34262,34264,34269,34271,34276],{"type":155,"value":34205},"反过来看，这也解释了 KDA 这类工作为什么重要。它让 ",{"type":150,"tag":181,"props":34207,"children":34209},{"className":34208},[184],[34210,34236],{"type":150,"tag":181,"props":34211,"children":34213},{"className":34212},[189],[34214],{"type":150,"tag":192,"props":34215,"children":34216},{"xmlns":194},[34217],{"type":150,"tag":197,"props":34218,"children":34219},{},[34220,34231],{"type":150,"tag":201,"props":34221,"children":34222},{},[34223,34227],{"type":150,"tag":215,"props":34224,"children":34225},{},[34226],{"type":155,"value":219},{"type":150,"tag":249,"props":34228,"children":34229},{},[34230],{"type":155,"value":5886},{"type":150,"tag":271,"props":34232,"children":34233},{"encoding":273},[34234],{"type":155,"value":34235},"1\\text{M}",{"type":150,"tag":181,"props":34237,"children":34239},{"className":34238,"ariaHidden":223},[280],[34240],{"type":150,"tag":181,"props":34241,"children":34243},{"className":34242},[285],[34244,34248,34253],{"type":150,"tag":181,"props":34245,"children":34247},{"className":34246,"style":5680},[290],[],{"type":150,"tag":181,"props":34249,"children":34251},{"className":34250},[296],[34252],{"type":155,"value":219},{"type":150,"tag":181,"props":34254,"children":34256},{"className":34255},[296,155],[34257],{"type":150,"tag":181,"props":34258,"children":34260},{"className":34259},[296],[34261],{"type":155,"value":5886},{"type":155,"value":34263}," context 和 long-horizon rollout 在计算上变得现实，于是以前喂不起的 RL trajectories 变得喂得起。架构在这里是",{"type":150,"tag":158,"props":34265,"children":34266},{},[34267],{"type":155,"value":34268},"开路的",{"type":155,"value":34270},"，不是",{"type":150,"tag":158,"props":34272,"children":34273},{},[34274],{"type":155,"value":34275},"发力的",{"type":155,"value":34277},"。DeepSeek 这边也一样，CSA\u002FHCA 让 1M context 变得足够便宜，价值主要落在这里。",{"type":150,"tag":151,"props":34279,"children":34280},{},[34281,34283,34352,34354,34433,34435,34440],{"type":155,"value":34282},"那这两条路线谁能赢？我觉得决胜点不在复杂度写 ",{"type":150,"tag":181,"props":34284,"children":34286},{"className":34285},[184],[34287,34320],{"type":150,"tag":181,"props":34288,"children":34290},{"className":34289},[189],[34291],{"type":150,"tag":192,"props":34292,"children":34293},{"xmlns":194},[34294],{"type":150,"tag":197,"props":34295,"children":34296},{},[34297,34316],{"type":150,"tag":201,"props":34298,"children":34299},{},[34300,34304,34308,34312],{"type":150,"tag":209,"props":34301,"children":34302},{},[34303],{"type":155,"value":6749},{"type":150,"tag":221,"props":34305,"children":34306},{"stretchy":789},[34307],{"type":155,"value":792},{"type":150,"tag":209,"props":34309,"children":34310},{},[34311],{"type":155,"value":5417},{"type":150,"tag":221,"props":34313,"children":34314},{"stretchy":789},[34315],{"type":155,"value":802},{"type":150,"tag":271,"props":34317,"children":34318},{"encoding":273},[34319],{"type":155,"value":31857},{"type":150,"tag":181,"props":34321,"children":34323},{"className":34322,"ariaHidden":223},[280],[34324],{"type":150,"tag":181,"props":34325,"children":34327},{"className":34326},[285],[34328,34332,34337,34342,34347],{"type":150,"tag":181,"props":34329,"children":34331},{"className":34330,"style":819},[290],[],{"type":150,"tag":181,"props":34333,"children":34335},{"className":34334,"style":6794},[296,301],[34336],{"type":155,"value":6749},{"type":150,"tag":181,"props":34338,"children":34340},{"className":34339},[824],[34341],{"type":155,"value":792},{"type":150,"tag":181,"props":34343,"children":34345},{"className":34344,"style":5606},[296,301],[34346],{"type":155,"value":5417},{"type":150,"tag":181,"props":34348,"children":34350},{"className":34349},[835],[34351],{"type":155,"value":802},{"type":155,"value":34353}," 还是 ",{"type":150,"tag":181,"props":34355,"children":34357},{"className":34356},[184],[34358,34396],{"type":150,"tag":181,"props":34359,"children":34361},{"className":34360},[189],[34362],{"type":150,"tag":192,"props":34363,"children":34364},{"xmlns":194},[34365],{"type":150,"tag":197,"props":34366,"children":34367},{},[34368,34391],{"type":150,"tag":201,"props":34369,"children":34370},{},[34371,34375,34379,34383,34387],{"type":150,"tag":209,"props":34372,"children":34373},{},[34374],{"type":155,"value":6749},{"type":150,"tag":221,"props":34376,"children":34377},{"stretchy":789},[34378],{"type":155,"value":792},{"type":150,"tag":209,"props":34380,"children":34381},{},[34382],{"type":155,"value":5417},{"type":150,"tag":209,"props":34384,"children":34385},{},[34386],{"type":155,"value":23188},{"type":150,"tag":221,"props":34388,"children":34389},{"stretchy":789},[34390],{"type":155,"value":802},{"type":150,"tag":271,"props":34392,"children":34393},{"encoding":273},[34394],{"type":155,"value":34395},"O(NB)",{"type":150,"tag":181,"props":34397,"children":34399},{"className":34398,"ariaHidden":223},[280],[34400],{"type":150,"tag":181,"props":34401,"children":34403},{"className":34402},[285],[34404,34408,34413,34418,34423,34428],{"type":150,"tag":181,"props":34405,"children":34407},{"className":34406,"style":819},[290],[],{"type":150,"tag":181,"props":34409,"children":34411},{"className":34410,"style":6794},[296,301],[34412],{"type":155,"value":6749},{"type":150,"tag":181,"props":34414,"children":34416},{"className":34415},[824],[34417],{"type":155,"value":792},{"type":150,"tag":181,"props":34419,"children":34421},{"className":34420,"style":5606},[296,301],[34422],{"type":155,"value":5417},{"type":150,"tag":181,"props":34424,"children":34426},{"className":34425,"style":27072},[296,301],[34427],{"type":155,"value":23188},{"type":150,"tag":181,"props":34429,"children":34431},{"className":34430},[835],[34432],{"type":155,"value":802},{"type":155,"value":34434},"，而在更朴素的地方：",{"type":150,"tag":158,"props":34436,"children":34437},{},[34438],{"type":155,"value":34439},"同样的显存带宽和 FLOPs 下，谁还能准确找回 50 万 token 前那个关键细节",{"type":155,"value":34441},"。这是长上下文模型最难在 benchmark 上糊弄过去的地方。",{"type":150,"tag":169,"props":34443,"children":34445},{"id":34444},"l-n-d",[34446],{"type":155,"value":34447},"L × N × D",{"type":150,"tag":151,"props":34449,"children":34450},{},[34451,34453,34565,34566,34608,34609,34651,34653,34695,34697,34739,34741,34783,34785,34827],{"type":155,"value":34452},"把前面的方案合起来看，它们动的其实是同一个三维对象，也就是 KV 的形状 ",{"type":150,"tag":181,"props":34454,"children":34456},{"className":34455},[184],[34457,34496],{"type":150,"tag":181,"props":34458,"children":34460},{"className":34459},[189],[34461],{"type":150,"tag":192,"props":34462,"children":34463},{"xmlns":194},[34464],{"type":150,"tag":197,"props":34465,"children":34466},{},[34467,34491],{"type":150,"tag":201,"props":34468,"children":34469},{},[34470,34474,34478,34482,34486],{"type":150,"tag":209,"props":34471,"children":34472},{},[34473],{"type":155,"value":5385},{"type":150,"tag":221,"props":34475,"children":34476},{},[34477],{"type":155,"value":5380},{"type":150,"tag":209,"props":34479,"children":34480},{},[34481],{"type":155,"value":5417},{"type":150,"tag":221,"props":34483,"children":34484},{},[34485],{"type":155,"value":5380},{"type":150,"tag":209,"props":34487,"children":34488},{},[34489],{"type":155,"value":34490},"D",{"type":150,"tag":271,"props":34492,"children":34493},{"encoding":273},[34494],{"type":155,"value":34495},"L \\times N \\times D",{"type":150,"tag":181,"props":34497,"children":34499},{"className":34498,"ariaHidden":223},[280],[34500,34526,34552],{"type":150,"tag":181,"props":34501,"children":34503},{"className":34502},[285],[34504,34508,34513,34517,34522],{"type":150,"tag":181,"props":34505,"children":34507},{"className":34506,"style":5470},[290],[],{"type":150,"tag":181,"props":34509,"children":34511},{"className":34510},[296,301],[34512],{"type":155,"value":5385},{"type":150,"tag":181,"props":34514,"children":34516},{"className":34515,"style":4993},[377],[],{"type":150,"tag":181,"props":34518,"children":34520},{"className":34519},[4998],[34521],{"type":155,"value":5380},{"type":150,"tag":181,"props":34523,"children":34525},{"className":34524,"style":4993},[377],[],{"type":150,"tag":181,"props":34527,"children":34529},{"className":34528},[285],[34530,34534,34539,34543,34548],{"type":150,"tag":181,"props":34531,"children":34533},{"className":34532,"style":5470},[290],[],{"type":150,"tag":181,"props":34535,"children":34537},{"className":34536,"style":5606},[296,301],[34538],{"type":155,"value":5417},{"type":150,"tag":181,"props":34540,"children":34542},{"className":34541,"style":4993},[377],[],{"type":150,"tag":181,"props":34544,"children":34546},{"className":34545},[4998],[34547],{"type":155,"value":5380},{"type":150,"tag":181,"props":34549,"children":34551},{"className":34550,"style":4993},[377],[],{"type":150,"tag":181,"props":34553,"children":34555},{"className":34554},[285],[34556,34560],{"type":150,"tag":181,"props":34557,"children":34559},{"className":34558,"style":5680},[290],[],{"type":150,"tag":181,"props":34561,"children":34563},{"className":34562,"style":6794},[296,301],[34564],{"type":155,"value":34490},{"type":155,"value":11152},{"type":150,"tag":181,"props":34567,"children":34569},{"className":34568},[184],[34570,34591],{"type":150,"tag":181,"props":34571,"children":34573},{"className":34572},[189],[34574],{"type":150,"tag":192,"props":34575,"children":34576},{"xmlns":194},[34577],{"type":150,"tag":197,"props":34578,"children":34579},{},[34580,34587],{"type":150,"tag":201,"props":34581,"children":34582},{},[34583],{"type":150,"tag":209,"props":34584,"children":34585},{},[34586],{"type":155,"value":5385},{"type":150,"tag":271,"props":34588,"children":34589},{"encoding":273},[34590],{"type":155,"value":5385},{"type":150,"tag":181,"props":34592,"children":34594},{"className":34593,"ariaHidden":223},[280],[34595],{"type":150,"tag":181,"props":34596,"children":34598},{"className":34597},[285],[34599,34603],{"type":150,"tag":181,"props":34600,"children":34602},{"className":34601,"style":5680},[290],[],{"type":150,"tag":181,"props":34604,"children":34606},{"className":34605},[296,301],[34607],{"type":155,"value":5385},{"type":155,"value":5688},{"type":150,"tag":181,"props":34610,"children":34612},{"className":34611},[184],[34613,34634],{"type":150,"tag":181,"props":34614,"children":34616},{"className":34615},[189],[34617],{"type":150,"tag":192,"props":34618,"children":34619},{"xmlns":194},[34620],{"type":150,"tag":197,"props":34621,"children":34622},{},[34623,34630],{"type":150,"tag":201,"props":34624,"children":34625},{},[34626],{"type":150,"tag":209,"props":34627,"children":34628},{},[34629],{"type":155,"value":5417},{"type":150,"tag":271,"props":34631,"children":34632},{"encoding":273},[34633],{"type":155,"value":5417},{"type":150,"tag":181,"props":34635,"children":34637},{"className":34636,"ariaHidden":223},[280],[34638],{"type":150,"tag":181,"props":34639,"children":34641},{"className":34640},[285],[34642,34646],{"type":150,"tag":181,"props":34643,"children":34645},{"className":34644,"style":5680},[290],[],{"type":150,"tag":181,"props":34647,"children":34649},{"className":34648,"style":5606},[296,301],[34650],{"type":155,"value":5417},{"type":155,"value":34652}," 是序列长度，",{"type":150,"tag":181,"props":34654,"children":34656},{"className":34655},[184],[34657,34678],{"type":150,"tag":181,"props":34658,"children":34660},{"className":34659},[189],[34661],{"type":150,"tag":192,"props":34662,"children":34663},{"xmlns":194},[34664],{"type":150,"tag":197,"props":34665,"children":34666},{},[34667,34674],{"type":150,"tag":201,"props":34668,"children":34669},{},[34670],{"type":150,"tag":209,"props":34671,"children":34672},{},[34673],{"type":155,"value":34490},{"type":150,"tag":271,"props":34675,"children":34676},{"encoding":273},[34677],{"type":155,"value":34490},{"type":150,"tag":181,"props":34679,"children":34681},{"className":34680,"ariaHidden":223},[280],[34682],{"type":150,"tag":181,"props":34683,"children":34685},{"className":34684},[285],[34686,34690],{"type":150,"tag":181,"props":34687,"children":34689},{"className":34688,"style":5680},[290],[],{"type":150,"tag":181,"props":34691,"children":34693},{"className":34692,"style":6794},[296,301],[34694],{"type":155,"value":34490},{"type":155,"value":34696}," 是每个 token 存下来的表示宽度。整条 DeepSeek 路线可以重新写成：GQA\u002FMLA 压 ",{"type":150,"tag":181,"props":34698,"children":34700},{"className":34699},[184],[34701,34722],{"type":150,"tag":181,"props":34702,"children":34704},{"className":34703},[189],[34705],{"type":150,"tag":192,"props":34706,"children":34707},{"xmlns":194},[34708],{"type":150,"tag":197,"props":34709,"children":34710},{},[34711,34718],{"type":150,"tag":201,"props":34712,"children":34713},{},[34714],{"type":150,"tag":209,"props":34715,"children":34716},{},[34717],{"type":155,"value":34490},{"type":150,"tag":271,"props":34719,"children":34720},{"encoding":273},[34721],{"type":155,"value":34490},{"type":150,"tag":181,"props":34723,"children":34725},{"className":34724,"ariaHidden":223},[280],[34726],{"type":150,"tag":181,"props":34727,"children":34729},{"className":34728},[285],[34730,34734],{"type":150,"tag":181,"props":34731,"children":34733},{"className":34732,"style":5680},[290],[],{"type":150,"tag":181,"props":34735,"children":34737},{"className":34736,"style":6794},[296,301],[34738],{"type":155,"value":34490},{"type":155,"value":34740},"，V4 的 CSA\u002FHCA 压 ",{"type":150,"tag":181,"props":34742,"children":34744},{"className":34743},[184],[34745,34766],{"type":150,"tag":181,"props":34746,"children":34748},{"className":34747},[189],[34749],{"type":150,"tag":192,"props":34750,"children":34751},{"xmlns":194},[34752],{"type":150,"tag":197,"props":34753,"children":34754},{},[34755,34762],{"type":150,"tag":201,"props":34756,"children":34757},{},[34758],{"type":150,"tag":209,"props":34759,"children":34760},{},[34761],{"type":155,"value":5417},{"type":150,"tag":271,"props":34763,"children":34764},{"encoding":273},[34765],{"type":155,"value":5417},{"type":150,"tag":181,"props":34767,"children":34769},{"className":34768,"ariaHidden":223},[280],[34770],{"type":150,"tag":181,"props":34771,"children":34773},{"className":34772},[285],[34774,34778],{"type":150,"tag":181,"props":34775,"children":34777},{"className":34776,"style":5680},[290],[],{"type":150,"tag":181,"props":34779,"children":34781},{"className":34780,"style":5606},[296,301],[34782],{"type":155,"value":5417},{"type":155,"value":34784},"，V4.1 压 ",{"type":150,"tag":181,"props":34786,"children":34788},{"className":34787},[184],[34789,34810],{"type":150,"tag":181,"props":34790,"children":34792},{"className":34791},[189],[34793],{"type":150,"tag":192,"props":34794,"children":34795},{"xmlns":194},[34796],{"type":150,"tag":197,"props":34797,"children":34798},{},[34799,34806],{"type":150,"tag":201,"props":34800,"children":34801},{},[34802],{"type":150,"tag":209,"props":34803,"children":34804},{},[34805],{"type":155,"value":5385},{"type":150,"tag":271,"props":34807,"children":34808},{"encoding":273},[34809],{"type":155,"value":5385},{"type":150,"tag":181,"props":34811,"children":34813},{"className":34812,"ariaHidden":223},[280],[34814],{"type":150,"tag":181,"props":34815,"children":34817},{"className":34816},[285],[34818,34822],{"type":150,"tag":181,"props":34819,"children":34821},{"className":34820,"style":5680},[290],[],{"type":150,"tag":181,"props":34823,"children":34825},{"className":34824},[296,301],[34826],{"type":155,"value":5385},{"type":155,"value":34828}," 和计算路径。到 V4.1，问题变成了：",{"type":150,"tag":151,"props":34830,"children":34831},{},[34832],{"type":150,"tag":158,"props":34833,"children":34834},{},[34835],{"type":155,"value":34836},"既然历史已经压缩过一次，为什么每一层还要重复为它付费？",{"type":150,"tag":151,"props":34838,"children":34839},{},[34840,34842,34884,34886,34928,34930,34972,34974,34979],{"type":155,"value":34841},"ds4.1f 采用的是一个新架构族：552B MoE，Causal-Encoder-Decoder，输入侧约 8B activated、输出侧约 16B activated。这个不对称的计算预算是关键。要理解为什么 8B 和 16B 是分开的两个数，得把那两个瓶颈各自的账算一遍。先定符号。",{"type":150,"tag":181,"props":34843,"children":34845},{"className":34844},[184],[34846,34867],{"type":150,"tag":181,"props":34847,"children":34849},{"className":34848},[189],[34850],{"type":150,"tag":192,"props":34851,"children":34852},{"xmlns":194},[34853],{"type":150,"tag":197,"props":34854,"children":34855},{},[34856,34863],{"type":150,"tag":201,"props":34857,"children":34858},{},[34859],{"type":150,"tag":209,"props":34860,"children":34861},{},[34862],{"type":155,"value":25017},{"type":150,"tag":271,"props":34864,"children":34865},{"encoding":273},[34866],{"type":155,"value":25017},{"type":150,"tag":181,"props":34868,"children":34870},{"className":34869,"ariaHidden":223},[280],[34871],{"type":150,"tag":181,"props":34872,"children":34874},{"className":34873},[285],[34875,34879],{"type":150,"tag":181,"props":34876,"children":34878},{"className":34877,"style":5680},[290],[],{"type":150,"tag":181,"props":34880,"children":34882},{"className":34881,"style":3691},[296,301],[34883],{"type":155,"value":25017},{"type":155,"value":34885}," 是参数量（MoE 里指 activated 部分），",{"type":150,"tag":181,"props":34887,"children":34889},{"className":34888},[184],[34890,34911],{"type":150,"tag":181,"props":34891,"children":34893},{"className":34892},[189],[34894],{"type":150,"tag":192,"props":34895,"children":34896},{"xmlns":194},[34897],{"type":150,"tag":197,"props":34898,"children":34899},{},[34900,34907],{"type":150,"tag":201,"props":34901,"children":34902},{},[34903],{"type":150,"tag":209,"props":34904,"children":34905},{},[34906],{"type":155,"value":22069},{"type":150,"tag":271,"props":34908,"children":34909},{"encoding":273},[34910],{"type":155,"value":22069},{"type":150,"tag":181,"props":34912,"children":34914},{"className":34913,"ariaHidden":223},[280],[34915],{"type":150,"tag":181,"props":34916,"children":34918},{"className":34917},[285],[34919,34923],{"type":150,"tag":181,"props":34920,"children":34922},{"className":34921,"style":761},[290],[],{"type":150,"tag":181,"props":34924,"children":34926},{"className":34925},[296,301],[34927],{"type":155,"value":22069},{"type":155,"value":34929}," 是每个参数的字节数，",{"type":150,"tag":181,"props":34931,"children":34933},{"className":34932},[184],[34934,34955],{"type":150,"tag":181,"props":34935,"children":34937},{"className":34936},[189],[34938],{"type":150,"tag":192,"props":34939,"children":34940},{"xmlns":194},[34941],{"type":150,"tag":197,"props":34942,"children":34943},{},[34944,34951],{"type":150,"tag":201,"props":34945,"children":34946},{},[34947],{"type":150,"tag":209,"props":34948,"children":34949},{},[34950],{"type":155,"value":5417},{"type":150,"tag":271,"props":34952,"children":34953},{"encoding":273},[34954],{"type":155,"value":5417},{"type":150,"tag":181,"props":34956,"children":34958},{"className":34957,"ariaHidden":223},[280],[34959],{"type":150,"tag":181,"props":34960,"children":34962},{"className":34961},[285],[34963,34967],{"type":150,"tag":181,"props":34964,"children":34966},{"className":34965,"style":5680},[290],[],{"type":150,"tag":181,"props":34968,"children":34970},{"className":34969,"style":5606},[296,301],[34971],{"type":155,"value":5417},{"type":155,"value":34973}," 是序列长度。",{"type":150,"tag":158,"props":34975,"children":34976},{},[34977],{"type":155,"value":34978},"输入侧",{"type":155,"value":34980},"面对的是长 prompt。",{"type":150,"tag":151,"props":34982,"children":34983},{},[34984,34986,35047,35049,35126],{"type":155,"value":34985},"1M 个 token 全部要走一遍编码器，计算量约 ",{"type":150,"tag":181,"props":34987,"children":34989},{"className":34988},[184],[34990,35020],{"type":150,"tag":181,"props":34991,"children":34993},{"className":34992},[189],[34994],{"type":150,"tag":192,"props":34995,"children":34996},{"xmlns":194},[34997],{"type":150,"tag":197,"props":34998,"children":34999},{},[35000,35015],{"type":150,"tag":201,"props":35001,"children":35002},{},[35003,35007,35011],{"type":150,"tag":215,"props":35004,"children":35005},{},[35006],{"type":155,"value":238},{"type":150,"tag":209,"props":35008,"children":35009},{},[35010],{"type":155,"value":25017},{"type":150,"tag":209,"props":35012,"children":35013},{},[35014],{"type":155,"value":5417},{"type":150,"tag":271,"props":35016,"children":35017},{"encoding":273},[35018],{"type":155,"value":35019},"2PN",{"type":150,"tag":181,"props":35021,"children":35023},{"className":35022,"ariaHidden":223},[280],[35024],{"type":150,"tag":181,"props":35025,"children":35027},{"className":35026},[285],[35028,35032,35037,35042],{"type":150,"tag":181,"props":35029,"children":35031},{"className":35030,"style":5680},[290],[],{"type":150,"tag":181,"props":35033,"children":35035},{"className":35034},[296],[35036],{"type":155,"value":238},{"type":150,"tag":181,"props":35038,"children":35040},{"className":35039,"style":3691},[296,301],[35041],{"type":155,"value":25017},{"type":150,"tag":181,"props":35043,"children":35045},{"className":35044,"style":5606},[296,301],[35046],{"type":155,"value":5417},{"type":155,"value":35048},"，而要读的权重只有 ",{"type":150,"tag":181,"props":35050,"children":35052},{"className":35051},[184],[35053,35083],{"type":150,"tag":181,"props":35054,"children":35056},{"className":35055},[189],[35057],{"type":150,"tag":192,"props":35058,"children":35059},{"xmlns":194},[35060],{"type":150,"tag":197,"props":35061,"children":35062},{},[35063,35078],{"type":150,"tag":201,"props":35064,"children":35065},{},[35066,35070,35074],{"type":150,"tag":209,"props":35067,"children":35068},{},[35069],{"type":155,"value":25017},{"type":150,"tag":221,"props":35071,"children":35072},{},[35073],{"type":155,"value":5380},{"type":150,"tag":209,"props":35075,"children":35076},{},[35077],{"type":155,"value":22069},{"type":150,"tag":271,"props":35079,"children":35080},{"encoding":273},[35081],{"type":155,"value":35082},"P \\times b",{"type":150,"tag":181,"props":35084,"children":35086},{"className":35085,"ariaHidden":223},[280],[35087,35113],{"type":150,"tag":181,"props":35088,"children":35090},{"className":35089},[285],[35091,35095,35100,35104,35109],{"type":150,"tag":181,"props":35092,"children":35094},{"className":35093,"style":5470},[290],[],{"type":150,"tag":181,"props":35096,"children":35098},{"className":35097,"style":3691},[296,301],[35099],{"type":155,"value":25017},{"type":150,"tag":181,"props":35101,"children":35103},{"className":35102,"style":4993},[377],[],{"type":150,"tag":181,"props":35105,"children":35107},{"className":35106},[4998],[35108],{"type":155,"value":5380},{"type":150,"tag":181,"props":35110,"children":35112},{"className":35111,"style":4993},[377],[],{"type":150,"tag":181,"props":35114,"children":35116},{"className":35115},[285],[35117,35121],{"type":150,"tag":181,"props":35118,"children":35120},{"className":35119,"style":761},[290],[],{"type":150,"tag":181,"props":35122,"children":35124},{"className":35123},[296,301],[35125],{"type":155,"value":22069},{"type":155,"value":723},{"type":150,"tag":151,"props":35128,"children":35129},{},[35130,35172,35174,35216,35218,35223,35225,35230,35232,35283,35285,35361],{"type":150,"tag":181,"props":35131,"children":35133},{"className":35132},[184],[35134,35155],{"type":150,"tag":181,"props":35135,"children":35137},{"className":35136},[189],[35138],{"type":150,"tag":192,"props":35139,"children":35140},{"xmlns":194},[35141],{"type":150,"tag":197,"props":35142,"children":35143},{},[35144,35151],{"type":150,"tag":201,"props":35145,"children":35146},{},[35147],{"type":150,"tag":209,"props":35148,"children":35149},{},[35150],{"type":155,"value":5417},{"type":150,"tag":271,"props":35152,"children":35153},{"encoding":273},[35154],{"type":155,"value":5417},{"type":150,"tag":181,"props":35156,"children":35158},{"className":35157,"ariaHidden":223},[280],[35159],{"type":150,"tag":181,"props":35160,"children":35162},{"className":35161},[285],[35163,35167],{"type":150,"tag":181,"props":35164,"children":35166},{"className":35165,"style":5680},[290],[],{"type":150,"tag":181,"props":35168,"children":35170},{"className":35169,"style":5606},[296,301],[35171],{"type":155,"value":5417},{"type":155,"value":35173}," 到百万量级时，算术强度能到几千 FLOPs\u002Fbyte，早就超过了硬件的算力带宽比。这一侧完全被算力卡住。想在 prefill 上省钱，唯一有效的办法就是把 ",{"type":150,"tag":181,"props":35175,"children":35177},{"className":35176},[184],[35178,35199],{"type":150,"tag":181,"props":35179,"children":35181},{"className":35180},[189],[35182],{"type":150,"tag":192,"props":35183,"children":35184},{"xmlns":194},[35185],{"type":150,"tag":197,"props":35186,"children":35187},{},[35188,35195],{"type":150,"tag":201,"props":35189,"children":35190},{},[35191],{"type":150,"tag":209,"props":35192,"children":35193},{},[35194],{"type":155,"value":25017},{"type":150,"tag":271,"props":35196,"children":35197},{"encoding":273},[35198],{"type":155,"value":25017},{"type":150,"tag":181,"props":35200,"children":35202},{"className":35201,"ariaHidden":223},[280],[35203],{"type":150,"tag":181,"props":35204,"children":35206},{"className":35205},[285],[35207,35211],{"type":150,"tag":181,"props":35208,"children":35210},{"className":35209,"style":5680},[290],[],{"type":150,"tag":181,"props":35212,"children":35214},{"className":35213,"style":3691},[296,301],[35215],{"type":155,"value":25017},{"type":155,"value":35217}," 压下去。",{"type":150,"tag":158,"props":35219,"children":35220},{},[35221],{"type":155,"value":35222},"8B 是这么来的。",{"type":155,"value":35224}," ",{"type":150,"tag":158,"props":35226,"children":35227},{},[35228],{"type":155,"value":35229},"输出侧",{"type":155,"value":35231},"面对的是逐个生成的 token，情况正好反过来。每生成一个 token，计算量只有 ",{"type":150,"tag":181,"props":35233,"children":35235},{"className":35234},[184],[35236,35261],{"type":150,"tag":181,"props":35237,"children":35239},{"className":35238},[189],[35240],{"type":150,"tag":192,"props":35241,"children":35242},{"xmlns":194},[35243],{"type":150,"tag":197,"props":35244,"children":35245},{},[35246,35257],{"type":150,"tag":201,"props":35247,"children":35248},{},[35249,35253],{"type":150,"tag":215,"props":35250,"children":35251},{},[35252],{"type":155,"value":238},{"type":150,"tag":209,"props":35254,"children":35255},{},[35256],{"type":155,"value":25017},{"type":150,"tag":271,"props":35258,"children":35259},{"encoding":273},[35260],{"type":155,"value":25502},{"type":150,"tag":181,"props":35262,"children":35264},{"className":35263,"ariaHidden":223},[280],[35265],{"type":150,"tag":181,"props":35266,"children":35268},{"className":35267},[285],[35269,35273,35278],{"type":150,"tag":181,"props":35270,"children":35272},{"className":35271,"style":5680},[290],[],{"type":150,"tag":181,"props":35274,"children":35276},{"className":35275},[296],[35277],{"type":155,"value":238},{"type":150,"tag":181,"props":35279,"children":35281},{"className":35280,"style":3691},[296,301],[35282],{"type":155,"value":25017},{"type":155,"value":35284},"，但要读两份东西：模型权重 ",{"type":150,"tag":181,"props":35286,"children":35288},{"className":35287},[184],[35289,35318],{"type":150,"tag":181,"props":35290,"children":35292},{"className":35291},[189],[35293],{"type":150,"tag":192,"props":35294,"children":35295},{"xmlns":194},[35296],{"type":150,"tag":197,"props":35297,"children":35298},{},[35299,35314],{"type":150,"tag":201,"props":35300,"children":35301},{},[35302,35306,35310],{"type":150,"tag":209,"props":35303,"children":35304},{},[35305],{"type":155,"value":25017},{"type":150,"tag":221,"props":35307,"children":35308},{},[35309],{"type":155,"value":5380},{"type":150,"tag":209,"props":35311,"children":35312},{},[35313],{"type":155,"value":22069},{"type":150,"tag":271,"props":35315,"children":35316},{"encoding":273},[35317],{"type":155,"value":35082},{"type":150,"tag":181,"props":35319,"children":35321},{"className":35320,"ariaHidden":223},[280],[35322,35348],{"type":150,"tag":181,"props":35323,"children":35325},{"className":35324},[285],[35326,35330,35335,35339,35344],{"type":150,"tag":181,"props":35327,"children":35329},{"className":35328,"style":5470},[290],[],{"type":150,"tag":181,"props":35331,"children":35333},{"className":35332,"style":3691},[296,301],[35334],{"type":155,"value":25017},{"type":150,"tag":181,"props":35336,"children":35338},{"className":35337,"style":4993},[377],[],{"type":150,"tag":181,"props":35340,"children":35342},{"className":35341},[4998],[35343],{"type":155,"value":5380},{"type":150,"tag":181,"props":35345,"children":35347},{"className":35346,"style":4993},[377],[],{"type":150,"tag":181,"props":35349,"children":35351},{"className":35350},[285],[35352,35356],{"type":150,"tag":181,"props":35353,"children":35355},{"className":35354,"style":761},[290],[],{"type":150,"tag":181,"props":35357,"children":35359},{"className":35358},[296,301],[35360],{"type":155,"value":22069},{"type":155,"value":35362},"，加上 KV Cache。而模型权重是必须整体读一遍的。",{"type":150,"tag":151,"props":35364,"children":35365},{},[35366],{"type":155,"value":35367},"拿 16B activated、BF16 存储来算：",{"type":150,"tag":181,"props":35369,"children":35371},{"className":35370},[1003],[35372],{"type":150,"tag":181,"props":35373,"children":35375},{"className":35374},[184],[35376,35429],{"type":150,"tag":181,"props":35377,"children":35379},{"className":35378},[189],[35380],{"type":150,"tag":192,"props":35381,"children":35382},{"xmlns":194,"display":1015},[35383],{"type":150,"tag":197,"props":35384,"children":35385},{},[35386,35424],{"type":150,"tag":201,"props":35387,"children":35388},{},[35389,35394,35398,35402,35406,35411,35415,35420],{"type":150,"tag":215,"props":35390,"children":35391},{},[35392],{"type":155,"value":35393},"16",{"type":150,"tag":249,"props":35395,"children":35396},{},[35397],{"type":155,"value":23188},{"type":150,"tag":221,"props":35399,"children":35400},{},[35401],{"type":155,"value":5380},{"type":150,"tag":215,"props":35403,"children":35404},{},[35405],{"type":155,"value":238},{"type":150,"tag":249,"props":35407,"children":35408},{},[35409],{"type":155,"value":35410}," bytes",{"type":150,"tag":221,"props":35412,"children":35413},{},[35414],{"type":155,"value":865},{"type":150,"tag":215,"props":35416,"children":35417},{},[35418],{"type":155,"value":35419},"32",{"type":150,"tag":249,"props":35421,"children":35422},{},[35423],{"type":155,"value":29293},{"type":150,"tag":271,"props":35425,"children":35426},{"encoding":273},[35427],{"type":155,"value":35428},"16\\text{B} \\times 2\\ \\text{bytes} = 32\\ \\text{GB}",{"type":150,"tag":181,"props":35430,"children":35432},{"className":35431,"ariaHidden":223},[280],[35433,35468,35508],{"type":150,"tag":181,"props":35434,"children":35436},{"className":35435},[285],[35437,35441,35446,35455,35459,35464],{"type":150,"tag":181,"props":35438,"children":35440},{"className":35439,"style":5470},[290],[],{"type":150,"tag":181,"props":35442,"children":35444},{"className":35443},[296],[35445],{"type":155,"value":35393},{"type":150,"tag":181,"props":35447,"children":35449},{"className":35448},[296,155],[35450],{"type":150,"tag":181,"props":35451,"children":35453},{"className":35452},[296],[35454],{"type":155,"value":23188},{"type":150,"tag":181,"props":35456,"children":35458},{"className":35457,"style":4993},[377],[],{"type":150,"tag":181,"props":35460,"children":35462},{"className":35461},[4998],[35463],{"type":155,"value":5380},{"type":150,"tag":181,"props":35465,"children":35467},{"className":35466,"style":4993},[377],[],{"type":150,"tag":181,"props":35469,"children":35471},{"className":35470},[285],[35472,35476,35481,35486,35495,35499,35504],{"type":150,"tag":181,"props":35473,"children":35475},{"className":35474,"style":941},[290],[],{"type":150,"tag":181,"props":35477,"children":35479},{"className":35478},[296],[35480],{"type":155,"value":238},{"type":150,"tag":181,"props":35482,"children":35484},{"className":35483},[377],[35485],{"type":155,"value":8576},{"type":150,"tag":181,"props":35487,"children":35489},{"className":35488},[296,155],[35490],{"type":150,"tag":181,"props":35491,"children":35493},{"className":35492},[296],[35494],{"type":155,"value":5426},{"type":150,"tag":181,"props":35496,"children":35498},{"className":35497,"style":658},[377],[],{"type":150,"tag":181,"props":35500,"children":35502},{"className":35501},[663],[35503],{"type":155,"value":865},{"type":150,"tag":181,"props":35505,"children":35507},{"className":35506,"style":658},[377],[],{"type":150,"tag":181,"props":35509,"children":35511},{"className":35510},[285],[35512,35516,35521,35526],{"type":150,"tag":181,"props":35513,"children":35515},{"className":35514,"style":5680},[290],[],{"type":150,"tag":181,"props":35517,"children":35519},{"className":35518},[296],[35520],{"type":155,"value":35419},{"type":150,"tag":181,"props":35522,"children":35524},{"className":35523},[377],[35525],{"type":155,"value":8576},{"type":150,"tag":181,"props":35527,"children":35529},{"className":35528},[296,155],[35530],{"type":150,"tag":181,"props":35531,"children":35533},{"className":35532},[296],[35534],{"type":155,"value":29439},{"type":150,"tag":151,"props":35536,"children":35537},{},[35538,35540,35664,35666,35708,35710],{"type":155,"value":35539},"每生成一个 token，光权重就要搬 32 GB。而这一个 token 的计算量是 ",{"type":150,"tag":181,"props":35541,"children":35543},{"className":35542},[184],[35544,35586],{"type":150,"tag":181,"props":35545,"children":35547},{"className":35546},[189],[35548],{"type":150,"tag":192,"props":35549,"children":35550},{"xmlns":194},[35551],{"type":150,"tag":197,"props":35552,"children":35553},{},[35554,35581],{"type":150,"tag":201,"props":35555,"children":35556},{},[35557,35561,35565,35569,35573,35577],{"type":150,"tag":215,"props":35558,"children":35559},{},[35560],{"type":155,"value":238},{"type":150,"tag":221,"props":35562,"children":35563},{},[35564],{"type":155,"value":5380},{"type":150,"tag":215,"props":35566,"children":35567},{},[35568],{"type":155,"value":35393},{"type":150,"tag":249,"props":35570,"children":35571},{},[35572],{"type":155,"value":23188},{"type":150,"tag":221,"props":35574,"children":35575},{},[35576],{"type":155,"value":865},{"type":150,"tag":215,"props":35578,"children":35579},{},[35580],{"type":155,"value":35419},{"type":150,"tag":271,"props":35582,"children":35583},{"encoding":273},[35584],{"type":155,"value":35585},"2 \\times 16\\text{B} = 32",{"type":150,"tag":181,"props":35587,"children":35589},{"className":35588,"ariaHidden":223},[280],[35590,35616,35651],{"type":150,"tag":181,"props":35591,"children":35593},{"className":35592},[285],[35594,35598,35603,35607,35612],{"type":150,"tag":181,"props":35595,"children":35597},{"className":35596,"style":5443},[290],[],{"type":150,"tag":181,"props":35599,"children":35601},{"className":35600},[296],[35602],{"type":155,"value":238},{"type":150,"tag":181,"props":35604,"children":35606},{"className":35605,"style":4993},[377],[],{"type":150,"tag":181,"props":35608,"children":35610},{"className":35609},[4998],[35611],{"type":155,"value":5380},{"type":150,"tag":181,"props":35613,"children":35615},{"className":35614,"style":4993},[377],[],{"type":150,"tag":181,"props":35617,"children":35619},{"className":35618},[285],[35620,35624,35629,35638,35642,35647],{"type":150,"tag":181,"props":35621,"children":35623},{"className":35622,"style":5680},[290],[],{"type":150,"tag":181,"props":35625,"children":35627},{"className":35626},[296],[35628],{"type":155,"value":35393},{"type":150,"tag":181,"props":35630,"children":35632},{"className":35631},[296,155],[35633],{"type":150,"tag":181,"props":35634,"children":35636},{"className":35635},[296],[35637],{"type":155,"value":23188},{"type":150,"tag":181,"props":35639,"children":35641},{"className":35640,"style":658},[377],[],{"type":150,"tag":181,"props":35643,"children":35645},{"className":35644},[663],[35646],{"type":155,"value":865},{"type":150,"tag":181,"props":35648,"children":35650},{"className":35649,"style":658},[377],[],{"type":150,"tag":181,"props":35652,"children":35654},{"className":35653},[285],[35655,35659],{"type":150,"tag":181,"props":35656,"children":35658},{"className":35657,"style":5012},[290],[],{"type":150,"tag":181,"props":35660,"children":35662},{"className":35661},[296],[35663],{"type":155,"value":35419},{"type":155,"value":35665}," GFLOPs。算术强度大约是 ",{"type":150,"tag":181,"props":35667,"children":35669},{"className":35668},[184],[35670,35691],{"type":150,"tag":181,"props":35671,"children":35673},{"className":35672},[189],[35674],{"type":150,"tag":192,"props":35675,"children":35676},{"xmlns":194},[35677],{"type":150,"tag":197,"props":35678,"children":35679},{},[35680,35687],{"type":150,"tag":201,"props":35681,"children":35682},{},[35683],{"type":150,"tag":215,"props":35684,"children":35685},{},[35686],{"type":155,"value":219},{"type":150,"tag":271,"props":35688,"children":35689},{"encoding":273},[35690],{"type":155,"value":219},{"type":150,"tag":181,"props":35692,"children":35694},{"className":35693,"ariaHidden":223},[280],[35695],{"type":150,"tag":181,"props":35696,"children":35698},{"className":35697},[285],[35699,35703],{"type":150,"tag":181,"props":35700,"children":35702},{"className":35701,"style":5012},[290],[],{"type":150,"tag":181,"props":35704,"children":35706},{"className":35705},[296],[35707],{"type":155,"value":219},{"type":155,"value":35709}," FLOP\u002Fbyte。现代 GPU 的算力带宽比是几十到几百 FLOP\u002Fbyte。差了将近两个数量级，意味着算力几乎全程在等内存，多算一点几乎不花钱。既然算力在空转等数据，就没有理由在生成端省参数量。",{"type":150,"tag":158,"props":35711,"children":35712},{},[35713],{"type":155,"value":35714},"16B 是这么来的。",{"type":150,"tag":4677,"props":35716,"children":35718},{"title":35717},"简化与推断边界",[35719,35724],{"type":150,"tag":151,"props":35720,"children":35721},{},[35722],{"type":155,"value":35723},"上面按\"每步只生成一个 token\"算。speculative decoding 会一次验证多个候选 token，test-time scaling、parallel decoding 也会改变每步的计算和内存特征，但不改变主线：无论一步生成几个 token，权重和 KV Cache 该读的还是要读。speculative decoding 甚至更吃带宽，草稿模型本身要读一遍权重，验证阶段还要再读主模型。",{"type":150,"tag":151,"props":35725,"children":35726},{},[35727],{"type":155,"value":35728},"8B\u002F16B 落在 prefill 和 decode 两侧、方向符合各自瓶颈，这是我的读法，官方没有给出设计理由。",{"type":150,"tag":151,"props":35730,"children":35731},{},[35732],{"type":155,"value":35733},"Compute bound 的那一侧压计算，memory bound 的那一侧压不住，就把能力堆上去。传统 decoder-only LLM 对 prompt 里每个 token 和 decode token 用几乎同一套昂贵 backbone，等于强迫两个瓶颈完全不同的阶段共用一套参数。",{"type":150,"tag":151,"props":35735,"children":35736},{},[35737,35739,35829],{"type":155,"value":35738},"如果输入有 ",{"type":150,"tag":181,"props":35740,"children":35742},{"className":35741},[184],[35743,35777],{"type":150,"tag":181,"props":35744,"children":35746},{"className":35745},[189],[35747],{"type":150,"tag":192,"props":35748,"children":35749},{"xmlns":194},[35750],{"type":150,"tag":197,"props":35751,"children":35752},{},[35753,35772],{"type":150,"tag":201,"props":35754,"children":35755},{},[35756,35760,35764,35768],{"type":150,"tag":209,"props":35757,"children":35758},{},[35759],{"type":155,"value":5417},{"type":150,"tag":221,"props":35761,"children":35762},{},[35763],{"type":155,"value":865},{"type":150,"tag":215,"props":35765,"children":35766},{},[35767],{"type":155,"value":219},{"type":150,"tag":249,"props":35769,"children":35770},{},[35771],{"type":155,"value":5886},{"type":150,"tag":271,"props":35773,"children":35774},{"encoding":273},[35775],{"type":155,"value":35776},"N = 1\\text{M}",{"type":150,"tag":181,"props":35778,"children":35780},{"className":35779,"ariaHidden":223},[280],[35781,35807],{"type":150,"tag":181,"props":35782,"children":35784},{"className":35783},[285],[35785,35789,35794,35798,35803],{"type":150,"tag":181,"props":35786,"children":35788},{"className":35787,"style":5680},[290],[],{"type":150,"tag":181,"props":35790,"children":35792},{"className":35791,"style":5606},[296,301],[35793],{"type":155,"value":5417},{"type":150,"tag":181,"props":35795,"children":35797},{"className":35796,"style":658},[377],[],{"type":150,"tag":181,"props":35799,"children":35801},{"className":35800},[663],[35802],{"type":155,"value":865},{"type":150,"tag":181,"props":35804,"children":35806},{"className":35805,"style":658},[377],[],{"type":150,"tag":181,"props":35808,"children":35810},{"className":35809},[285],[35811,35815,35820],{"type":150,"tag":181,"props":35812,"children":35814},{"className":35813,"style":5680},[290],[],{"type":150,"tag":181,"props":35816,"children":35818},{"className":35817},[296],[35819],{"type":155,"value":219},{"type":150,"tag":181,"props":35821,"children":35823},{"className":35822},[296,155],[35824],{"type":150,"tag":181,"props":35825,"children":35827},{"className":35826},[296],[35828],{"type":155,"value":5886},{"type":155,"value":35830},"，那么即使 attention 已经 sparse 或 compressed，你依然需要对百万 token 做很多层计算。V4.1 攻击的不只是 attention matrix：",{"type":150,"tag":181,"props":35832,"children":35834},{"className":35833},[1003],[35835],{"type":150,"tag":181,"props":35836,"children":35838},{"className":35837},[184],[35839,35874],{"type":150,"tag":181,"props":35840,"children":35842},{"className":35841},[189],[35843],{"type":150,"tag":192,"props":35844,"children":35845},{"xmlns":194,"display":1015},[35846],{"type":150,"tag":197,"props":35847,"children":35848},{},[35849,35869],{"type":150,"tag":201,"props":35850,"children":35851},{},[35852,35857],{"type":150,"tag":209,"props":35853,"children":35854},{},[35855],{"type":155,"value":35856},"Q",{"type":150,"tag":565,"props":35858,"children":35859},{},[35860,35865],{"type":150,"tag":209,"props":35861,"children":35862},{},[35863],{"type":155,"value":35864},"K",{"type":150,"tag":209,"props":35866,"children":35867},{"mathvariant":1618},[35868],{"type":155,"value":1874},{"type":150,"tag":271,"props":35870,"children":35871},{"encoding":273},[35872],{"type":155,"value":35873},"QK^\\top",{"type":150,"tag":181,"props":35875,"children":35877},{"className":35876,"ariaHidden":223},[280],[35878],{"type":150,"tag":181,"props":35879,"children":35881},{"className":35880},[285],[35882,35887,35892],{"type":150,"tag":181,"props":35883,"children":35886},{"className":35884,"style":35885},[290],"height:1.0935em;vertical-align:-0.1944em;",[],{"type":150,"tag":181,"props":35888,"children":35890},{"className":35889},[296,301],[35891],{"type":155,"value":35856},{"type":150,"tag":181,"props":35893,"children":35895},{"className":35894},[296],[35896,35901],{"type":150,"tag":181,"props":35897,"children":35899},{"className":35898,"style":6199},[296,301],[35900],{"type":155,"value":35864},{"type":150,"tag":181,"props":35902,"children":35904},{"className":35903},[307],[35905],{"type":150,"tag":181,"props":35906,"children":35908},{"className":35907},[312],[35909],{"type":150,"tag":181,"props":35910,"children":35912},{"className":35911},[318],[35913],{"type":150,"tag":181,"props":35914,"children":35916},{"className":35915,"style":3821},[323],[35917],{"type":150,"tag":181,"props":35918,"children":35919},{"style":3730},[35920,35924],{"type":150,"tag":181,"props":35921,"children":35923},{"className":35922,"style":334},[333],[],{"type":150,"tag":181,"props":35925,"children":35927},{"className":35926},[339,340,341,342],[35928],{"type":150,"tag":181,"props":35929,"children":35931},{"className":35930},[296,342],[35932],{"type":155,"value":1874},{"type":150,"tag":151,"props":35934,"children":35935},{},[35936,35938,36023,36025,36146,36148,36269,36270,36312,36314,36399],{"type":155,"value":35937},"还攻击整个长序列每层 FFN\u002FMoE 的计算。前面账单上的第二项一直被 ",{"type":150,"tag":181,"props":35939,"children":35941},{"className":35940},[184],[35942,35970],{"type":150,"tag":181,"props":35943,"children":35945},{"className":35944},[189],[35946],{"type":150,"tag":192,"props":35947,"children":35948},{"xmlns":194},[35949],{"type":150,"tag":197,"props":35950,"children":35951},{},[35952,35966],{"type":150,"tag":201,"props":35953,"children":35954},{},[35955],{"type":150,"tag":565,"props":35956,"children":35957},{},[35958,35962],{"type":150,"tag":209,"props":35959,"children":35960},{},[35961],{"type":155,"value":5417},{"type":150,"tag":215,"props":35963,"children":35964},{},[35965],{"type":155,"value":238},{"type":150,"tag":271,"props":35967,"children":35968},{"encoding":273},[35969],{"type":155,"value":7343},{"type":150,"tag":181,"props":35971,"children":35973},{"className":35972,"ariaHidden":223},[280],[35974],{"type":150,"tag":181,"props":35975,"children":35977},{"className":35976},[285],[35978,35982],{"type":150,"tag":181,"props":35979,"children":35981},{"className":35980,"style":6831},[290],[],{"type":150,"tag":181,"props":35983,"children":35985},{"className":35984},[296],[35986,35991],{"type":150,"tag":181,"props":35987,"children":35989},{"className":35988,"style":5606},[296,301],[35990],{"type":155,"value":5417},{"type":150,"tag":181,"props":35992,"children":35994},{"className":35993},[307],[35995],{"type":150,"tag":181,"props":35996,"children":35998},{"className":35997},[312],[35999],{"type":150,"tag":181,"props":36000,"children":36002},{"className":36001},[318],[36003],{"type":150,"tag":181,"props":36004,"children":36006},{"className":36005,"style":6831},[323],[36007],{"type":150,"tag":181,"props":36008,"children":36009},{"style":707},[36010,36014],{"type":150,"tag":181,"props":36011,"children":36013},{"className":36012,"style":334},[333],[],{"type":150,"tag":181,"props":36015,"children":36017},{"className":36016},[339,340,341,342],[36018],{"type":150,"tag":181,"props":36019,"children":36021},{"className":36020},[296,342],[36022],{"type":155,"value":238},{"type":155,"value":36024}," 的光环盖着：attention 那项是 ",{"type":150,"tag":181,"props":36026,"children":36028},{"className":36027},[184],[36029,36073],{"type":150,"tag":181,"props":36030,"children":36032},{"className":36031},[189],[36033],{"type":150,"tag":192,"props":36034,"children":36035},{"xmlns":194},[36036],{"type":150,"tag":197,"props":36037,"children":36038},{},[36039,36069],{"type":150,"tag":201,"props":36040,"children":36041},{},[36042,36046,36050,36061,36065],{"type":150,"tag":209,"props":36043,"children":36044},{},[36045],{"type":155,"value":6749},{"type":150,"tag":221,"props":36047,"children":36048},{"stretchy":789},[36049],{"type":155,"value":792},{"type":150,"tag":565,"props":36051,"children":36052},{},[36053,36057],{"type":150,"tag":209,"props":36054,"children":36055},{},[36056],{"type":155,"value":5417},{"type":150,"tag":215,"props":36058,"children":36059},{},[36060],{"type":155,"value":238},{"type":150,"tag":209,"props":36062,"children":36063},{},[36064],{"type":155,"value":578},{"type":150,"tag":221,"props":36066,"children":36067},{"stretchy":789},[36068],{"type":155,"value":802},{"type":150,"tag":271,"props":36070,"children":36071},{"encoding":273},[36072],{"type":155,"value":6904},{"type":150,"tag":181,"props":36074,"children":36076},{"className":36075,"ariaHidden":223},[280],[36077],{"type":150,"tag":181,"props":36078,"children":36080},{"className":36079},[285],[36081,36085,36090,36095,36136,36141],{"type":150,"tag":181,"props":36082,"children":36084},{"className":36083,"style":6789},[290],[],{"type":150,"tag":181,"props":36086,"children":36088},{"className":36087,"style":6794},[296,301],[36089],{"type":155,"value":6749},{"type":150,"tag":181,"props":36091,"children":36093},{"className":36092},[824],[36094],{"type":155,"value":792},{"type":150,"tag":181,"props":36096,"children":36098},{"className":36097},[296],[36099,36104],{"type":150,"tag":181,"props":36100,"children":36102},{"className":36101,"style":5606},[296,301],[36103],{"type":155,"value":5417},{"type":150,"tag":181,"props":36105,"children":36107},{"className":36106},[307],[36108],{"type":150,"tag":181,"props":36109,"children":36111},{"className":36110},[312],[36112],{"type":150,"tag":181,"props":36113,"children":36115},{"className":36114},[318],[36116],{"type":150,"tag":181,"props":36117,"children":36119},{"className":36118,"style":6831},[323],[36120],{"type":150,"tag":181,"props":36121,"children":36122},{"style":707},[36123,36127],{"type":150,"tag":181,"props":36124,"children":36126},{"className":36125,"style":334},[333],[],{"type":150,"tag":181,"props":36128,"children":36130},{"className":36129},[339,340,341,342],[36131],{"type":150,"tag":181,"props":36132,"children":36134},{"className":36133},[296,342],[36135],{"type":155,"value":238},{"type":150,"tag":181,"props":36137,"children":36139},{"className":36138},[296,301],[36140],{"type":155,"value":578},{"type":150,"tag":181,"props":36142,"children":36144},{"className":36143},[835],[36145],{"type":155,"value":802},{"type":155,"value":36147},"，FFN\u002FMoE 那项是 ",{"type":150,"tag":181,"props":36149,"children":36151},{"className":36150},[184],[36152,36196],{"type":150,"tag":181,"props":36153,"children":36155},{"className":36154},[189],[36156],{"type":150,"tag":192,"props":36157,"children":36158},{"xmlns":194},[36159],{"type":150,"tag":197,"props":36160,"children":36161},{},[36162,36192],{"type":150,"tag":201,"props":36163,"children":36164},{},[36165,36169,36173,36177,36188],{"type":150,"tag":209,"props":36166,"children":36167},{},[36168],{"type":155,"value":6749},{"type":150,"tag":221,"props":36170,"children":36171},{"stretchy":789},[36172],{"type":155,"value":792},{"type":150,"tag":209,"props":36174,"children":36175},{},[36176],{"type":155,"value":5417},{"type":150,"tag":565,"props":36178,"children":36179},{},[36180,36184],{"type":150,"tag":209,"props":36181,"children":36182},{},[36183],{"type":155,"value":578},{"type":150,"tag":215,"props":36185,"children":36186},{},[36187],{"type":155,"value":238},{"type":150,"tag":221,"props":36189,"children":36190},{"stretchy":789},[36191],{"type":155,"value":802},{"type":150,"tag":271,"props":36193,"children":36194},{"encoding":273},[36195],{"type":155,"value":6777},{"type":150,"tag":181,"props":36197,"children":36199},{"className":36198,"ariaHidden":223},[280],[36200],{"type":150,"tag":181,"props":36201,"children":36203},{"className":36202},[285],[36204,36208,36213,36218,36223,36264],{"type":150,"tag":181,"props":36205,"children":36207},{"className":36206,"style":6789},[290],[],{"type":150,"tag":181,"props":36209,"children":36211},{"className":36210,"style":6794},[296,301],[36212],{"type":155,"value":6749},{"type":150,"tag":181,"props":36214,"children":36216},{"className":36215},[824],[36217],{"type":155,"value":792},{"type":150,"tag":181,"props":36219,"children":36221},{"className":36220,"style":5606},[296,301],[36222],{"type":155,"value":5417},{"type":150,"tag":181,"props":36224,"children":36226},{"className":36225},[296],[36227,36232],{"type":150,"tag":181,"props":36228,"children":36230},{"className":36229},[296,301],[36231],{"type":155,"value":578},{"type":150,"tag":181,"props":36233,"children":36235},{"className":36234},[307],[36236],{"type":150,"tag":181,"props":36237,"children":36239},{"className":36238},[312],[36240],{"type":150,"tag":181,"props":36241,"children":36243},{"className":36242},[318],[36244],{"type":150,"tag":181,"props":36245,"children":36247},{"className":36246,"style":6831},[323],[36248],{"type":150,"tag":181,"props":36249,"children":36250},{"style":707},[36251,36255],{"type":150,"tag":181,"props":36252,"children":36254},{"className":36253,"style":334},[333],[],{"type":150,"tag":181,"props":36256,"children":36258},{"className":36257},[339,340,341,342],[36259],{"type":150,"tag":181,"props":36260,"children":36262},{"className":36261},[296,342],[36263],{"type":155,"value":238},{"type":150,"tag":181,"props":36265,"children":36267},{"className":36266},[835],[36268],{"type":155,"value":802},{"type":155,"value":723},{"type":150,"tag":181,"props":36271,"children":36273},{"className":36272},[184],[36274,36295],{"type":150,"tag":181,"props":36275,"children":36277},{"className":36276},[189],[36278],{"type":150,"tag":192,"props":36279,"children":36280},{"xmlns":194},[36281],{"type":150,"tag":197,"props":36282,"children":36283},{},[36284,36291],{"type":150,"tag":201,"props":36285,"children":36286},{},[36287],{"type":150,"tag":209,"props":36288,"children":36289},{},[36290],{"type":155,"value":5417},{"type":150,"tag":271,"props":36292,"children":36293},{"encoding":273},[36294],{"type":155,"value":5417},{"type":150,"tag":181,"props":36296,"children":36298},{"className":36297,"ariaHidden":223},[280],[36299],{"type":150,"tag":181,"props":36300,"children":36302},{"className":36301},[285],[36303,36307],{"type":150,"tag":181,"props":36304,"children":36306},{"className":36305,"style":5680},[290],[],{"type":150,"tag":181,"props":36308,"children":36310},{"className":36309,"style":5606},[296,301],[36311],{"type":155,"value":5417},{"type":155,"value":36313}," 大到百万量级、",{"type":150,"tag":181,"props":36315,"children":36317},{"className":36316},[184],[36318,36346],{"type":150,"tag":181,"props":36319,"children":36321},{"className":36320},[189],[36322],{"type":150,"tag":192,"props":36323,"children":36324},{"xmlns":194},[36325],{"type":150,"tag":197,"props":36326,"children":36327},{},[36328,36342],{"type":150,"tag":201,"props":36329,"children":36330},{},[36331],{"type":150,"tag":565,"props":36332,"children":36333},{},[36334,36338],{"type":150,"tag":209,"props":36335,"children":36336},{},[36337],{"type":155,"value":5417},{"type":150,"tag":215,"props":36339,"children":36340},{},[36341],{"type":155,"value":238},{"type":150,"tag":271,"props":36343,"children":36344},{"encoding":273},[36345],{"type":155,"value":7343},{"type":150,"tag":181,"props":36347,"children":36349},{"className":36348,"ariaHidden":223},[280],[36350],{"type":150,"tag":181,"props":36351,"children":36353},{"className":36352},[285],[36354,36358],{"type":150,"tag":181,"props":36355,"children":36357},{"className":36356,"style":6831},[290],[],{"type":150,"tag":181,"props":36359,"children":36361},{"className":36360},[296],[36362,36367],{"type":150,"tag":181,"props":36363,"children":36365},{"className":36364,"style":5606},[296,301],[36366],{"type":155,"value":5417},{"type":150,"tag":181,"props":36368,"children":36370},{"className":36369},[307],[36371],{"type":150,"tag":181,"props":36372,"children":36374},{"className":36373},[312],[36375],{"type":150,"tag":181,"props":36376,"children":36378},{"className":36377},[318],[36379],{"type":150,"tag":181,"props":36380,"children":36382},{"className":36381,"style":6831},[323],[36383],{"type":150,"tag":181,"props":36384,"children":36385},{"style":707},[36386,36390],{"type":150,"tag":181,"props":36387,"children":36389},{"className":36388,"style":334},[333],[],{"type":150,"tag":181,"props":36391,"children":36393},{"className":36392},[339,340,341,342],[36394],{"type":150,"tag":181,"props":36395,"children":36397},{"className":36396},[296,342],[36398],{"type":155,"value":238},{"type":155,"value":36400}," 又被打下来之后，后者因为每一层都要对每个 token 算一遍，纹丝不动。",{"type":150,"tag":151,"props":36402,"children":36403},{},[36404],{"type":155,"value":36405},"所以 V4.1 做 asymmetric compute，本质上是：attention 都已经省下来了，总不能剩下的网络继续拿同样预算烧百万 token。终于有人检查了完整账单，而不是只盯着最显眼的那一行。",{"type":150,"tag":151,"props":36407,"children":36408},{},[36409,36411,36418],{"type":155,"value":36410},"省下来的成本是多少？ds4.1f 的技术报告（",{"type":150,"tag":9663,"props":36412,"children":36415},{"href":36413,"rel":36414},"https:\u002F\u002Fwww.alphaxiv.org\u002Fabs\u002F2609.deepseek-v4-1-flash",[9667],[36416],{"type":155,"value":36417},"DeepSeek-V4.1-Flash: Pushing the Limits of KV Cache Compression",{"type":155,"value":36419},"）给了具体的数。相比上一代 V4-Flash：",{"type":150,"tag":23633,"props":36421,"children":36422},{},[36423,36447],{"type":150,"tag":23637,"props":36424,"children":36425},{},[36426,36428,36433,36435,36440,36442],{"type":155,"value":36427},"常驻 ",{"type":150,"tag":158,"props":36429,"children":36430},{},[36431],{"type":155,"value":36432},"HBM",{"type":155,"value":36434}," 的 global KV cache：",{"type":150,"tag":158,"props":36436,"children":36437},{},[36438],{"type":155,"value":36439},"890 字节 \u002F token",{"type":155,"value":36441},"，约为原来的 ",{"type":150,"tag":158,"props":36443,"children":36444},{},[36445],{"type":155,"value":36446},"1\u002F4",{"type":150,"tag":23637,"props":36448,"children":36449},{},[36450,36452,36457,36459],{"type":155,"value":36451},"落在 ",{"type":150,"tag":158,"props":36453,"children":36454},{},[36455],{"type":155,"value":36456},"SSD 或 host memory",{"type":155,"value":36458}," 的 persistent KV cache：约为原来的 ",{"type":150,"tag":158,"props":36460,"children":36461},{},[36462],{"type":155,"value":36463},"1\u002F8",{"type":150,"tag":151,"props":36465,"children":36466},{},[36467,36469,36511,36513,36555,36557,36599],{"type":155,"value":36468},"报告把这些收益归因于\"entry size、sequence dimension、layer dimension 三个方向上的协同优化\"，落到 CSA2 上就是三个模式：投影让 head 之间共享信息（压 ",{"type":150,"tag":181,"props":36470,"children":36472},{"className":36471},[184],[36473,36494],{"type":150,"tag":181,"props":36474,"children":36476},{"className":36475},[189],[36477],{"type":150,"tag":192,"props":36478,"children":36479},{"xmlns":194},[36480],{"type":150,"tag":197,"props":36481,"children":36482},{},[36483,36490],{"type":150,"tag":201,"props":36484,"children":36485},{},[36486],{"type":150,"tag":209,"props":36487,"children":36488},{},[36489],{"type":155,"value":34490},{"type":150,"tag":271,"props":36491,"children":36492},{"encoding":273},[36493],{"type":155,"value":34490},{"type":150,"tag":181,"props":36495,"children":36497},{"className":36496,"ariaHidden":223},[280],[36498],{"type":150,"tag":181,"props":36499,"children":36501},{"className":36500},[285],[36502,36506],{"type":150,"tag":181,"props":36503,"children":36505},{"className":36504,"style":5680},[290],[],{"type":150,"tag":181,"props":36507,"children":36509},{"className":36508,"style":6794},[296,301],[36510],{"type":155,"value":34490},{"type":155,"value":36512},"）、把多个 token 合成一个 KV entry（压 ",{"type":150,"tag":181,"props":36514,"children":36516},{"className":36515},[184],[36517,36538],{"type":150,"tag":181,"props":36518,"children":36520},{"className":36519},[189],[36521],{"type":150,"tag":192,"props":36522,"children":36523},{"xmlns":194},[36524],{"type":150,"tag":197,"props":36525,"children":36526},{},[36527,36534],{"type":150,"tag":201,"props":36528,"children":36529},{},[36530],{"type":150,"tag":209,"props":36531,"children":36532},{},[36533],{"type":155,"value":5417},{"type":150,"tag":271,"props":36535,"children":36536},{"encoding":273},[36537],{"type":155,"value":5417},{"type":150,"tag":181,"props":36539,"children":36541},{"className":36540,"ariaHidden":223},[280],[36542],{"type":150,"tag":181,"props":36543,"children":36545},{"className":36544},[285],[36546,36550],{"type":150,"tag":181,"props":36547,"children":36549},{"className":36548,"style":5680},[290],[],{"type":150,"tag":181,"props":36551,"children":36553},{"className":36552,"style":5606},[296,301],[36554],{"type":155,"value":5417},{"type":155,"value":36556},"）、跨层复用 KV（压 ",{"type":150,"tag":181,"props":36558,"children":36560},{"className":36559},[184],[36561,36582],{"type":150,"tag":181,"props":36562,"children":36564},{"className":36563},[189],[36565],{"type":150,"tag":192,"props":36566,"children":36567},{"xmlns":194},[36568],{"type":150,"tag":197,"props":36569,"children":36570},{},[36571,36578],{"type":150,"tag":201,"props":36572,"children":36573},{},[36574],{"type":150,"tag":209,"props":36575,"children":36576},{},[36577],{"type":155,"value":5385},{"type":150,"tag":271,"props":36579,"children":36580},{"encoding":273},[36581],{"type":155,"value":5385},{"type":150,"tag":181,"props":36583,"children":36585},{"className":36584,"ariaHidden":223},[280],[36586],{"type":150,"tag":181,"props":36587,"children":36589},{"className":36588},[285],[36590,36594],{"type":150,"tag":181,"props":36591,"children":36593},{"className":36592,"style":5680},[290],[],{"type":150,"tag":181,"props":36595,"children":36597},{"className":36596},[296,301],[36598],{"type":155,"value":5385},{"type":155,"value":36600},"）。再加上主 KV cache 本身换成 FP4。",{"type":150,"tag":151,"props":36602,"children":36603},{},[36604,36720],{"type":150,"tag":158,"props":36605,"children":36606},{},[36607,36609,36719],{"type":155,"value":36608},"这不就是前面的 ",{"type":150,"tag":181,"props":36610,"children":36612},{"className":36611},[184],[36613,36650],{"type":150,"tag":181,"props":36614,"children":36616},{"className":36615},[189],[36617],{"type":150,"tag":192,"props":36618,"children":36619},{"xmlns":194},[36620],{"type":150,"tag":197,"props":36621,"children":36622},{},[36623,36646],{"type":150,"tag":201,"props":36624,"children":36625},{},[36626,36630,36634,36638,36642],{"type":150,"tag":209,"props":36627,"children":36628},{},[36629],{"type":155,"value":5385},{"type":150,"tag":221,"props":36631,"children":36632},{},[36633],{"type":155,"value":5380},{"type":150,"tag":209,"props":36635,"children":36636},{},[36637],{"type":155,"value":5417},{"type":150,"tag":221,"props":36639,"children":36640},{},[36641],{"type":155,"value":5380},{"type":150,"tag":209,"props":36643,"children":36644},{},[36645],{"type":155,"value":34490},{"type":150,"tag":271,"props":36647,"children":36648},{"encoding":273},[36649],{"type":155,"value":34495},{"type":150,"tag":181,"props":36651,"children":36653},{"className":36652,"ariaHidden":223},[280],[36654,36680,36706],{"type":150,"tag":181,"props":36655,"children":36657},{"className":36656},[285],[36658,36662,36667,36671,36676],{"type":150,"tag":181,"props":36659,"children":36661},{"className":36660,"style":5470},[290],[],{"type":150,"tag":181,"props":36663,"children":36665},{"className":36664},[296,301],[36666],{"type":155,"value":5385},{"type":150,"tag":181,"props":36668,"children":36670},{"className":36669,"style":4993},[377],[],{"type":150,"tag":181,"props":36672,"children":36674},{"className":36673},[4998],[36675],{"type":155,"value":5380},{"type":150,"tag":181,"props":36677,"children":36679},{"className":36678,"style":4993},[377],[],{"type":150,"tag":181,"props":36681,"children":36683},{"className":36682},[285],[36684,36688,36693,36697,36702],{"type":150,"tag":181,"props":36685,"children":36687},{"className":36686,"style":5470},[290],[],{"type":150,"tag":181,"props":36689,"children":36691},{"className":36690,"style":5606},[296,301],[36692],{"type":155,"value":5417},{"type":150,"tag":181,"props":36694,"children":36696},{"className":36695,"style":4993},[377],[],{"type":150,"tag":181,"props":36698,"children":36700},{"className":36699},[4998],[36701],{"type":155,"value":5380},{"type":150,"tag":181,"props":36703,"children":36705},{"className":36704,"style":4993},[377],[],{"type":150,"tag":181,"props":36707,"children":36709},{"className":36708},[285],[36710,36714],{"type":150,"tag":181,"props":36711,"children":36713},{"className":36712,"style":5680},[290],[],{"type":150,"tag":181,"props":36715,"children":36717},{"className":36716,"style":6794},[296,301],[36718],{"type":155,"value":34490},{"type":155,"value":723},{"type":155,"value":36721}," 前面那条\"压宽度 → 压长度 → 压深度\"的路线，在 ds4.1f 上是第一次三个方向同时动刀。",{"type":150,"tag":151,"props":36723,"children":36724},{},[36725,36727,36769],{"type":155,"value":36726},"压 ",{"type":150,"tag":181,"props":36728,"children":36730},{"className":36729},[184],[36731,36752],{"type":150,"tag":181,"props":36732,"children":36734},{"className":36733},[189],[36735],{"type":150,"tag":192,"props":36736,"children":36737},{"xmlns":194},[36738],{"type":150,"tag":197,"props":36739,"children":36740},{},[36741,36748],{"type":150,"tag":201,"props":36742,"children":36743},{},[36744],{"type":150,"tag":209,"props":36745,"children":36746},{},[36747],{"type":155,"value":5385},{"type":150,"tag":271,"props":36749,"children":36750},{"encoding":273},[36751],{"type":155,"value":5385},{"type":150,"tag":181,"props":36753,"children":36755},{"className":36754,"ariaHidden":223},[280],[36756],{"type":150,"tag":181,"props":36757,"children":36759},{"className":36758},[285],[36760,36764],{"type":150,"tag":181,"props":36761,"children":36763},{"className":36762,"style":5680},[290],[],{"type":150,"tag":181,"props":36765,"children":36767},{"className":36766},[296,301],[36768],{"type":155,"value":5385},{"type":155,"value":36770}," 这一个方向前面一直只是断言，说清楚它得看 CSA2 的三种模式：",{"type":150,"tag":23633,"props":36772,"children":36773},{},[36774,36784,36801],{"type":150,"tag":23637,"props":36775,"children":36776},{},[36777,36782],{"type":150,"tag":158,"props":36778,"children":36779},{},[36780],{"type":155,"value":36781},"Full",{"type":155,"value":36783},"：这层自己算主 K、V，投影一个 indexer key，产出一组新的 Top-K 索引",{"type":150,"tag":23637,"props":36785,"children":36786},{},[36787,36792,36794,36799],{"type":150,"tag":158,"props":36788,"children":36789},{},[36790],{"type":155,"value":36791},"Reindex",{"type":155,"value":36793},"：主 K、V 和 indexer key 都从前一层继承，只重算 indexer query，于是选出一组",{"type":150,"tag":158,"props":36795,"children":36796},{},[36797],{"type":155,"value":36798},"不同",{"type":155,"value":36800},"的 Top-K",{"type":150,"tag":23637,"props":36802,"children":36803},{},[36804,36809],{"type":150,"tag":158,"props":36805,"children":36806},{},[36807],{"type":155,"value":36808},"Reuse",{"type":155,"value":36810},"：主 K、V 和 Top-K 索引全部继承，这层几乎不做索引计算",{"type":150,"tag":151,"props":36812,"children":36813},{},[36814],{"type":155,"value":36815},"再配一个分层的 indexer：第一个 Full 层先圈出一个 block 级的候选池，后面的 Reindex 层只在这个池子里打分。于是每个 query 的索引开销被限住，context 是一万还是一百万都一样。",{"type":150,"tag":151,"props":36817,"children":36818},{},[36819],{"type":155,"value":36820},"这就是\"每一层不再重复为同一份历史付费\"的具体做法：多数层复用索引，少数层重建。",{"type":150,"tag":151,"props":36822,"children":36823},{},[36824],{"type":155,"value":36825},"持久层那 1\u002F8 靠的是另一个部署优化，叫 SWA Bounded Replay：局部窗口的 cache 不再常驻，请求恢复时只重放最近一小段就能近似重建，不必把整条序列重算一遍。",{"type":150,"tag":151,"props":36827,"children":36828},{},[36829,36831,36935,36937,37377],{"type":155,"value":36830},"另外 CED 让 prefill 的复杂度从 ",{"type":150,"tag":181,"props":36832,"children":36834},{"className":36833},[184],[36835,36877],{"type":150,"tag":181,"props":36836,"children":36838},{"className":36837},[189],[36839],{"type":150,"tag":192,"props":36840,"children":36841},{"xmlns":194},[36842],{"type":150,"tag":197,"props":36843,"children":36844},{},[36845,36872],{"type":150,"tag":201,"props":36846,"children":36847},{},[36848,36852,36856,36860,36864,36868],{"type":150,"tag":209,"props":36849,"children":36850},{},[36851],{"type":155,"value":6749},{"type":150,"tag":221,"props":36853,"children":36854},{"stretchy":789},[36855],{"type":155,"value":792},{"type":150,"tag":209,"props":36857,"children":36858},{},[36859],{"type":155,"value":5417},{"type":150,"tag":221,"props":36861,"children":36862},{},[36863],{"type":155,"value":25037},{"type":150,"tag":209,"props":36865,"children":36866},{},[36867],{"type":155,"value":5385},{"type":150,"tag":221,"props":36869,"children":36870},{"stretchy":789},[36871],{"type":155,"value":802},{"type":150,"tag":271,"props":36873,"children":36874},{"encoding":273},[36875],{"type":155,"value":36876},"O(N \\cdot L)",{"type":150,"tag":181,"props":36878,"children":36880},{"className":36879,"ariaHidden":223},[280],[36881,36917],{"type":150,"tag":181,"props":36882,"children":36884},{"className":36883},[285],[36885,36889,36894,36899,36904,36908,36913],{"type":150,"tag":181,"props":36886,"children":36888},{"className":36887,"style":819},[290],[],{"type":150,"tag":181,"props":36890,"children":36892},{"className":36891,"style":6794},[296,301],[36893],{"type":155,"value":6749},{"type":150,"tag":181,"props":36895,"children":36897},{"className":36896},[824],[36898],{"type":155,"value":792},{"type":150,"tag":181,"props":36900,"children":36902},{"className":36901,"style":5606},[296,301],[36903],{"type":155,"value":5417},{"type":150,"tag":181,"props":36905,"children":36907},{"className":36906,"style":4993},[377],[],{"type":150,"tag":181,"props":36909,"children":36911},{"className":36910},[4998],[36912],{"type":155,"value":25037},{"type":150,"tag":181,"props":36914,"children":36916},{"className":36915,"style":4993},[377],[],{"type":150,"tag":181,"props":36918,"children":36920},{"className":36919},[285],[36921,36925,36930],{"type":150,"tag":181,"props":36922,"children":36924},{"className":36923,"style":819},[290],[],{"type":150,"tag":181,"props":36926,"children":36928},{"className":36927},[296,301],[36929],{"type":155,"value":5385},{"type":150,"tag":181,"props":36931,"children":36933},{"className":36932},[835],[36934],{"type":155,"value":802},{"type":155,"value":36936}," 降到大约 ",{"type":150,"tag":181,"props":36938,"children":36940},{"className":36939},[184],[36941,37021],{"type":150,"tag":181,"props":36942,"children":36944},{"className":36943},[189],[36945],{"type":150,"tag":192,"props":36946,"children":36947},{"xmlns":194},[36948],{"type":150,"tag":197,"props":36949,"children":36950},{},[36951,37016],{"type":150,"tag":201,"props":36952,"children":36953},{},[36954,36958,36962,36966,36970,36981,36985,36997,37001,37012],{"type":150,"tag":209,"props":36955,"children":36956},{},[36957],{"type":155,"value":6749},{"type":150,"tag":221,"props":36959,"children":36960},{"stretchy":789},[36961],{"type":155,"value":792},{"type":150,"tag":209,"props":36963,"children":36964},{},[36965],{"type":155,"value":5417},{"type":150,"tag":221,"props":36967,"children":36968},{},[36969],{"type":155,"value":25037},{"type":150,"tag":1774,"props":36971,"children":36972},{},[36973,36977],{"type":150,"tag":209,"props":36974,"children":36975},{},[36976],{"type":155,"value":5385},{"type":150,"tag":215,"props":36978,"children":36979},{},[36980],{"type":155,"value":238},{"type":150,"tag":221,"props":36982,"children":36983},{},[36984],{"type":155,"value":4962},{"type":150,"tag":205,"props":36986,"children":36987},{},[36988,36992],{"type":150,"tag":209,"props":36989,"children":36990},{},[36991],{"type":155,"value":1639},{"type":150,"tag":249,"props":36993,"children":36994},{},[36995],{"type":155,"value":36996},"win",{"type":150,"tag":221,"props":36998,"children":36999},{},[37000],{"type":155,"value":25037},{"type":150,"tag":1774,"props":37002,"children":37003},{},[37004,37008],{"type":150,"tag":209,"props":37005,"children":37006},{},[37007],{"type":155,"value":5385},{"type":150,"tag":215,"props":37009,"children":37010},{},[37011],{"type":155,"value":238},{"type":150,"tag":221,"props":37013,"children":37014},{"stretchy":789},[37015],{"type":155,"value":802},{"type":150,"tag":271,"props":37017,"children":37018},{"encoding":273},[37019],{"type":155,"value":37020},"O(N \\cdot \\frac{L}{2} + n_{\\text{win}} \\cdot \\frac{L}{2})",{"type":150,"tag":181,"props":37022,"children":37024},{"className":37023,"ariaHidden":223},[280],[37025,37061,37182,37269],{"type":150,"tag":181,"props":37026,"children":37028},{"className":37027},[285],[37029,37033,37038,37043,37048,37052,37057],{"type":150,"tag":181,"props":37030,"children":37032},{"className":37031,"style":819},[290],[],{"type":150,"tag":181,"props":37034,"children":37036},{"className":37035,"style":6794},[296,301],[37037],{"type":155,"value":6749},{"type":150,"tag":181,"props":37039,"children":37041},{"className":37040},[824],[37042],{"type":155,"value":792},{"type":150,"tag":181,"props":37044,"children":37046},{"className":37045,"style":5606},[296,301],[37047],{"type":155,"value":5417},{"type":150,"tag":181,"props":37049,"children":37051},{"className":37050,"style":4993},[377],[],{"type":150,"tag":181,"props":37053,"children":37055},{"className":37054},[4998],[37056],{"type":155,"value":25037},{"type":150,"tag":181,"props":37058,"children":37060},{"className":37059,"style":4993},[377],[],{"type":150,"tag":181,"props":37062,"children":37064},{"className":37063},[285],[37065,37070,37169,37173,37178],{"type":150,"tag":181,"props":37066,"children":37069},{"className":37067,"style":37068},[290],"height:1.2173em;vertical-align:-0.345em;",[],{"type":150,"tag":181,"props":37071,"children":37073},{"className":37072},[296],[37074,37078,37165],{"type":150,"tag":181,"props":37075,"children":37077},{"className":37076},[824,2482],[],{"type":150,"tag":181,"props":37079,"children":37081},{"className":37080},[1774],[37082],{"type":150,"tag":181,"props":37083,"children":37085},{"className":37084},[312,313],[37086,37153],{"type":150,"tag":181,"props":37087,"children":37089},{"className":37088},[318],[37090,37148],{"type":150,"tag":181,"props":37091,"children":37094},{"className":37092,"style":37093},[323],"height:0.8723em;",[37095,37116,37127],{"type":150,"tag":181,"props":37096,"children":37098},{"style":37097},"top:-2.655em;",[37099,37103],{"type":150,"tag":181,"props":37100,"children":37102},{"className":37101,"style":8240},[333],[],{"type":150,"tag":181,"props":37104,"children":37106},{"className":37105},[339,340,341,342],[37107],{"type":150,"tag":181,"props":37108,"children":37110},{"className":37109},[296,342],[37111],{"type":150,"tag":181,"props":37112,"children":37114},{"className":37113},[296,342],[37115],{"type":155,"value":238},{"type":150,"tag":181,"props":37117,"children":37118},{"style":8253},[37119,37123],{"type":150,"tag":181,"props":37120,"children":37122},{"className":37121,"style":8240},[333],[],{"type":150,"tag":181,"props":37124,"children":37126},{"className":37125,"style":2804},[2803],[],{"type":150,"tag":181,"props":37128,"children":37130},{"style":37129},"top:-3.394em;",[37131,37135],{"type":150,"tag":181,"props":37132,"children":37134},{"className":37133,"style":8240},[333],[],{"type":150,"tag":181,"props":37136,"children":37138},{"className":37137},[339,340,341,342],[37139],{"type":150,"tag":181,"props":37140,"children":37142},{"className":37141},[296,342],[37143],{"type":150,"tag":181,"props":37144,"children":37146},{"className":37145},[296,301,342],[37147],{"type":155,"value":5385},{"type":150,"tag":181,"props":37149,"children":37151},{"className":37150},[352],[37152],{"type":155,"value":355},{"type":150,"tag":181,"props":37154,"children":37156},{"className":37155},[318],[37157],{"type":150,"tag":181,"props":37158,"children":37161},{"className":37159,"style":37160},[323],"height:0.345em;",[37162],{"type":150,"tag":181,"props":37163,"children":37164},{},[],{"type":150,"tag":181,"props":37166,"children":37168},{"className":37167},[835,2482],[],{"type":150,"tag":181,"props":37170,"children":37172},{"className":37171,"style":4993},[377],[],{"type":150,"tag":181,"props":37174,"children":37176},{"className":37175},[4998],[37177],{"type":155,"value":4962},{"type":150,"tag":181,"props":37179,"children":37181},{"className":37180,"style":4993},[377],[],{"type":150,"tag":181,"props":37183,"children":37185},{"className":37184},[285],[37186,37191,37256,37260,37265],{"type":150,"tag":181,"props":37187,"children":37190},{"className":37188,"style":37189},[290],"height:0.5945em;vertical-align:-0.15em;",[],{"type":150,"tag":181,"props":37192,"children":37194},{"className":37193},[296],[37195,37200],{"type":150,"tag":181,"props":37196,"children":37198},{"className":37197},[296,301],[37199],{"type":155,"value":1639},{"type":150,"tag":181,"props":37201,"children":37203},{"className":37202},[307],[37204],{"type":150,"tag":181,"props":37205,"children":37207},{"className":37206},[312,313],[37208,37245],{"type":150,"tag":181,"props":37209,"children":37211},{"className":37210},[318],[37212,37240],{"type":150,"tag":181,"props":37213,"children":37215},{"className":37214,"style":6304},[323],[37216],{"type":150,"tag":181,"props":37217,"children":37218},{"style":328},[37219,37223],{"type":150,"tag":181,"props":37220,"children":37222},{"className":37221,"style":334},[333],[],{"type":150,"tag":181,"props":37224,"children":37226},{"className":37225},[339,340,341,342],[37227],{"type":150,"tag":181,"props":37228,"children":37230},{"className":37229},[296,342],[37231],{"type":150,"tag":181,"props":37232,"children":37234},{"className":37233},[296,155,342],[37235],{"type":150,"tag":181,"props":37236,"children":37238},{"className":37237},[296,342],[37239],{"type":155,"value":36996},{"type":150,"tag":181,"props":37241,"children":37243},{"className":37242},[352],[37244],{"type":155,"value":355},{"type":150,"tag":181,"props":37246,"children":37248},{"className":37247},[318],[37249],{"type":150,"tag":181,"props":37250,"children":37252},{"className":37251,"style":363},[323],[37253],{"type":150,"tag":181,"props":37254,"children":37255},{},[],{"type":150,"tag":181,"props":37257,"children":37259},{"className":37258,"style":4993},[377],[],{"type":150,"tag":181,"props":37261,"children":37263},{"className":37262},[4998],[37264],{"type":155,"value":25037},{"type":150,"tag":181,"props":37266,"children":37268},{"className":37267,"style":4993},[377],[],{"type":150,"tag":181,"props":37270,"children":37272},{"className":37271},[285],[37273,37277,37372],{"type":150,"tag":181,"props":37274,"children":37276},{"className":37275,"style":37068},[290],[],{"type":150,"tag":181,"props":37278,"children":37280},{"className":37279},[296],[37281,37285,37368],{"type":150,"tag":181,"props":37282,"children":37284},{"className":37283},[824,2482],[],{"type":150,"tag":181,"props":37286,"children":37288},{"className":37287},[1774],[37289],{"type":150,"tag":181,"props":37290,"children":37292},{"className":37291},[312,313],[37293,37357],{"type":150,"tag":181,"props":37294,"children":37296},{"className":37295},[318],[37297,37352],{"type":150,"tag":181,"props":37298,"children":37300},{"className":37299,"style":37093},[323],[37301,37321,37332],{"type":150,"tag":181,"props":37302,"children":37303},{"style":37097},[37304,37308],{"type":150,"tag":181,"props":37305,"children":37307},{"className":37306,"style":8240},[333],[],{"type":150,"tag":181,"props":37309,"children":37311},{"className":37310},[339,340,341,342],[37312],{"type":150,"tag":181,"props":37313,"children":37315},{"className":37314},[296,342],[37316],{"type":150,"tag":181,"props":37317,"children":37319},{"className":37318},[296,342],[37320],{"type":155,"value":238},{"type":150,"tag":181,"props":37322,"children":37323},{"style":8253},[37324,37328],{"type":150,"tag":181,"props":37325,"children":37327},{"className":37326,"style":8240},[333],[],{"type":150,"tag":181,"props":37329,"children":37331},{"className":37330,"style":2804},[2803],[],{"type":150,"tag":181,"props":37333,"children":37334},{"style":37129},[37335,37339],{"type":150,"tag":181,"props":37336,"children":37338},{"className":37337,"style":8240},[333],[],{"type":150,"tag":181,"props":37340,"children":37342},{"className":37341},[339,340,341,342],[37343],{"type":150,"tag":181,"props":37344,"children":37346},{"className":37345},[296,342],[37347],{"type":150,"tag":181,"props":37348,"children":37350},{"className":37349},[296,301,342],[37351],{"type":155,"value":5385},{"type":150,"tag":181,"props":37353,"children":37355},{"className":37354},[352],[37356],{"type":155,"value":355},{"type":150,"tag":181,"props":37358,"children":37360},{"className":37359},[318],[37361],{"type":150,"tag":181,"props":37362,"children":37364},{"className":37363,"style":37160},[323],[37365],{"type":150,"tag":181,"props":37366,"children":37367},{},[],{"type":150,"tag":181,"props":37369,"children":37371},{"className":37370},[835,2482],[],{"type":150,"tag":181,"props":37373,"children":37375},{"className":37374},[835],[37376],{"type":155,"value":802},{"type":155,"value":37378},"，长输入的计算量差不多减半。",{"type":150,"tag":151,"props":37380,"children":37381},{},[37382,37384,37391],{"type":155,"value":37383},"那 890 字节 \u002F token 在真实硬件上意味着什么。拿 B200 算一遍（",{"type":150,"tag":9663,"props":37385,"children":37388},{"href":37386,"rel":37387},"https:\u002F\u002Finferencex.semianalysis.com\u002Fchips\u002Fb200",[9667],[37389],{"type":155,"value":37390},"规格",{"type":155,"value":37392},"）。",{"type":150,"tag":151,"props":37394,"children":37395},{},[37396,37398,37403],{"type":155,"value":37397},"先说部署形态。552B 权重按 FP4 存是 276 GB，单卡 180 GB 装不下，所以要拆卡。",{"type":150,"tag":158,"props":37399,"children":37400},{},[37401],{"type":155,"value":37402},"四卡（TP4）就够",{"type":155,"value":37404},"：每卡 69 GB 权重，还剩 111 GB 放 KV。",{"type":150,"tag":151,"props":37406,"children":37407},{},[37408,37410,37415],{"type":155,"value":37409},"下面算的都是",{"type":150,"tag":158,"props":37411,"children":37412},{},[37413],{"type":155,"value":37414},"每张卡",{"type":155,"value":37416},"的数字。TP4 之后每卡只承担四分之一，所以字面值和耗时都除以 4，算术强度不变。",{"type":150,"tag":151,"props":37418,"children":37419},{},[37420],{"type":155,"value":37421},"先把精度定死，因为算力和字节数都随它变：",{"type":150,"tag":22173,"props":37423,"children":37424},{},[37425,37444],{"type":150,"tag":22177,"props":37426,"children":37427},{},[37428],{"type":150,"tag":22181,"props":37429,"children":37430},{},[37431,37434,37439],{"type":150,"tag":22185,"props":37432,"children":37433},{},[],{"type":150,"tag":22185,"props":37435,"children":37436},{},[37437],{"type":155,"value":37438},"精度",{"type":150,"tag":22185,"props":37440,"children":37441},{},[37442],{"type":155,"value":37443},"依据",{"type":150,"tag":22199,"props":37445,"children":37446},{},[37447,37464,37482],{"type":150,"tag":22181,"props":37448,"children":37449},{},[37450,37454,37459],{"type":150,"tag":22206,"props":37451,"children":37452},{},[37453],{"type":155,"value":6072},{"type":150,"tag":22206,"props":37455,"children":37456},{},[37457],{"type":155,"value":37458},"FP4",{"type":150,"tag":22206,"props":37460,"children":37461},{},[37462],{"type":155,"value":37463},"每参数 0.5 字节",{"type":150,"tag":22181,"props":37465,"children":37466},{},[37467,37472,37477],{"type":150,"tag":22206,"props":37468,"children":37469},{},[37470],{"type":155,"value":37471},"KV cache",{"type":150,"tag":22206,"props":37473,"children":37474},{},[37475],{"type":155,"value":37476},"FP4（MXFP4）",{"type":150,"tag":22206,"props":37478,"children":37479},{},[37480],{"type":155,"value":37481},"技术报告的 QAT 就是按 MXFP4 做的，890 字节 \u002F token 是这个精度下的值",{"type":150,"tag":22181,"props":37483,"children":37484},{},[37485,37490,37494],{"type":150,"tag":22206,"props":37486,"children":37487},{},[37488],{"type":155,"value":37489},"算力基准",{"type":150,"tag":22206,"props":37491,"children":37492},{},[37493],{"type":155,"value":37458},{"type":150,"tag":22206,"props":37495,"children":37496},{},[37497],{"type":155,"value":37498},"B200 dense 9000 TFLOP\u002Fs",{"type":150,"tag":151,"props":37500,"children":37501},{},[37502],{"type":155,"value":37503},"三项同一把尺子，拐点和字节数才可比。",{"type":150,"tag":151,"props":37505,"children":37506},{},[37507],{"type":155,"value":37508},"B200 的 BF16 \u002F FP8 \u002F FP4 dense 算力分别是 2250 \u002F 4500 \u002F 9000 TFLOP\u002Fs，HBM3e 带宽 8 TB\u002Fs。FP4 的拐点是：",{"type":150,"tag":181,"props":37510,"children":37512},{"className":37511},[1003],[37513],{"type":150,"tag":181,"props":37514,"children":37516},{"className":37515},[184],[37517,37574],{"type":150,"tag":181,"props":37518,"children":37520},{"className":37519},[189],[37521],{"type":150,"tag":192,"props":37522,"children":37523},{"xmlns":194,"display":1015},[37524],{"type":150,"tag":197,"props":37525,"children":37526},{},[37527,37569],{"type":150,"tag":201,"props":37528,"children":37529},{},[37530,37556,37560,37565],{"type":150,"tag":1774,"props":37531,"children":37532},{},[37533,37545],{"type":150,"tag":201,"props":37534,"children":37535},{},[37536,37541],{"type":150,"tag":215,"props":37537,"children":37538},{},[37539],{"type":155,"value":37540},"9000",{"type":150,"tag":249,"props":37542,"children":37543},{},[37544],{"type":155,"value":23421},{"type":150,"tag":201,"props":37546,"children":37547},{},[37548,37552],{"type":150,"tag":215,"props":37549,"children":37550},{},[37551],{"type":155,"value":9202},{"type":150,"tag":249,"props":37553,"children":37554},{},[37555],{"type":155,"value":23434},{"type":150,"tag":221,"props":37557,"children":37558},{},[37559],{"type":155,"value":865},{"type":150,"tag":215,"props":37561,"children":37562},{},[37563],{"type":155,"value":37564},"1125",{"type":150,"tag":249,"props":37566,"children":37567},{},[37568],{"type":155,"value":23448},{"type":150,"tag":271,"props":37570,"children":37571},{"encoding":273},[37572],{"type":155,"value":37573},"\\frac{9000\\ \\text{TFLOPS}}{8\\ \\text{TB\u002Fs}} = 1125\\ \\text{FLOP\u002Fbyte}",{"type":150,"tag":181,"props":37575,"children":37577},{"className":37576,"ariaHidden":223},[280],[37578,37714],{"type":150,"tag":181,"props":37579,"children":37581},{"className":37580},[285],[37582,37586,37701,37705,37710],{"type":150,"tag":181,"props":37583,"children":37585},{"className":37584,"style":23465},[290],[],{"type":150,"tag":181,"props":37587,"children":37589},{"className":37588},[296],[37590,37594,37697],{"type":150,"tag":181,"props":37591,"children":37593},{"className":37592},[824,2482],[],{"type":150,"tag":181,"props":37595,"children":37597},{"className":37596},[1774],[37598],{"type":150,"tag":181,"props":37599,"children":37601},{"className":37600},[312,313],[37602,37686],{"type":150,"tag":181,"props":37603,"children":37605},{"className":37604},[318],[37606,37681],{"type":150,"tag":181,"props":37607,"children":37609},{"className":37608,"style":23490},[323],[37610,37640,37651],{"type":150,"tag":181,"props":37611,"children":37612},{"style":8235},[37613,37617],{"type":150,"tag":181,"props":37614,"children":37616},{"className":37615,"style":8240},[333],[],{"type":150,"tag":181,"props":37618,"children":37620},{"className":37619},[296],[37621,37626,37631],{"type":150,"tag":181,"props":37622,"children":37624},{"className":37623},[296],[37625],{"type":155,"value":9202},{"type":150,"tag":181,"props":37627,"children":37629},{"className":37628},[377],[37630],{"type":155,"value":8576},{"type":150,"tag":181,"props":37632,"children":37634},{"className":37633},[296,155],[37635],{"type":150,"tag":181,"props":37636,"children":37638},{"className":37637},[296],[37639],{"type":155,"value":23522},{"type":150,"tag":181,"props":37641,"children":37642},{"style":8253},[37643,37647],{"type":150,"tag":181,"props":37644,"children":37646},{"className":37645,"style":8240},[333],[],{"type":150,"tag":181,"props":37648,"children":37650},{"className":37649,"style":2804},[2803],[],{"type":150,"tag":181,"props":37652,"children":37653},{"style":8265},[37654,37658],{"type":150,"tag":181,"props":37655,"children":37657},{"className":37656,"style":8240},[333],[],{"type":150,"tag":181,"props":37659,"children":37661},{"className":37660},[296],[37662,37667,37672],{"type":150,"tag":181,"props":37663,"children":37665},{"className":37664},[296],[37666],{"type":155,"value":37540},{"type":150,"tag":181,"props":37668,"children":37670},{"className":37669},[377],[37671],{"type":155,"value":8576},{"type":150,"tag":181,"props":37673,"children":37675},{"className":37674},[296,155],[37676],{"type":150,"tag":181,"props":37677,"children":37679},{"className":37678},[296],[37680],{"type":155,"value":23564},{"type":150,"tag":181,"props":37682,"children":37684},{"className":37683},[352],[37685],{"type":155,"value":355},{"type":150,"tag":181,"props":37687,"children":37689},{"className":37688},[318],[37690],{"type":150,"tag":181,"props":37691,"children":37693},{"className":37692,"style":23577},[323],[37694],{"type":150,"tag":181,"props":37695,"children":37696},{},[],{"type":150,"tag":181,"props":37698,"children":37700},{"className":37699},[835,2482],[],{"type":150,"tag":181,"props":37702,"children":37704},{"className":37703,"style":658},[377],[],{"type":150,"tag":181,"props":37706,"children":37708},{"className":37707},[663],[37709],{"type":155,"value":865},{"type":150,"tag":181,"props":37711,"children":37713},{"className":37712,"style":658},[377],[],{"type":150,"tag":181,"props":37715,"children":37717},{"className":37716},[285],[37718,37722,37727,37732],{"type":150,"tag":181,"props":37719,"children":37721},{"className":37720,"style":819},[290],[],{"type":150,"tag":181,"props":37723,"children":37725},{"className":37724},[296],[37726],{"type":155,"value":37564},{"type":150,"tag":181,"props":37728,"children":37730},{"className":37729},[377],[37731],{"type":155,"value":8576},{"type":150,"tag":181,"props":37733,"children":37735},{"className":37734},[296,155],[37736],{"type":150,"tag":181,"props":37737,"children":37739},{"className":37738},[296],[37740],{"type":155,"value":23626},{"type":150,"tag":151,"props":37742,"children":37743},{},[37744],{"type":155,"value":37745},"decode 侧每 token 要搬多少？16B activated，FP4 存储：",{"type":150,"tag":181,"props":37747,"children":37749},{"className":37748},[1003],[37750],{"type":150,"tag":181,"props":37751,"children":37753},{"className":37752},[184],[37754,37805],{"type":150,"tag":181,"props":37755,"children":37757},{"className":37756},[189],[37758],{"type":150,"tag":192,"props":37759,"children":37760},{"xmlns":194,"display":1015},[37761],{"type":150,"tag":197,"props":37762,"children":37763},{},[37764,37800],{"type":150,"tag":201,"props":37765,"children":37766},{},[37767,37771,37775,37779,37784,37788,37792,37796],{"type":150,"tag":215,"props":37768,"children":37769},{},[37770],{"type":155,"value":35393},{"type":150,"tag":249,"props":37772,"children":37773},{},[37774],{"type":155,"value":23188},{"type":150,"tag":221,"props":37776,"children":37777},{},[37778],{"type":155,"value":5380},{"type":150,"tag":215,"props":37780,"children":37781},{},[37782],{"type":155,"value":37783},"0.5",{"type":150,"tag":249,"props":37785,"children":37786},{},[37787],{"type":155,"value":24730},{"type":150,"tag":221,"props":37789,"children":37790},{},[37791],{"type":155,"value":865},{"type":150,"tag":215,"props":37793,"children":37794},{},[37795],{"type":155,"value":9202},{"type":150,"tag":249,"props":37797,"children":37798},{},[37799],{"type":155,"value":29293},{"type":150,"tag":271,"props":37801,"children":37802},{"encoding":273},[37803],{"type":155,"value":37804},"16\\text{B} \\times 0.5\\ \\text{字节} = 8\\ \\text{GB}",{"type":150,"tag":181,"props":37806,"children":37808},{"className":37807,"ariaHidden":223},[280],[37809,37844,37884],{"type":150,"tag":181,"props":37810,"children":37812},{"className":37811},[285],[37813,37817,37822,37831,37835,37840],{"type":150,"tag":181,"props":37814,"children":37816},{"className":37815,"style":5470},[290],[],{"type":150,"tag":181,"props":37818,"children":37820},{"className":37819},[296],[37821],{"type":155,"value":35393},{"type":150,"tag":181,"props":37823,"children":37825},{"className":37824},[296,155],[37826],{"type":150,"tag":181,"props":37827,"children":37829},{"className":37828},[296],[37830],{"type":155,"value":23188},{"type":150,"tag":181,"props":37832,"children":37834},{"className":37833,"style":4993},[377],[],{"type":150,"tag":181,"props":37836,"children":37838},{"className":37837},[4998],[37839],{"type":155,"value":5380},{"type":150,"tag":181,"props":37841,"children":37843},{"className":37842,"style":4993},[377],[],{"type":150,"tag":181,"props":37845,"children":37847},{"className":37846},[285],[37848,37852,37857,37862,37871,37875,37880],{"type":150,"tag":181,"props":37849,"children":37851},{"className":37850,"style":5680},[290],[],{"type":150,"tag":181,"props":37853,"children":37855},{"className":37854},[296],[37856],{"type":155,"value":37783},{"type":150,"tag":181,"props":37858,"children":37860},{"className":37859},[377],[37861],{"type":155,"value":8576},{"type":150,"tag":181,"props":37863,"children":37865},{"className":37864},[296,155],[37866],{"type":150,"tag":181,"props":37867,"children":37869},{"className":37868},[296,24835],[37870],{"type":155,"value":24838},{"type":150,"tag":181,"props":37872,"children":37874},{"className":37873,"style":658},[377],[],{"type":150,"tag":181,"props":37876,"children":37878},{"className":37877},[663],[37879],{"type":155,"value":865},{"type":150,"tag":181,"props":37881,"children":37883},{"className":37882,"style":658},[377],[],{"type":150,"tag":181,"props":37885,"children":37887},{"className":37886},[285],[37888,37892,37897,37902],{"type":150,"tag":181,"props":37889,"children":37891},{"className":37890,"style":5680},[290],[],{"type":150,"tag":181,"props":37893,"children":37895},{"className":37894},[296],[37896],{"type":155,"value":9202},{"type":150,"tag":181,"props":37898,"children":37900},{"className":37899},[377],[37901],{"type":155,"value":8576},{"type":150,"tag":181,"props":37903,"children":37905},{"className":37904},[296,155],[37906],{"type":150,"tag":181,"props":37907,"children":37909},{"className":37908},[296],[37910],{"type":155,"value":29439},{"type":150,"tag":151,"props":37912,"children":37913},{},[37914,37916,38078],{"type":155,"value":37915},"KV 那边，1M context 时是 ",{"type":150,"tag":181,"props":37917,"children":37919},{"className":37918},[184],[37920,37968],{"type":150,"tag":181,"props":37921,"children":37923},{"className":37922},[189],[37924],{"type":150,"tag":192,"props":37925,"children":37926},{"xmlns":194},[37927],{"type":150,"tag":197,"props":37928,"children":37929},{},[37930,37963],{"type":150,"tag":201,"props":37931,"children":37932},{},[37933,37938,37942,37954,37958],{"type":150,"tag":215,"props":37934,"children":37935},{},[37936],{"type":155,"value":37937},"890",{"type":150,"tag":221,"props":37939,"children":37940},{},[37941],{"type":155,"value":5380},{"type":150,"tag":565,"props":37943,"children":37944},{},[37945,37949],{"type":150,"tag":215,"props":37946,"children":37947},{},[37948],{"type":155,"value":25633},{"type":150,"tag":215,"props":37950,"children":37951},{},[37952],{"type":155,"value":37953},"6",{"type":150,"tag":221,"props":37955,"children":37956},{},[37957],{"type":155,"value":5876},{"type":150,"tag":215,"props":37959,"children":37960},{},[37961],{"type":155,"value":37962},"0.93",{"type":150,"tag":271,"props":37964,"children":37965},{"encoding":273},[37966],{"type":155,"value":37967},"890 \\times 10^6 \\approx 0.93",{"type":150,"tag":181,"props":37969,"children":37971},{"className":37970,"ariaHidden":223},[280],[37972,37998,38065],{"type":150,"tag":181,"props":37973,"children":37975},{"className":37974},[285],[37976,37980,37985,37989,37994],{"type":150,"tag":181,"props":37977,"children":37979},{"className":37978,"style":5443},[290],[],{"type":150,"tag":181,"props":37981,"children":37983},{"className":37982},[296],[37984],{"type":155,"value":37937},{"type":150,"tag":181,"props":37986,"children":37988},{"className":37987,"style":4993},[377],[],{"type":150,"tag":181,"props":37990,"children":37992},{"className":37991},[4998],[37993],{"type":155,"value":5380},{"type":150,"tag":181,"props":37995,"children":37997},{"className":37996,"style":4993},[377],[],{"type":150,"tag":181,"props":37999,"children":38001},{"className":38000},[285],[38002,38006,38011,38052,38056,38061],{"type":150,"tag":181,"props":38003,"children":38005},{"className":38004,"style":6831},[290],[],{"type":150,"tag":181,"props":38007,"children":38009},{"className":38008},[296],[38010],{"type":155,"value":219},{"type":150,"tag":181,"props":38012,"children":38014},{"className":38013},[296],[38015,38020],{"type":150,"tag":181,"props":38016,"children":38018},{"className":38017},[296],[38019],{"type":155,"value":25836},{"type":150,"tag":181,"props":38021,"children":38023},{"className":38022},[307],[38024],{"type":150,"tag":181,"props":38025,"children":38027},{"className":38026},[312],[38028],{"type":150,"tag":181,"props":38029,"children":38031},{"className":38030},[318],[38032],{"type":150,"tag":181,"props":38033,"children":38035},{"className":38034,"style":6831},[323],[38036],{"type":150,"tag":181,"props":38037,"children":38038},{"style":707},[38039,38043],{"type":150,"tag":181,"props":38040,"children":38042},{"className":38041,"style":334},[333],[],{"type":150,"tag":181,"props":38044,"children":38046},{"className":38045},[339,340,341,342],[38047],{"type":150,"tag":181,"props":38048,"children":38050},{"className":38049},[296,342],[38051],{"type":155,"value":37953},{"type":150,"tag":181,"props":38053,"children":38055},{"className":38054,"style":658},[377],[],{"type":150,"tag":181,"props":38057,"children":38059},{"className":38058},[663],[38060],{"type":155,"value":5876},{"type":150,"tag":181,"props":38062,"children":38064},{"className":38063,"style":658},[377],[],{"type":150,"tag":181,"props":38066,"children":38068},{"className":38067},[285],[38069,38073],{"type":150,"tag":181,"props":38070,"children":38072},{"className":38071,"style":5012},[290],[],{"type":150,"tag":181,"props":38074,"children":38076},{"className":38075},[296],[38077],{"type":155,"value":37962},{"type":155,"value":38079}," GB。加起来 8.93 GB。算术强度：",{"type":150,"tag":181,"props":38081,"children":38083},{"className":38082},[1003],[38084],{"type":150,"tag":181,"props":38085,"children":38087},{"className":38086},[184],[38088,38161],{"type":150,"tag":181,"props":38089,"children":38091},{"className":38090},[189],[38092],{"type":150,"tag":192,"props":38093,"children":38094},{"xmlns":194,"display":1015},[38095],{"type":150,"tag":197,"props":38096,"children":38097},{},[38098,38156],{"type":150,"tag":201,"props":38099,"children":38100},{},[38101,38105,38109,38143,38147,38152],{"type":150,"tag":209,"props":38102,"children":38103},{},[38104],{"type":155,"value":24937},{"type":150,"tag":221,"props":38106,"children":38107},{},[38108],{"type":155,"value":865},{"type":150,"tag":1774,"props":38110,"children":38111},{},[38112,38131],{"type":150,"tag":201,"props":38113,"children":38114},{},[38115,38119,38123,38127],{"type":150,"tag":215,"props":38116,"children":38117},{},[38118],{"type":155,"value":238},{"type":150,"tag":221,"props":38120,"children":38121},{},[38122],{"type":155,"value":5380},{"type":150,"tag":215,"props":38124,"children":38125},{},[38126],{"type":155,"value":35393},{"type":150,"tag":249,"props":38128,"children":38129},{},[38130],{"type":155,"value":23188},{"type":150,"tag":201,"props":38132,"children":38133},{},[38134,38139],{"type":150,"tag":215,"props":38135,"children":38136},{},[38137],{"type":155,"value":38138},"8.93",{"type":150,"tag":249,"props":38140,"children":38141},{},[38142],{"type":155,"value":29293},{"type":150,"tag":221,"props":38144,"children":38145},{},[38146],{"type":155,"value":5876},{"type":150,"tag":215,"props":38148,"children":38149},{},[38150],{"type":155,"value":38151},"3.6",{"type":150,"tag":249,"props":38153,"children":38154},{},[38155],{"type":155,"value":23448},{"type":150,"tag":271,"props":38157,"children":38158},{"encoding":273},[38159],{"type":155,"value":38160},"I = \\frac{2 \\times 16\\text{B}}{8.93\\ \\text{GB}} \\approx 3.6\\ \\text{FLOP\u002Fbyte}",{"type":150,"tag":181,"props":38162,"children":38164},{"className":38163,"ariaHidden":223},[280],[38165,38191,38340],{"type":150,"tag":181,"props":38166,"children":38168},{"className":38167},[285],[38169,38173,38178,38182,38187],{"type":150,"tag":181,"props":38170,"children":38172},{"className":38171,"style":5680},[290],[],{"type":150,"tag":181,"props":38174,"children":38176},{"className":38175,"style":24957},[296,301],[38177],{"type":155,"value":24937},{"type":150,"tag":181,"props":38179,"children":38181},{"className":38180,"style":658},[377],[],{"type":150,"tag":181,"props":38183,"children":38185},{"className":38184},[663],[38186],{"type":155,"value":865},{"type":150,"tag":181,"props":38188,"children":38190},{"className":38189,"style":658},[377],[],{"type":150,"tag":181,"props":38192,"children":38194},{"className":38193},[285],[38195,38199,38327,38331,38336],{"type":150,"tag":181,"props":38196,"children":38198},{"className":38197,"style":31545},[290],[],{"type":150,"tag":181,"props":38200,"children":38202},{"className":38201},[296],[38203,38207,38323],{"type":150,"tag":181,"props":38204,"children":38206},{"className":38205},[824,2482],[],{"type":150,"tag":181,"props":38208,"children":38210},{"className":38209},[1774],[38211],{"type":150,"tag":181,"props":38212,"children":38214},{"className":38213},[312,313],[38215,38312],{"type":150,"tag":181,"props":38216,"children":38218},{"className":38217},[318],[38219,38307],{"type":150,"tag":181,"props":38220,"children":38222},{"className":38221,"style":23490},[323],[38223,38253,38264],{"type":150,"tag":181,"props":38224,"children":38225},{"style":8235},[38226,38230],{"type":150,"tag":181,"props":38227,"children":38229},{"className":38228,"style":8240},[333],[],{"type":150,"tag":181,"props":38231,"children":38233},{"className":38232},[296],[38234,38239,38244],{"type":150,"tag":181,"props":38235,"children":38237},{"className":38236},[296],[38238],{"type":155,"value":38138},{"type":150,"tag":181,"props":38240,"children":38242},{"className":38241},[377],[38243],{"type":155,"value":8576},{"type":150,"tag":181,"props":38245,"children":38247},{"className":38246},[296,155],[38248],{"type":150,"tag":181,"props":38249,"children":38251},{"className":38250},[296],[38252],{"type":155,"value":29439},{"type":150,"tag":181,"props":38254,"children":38255},{"style":8253},[38256,38260],{"type":150,"tag":181,"props":38257,"children":38259},{"className":38258,"style":8240},[333],[],{"type":150,"tag":181,"props":38261,"children":38263},{"className":38262,"style":2804},[2803],[],{"type":150,"tag":181,"props":38265,"children":38266},{"style":8265},[38267,38271],{"type":150,"tag":181,"props":38268,"children":38270},{"className":38269,"style":8240},[333],[],{"type":150,"tag":181,"props":38272,"children":38274},{"className":38273},[296],[38275,38280,38284,38289,38293,38298],{"type":150,"tag":181,"props":38276,"children":38278},{"className":38277},[296],[38279],{"type":155,"value":238},{"type":150,"tag":181,"props":38281,"children":38283},{"className":38282,"style":4993},[377],[],{"type":150,"tag":181,"props":38285,"children":38287},{"className":38286},[4998],[38288],{"type":155,"value":5380},{"type":150,"tag":181,"props":38290,"children":38292},{"className":38291,"style":4993},[377],[],{"type":150,"tag":181,"props":38294,"children":38296},{"className":38295},[296],[38297],{"type":155,"value":35393},{"type":150,"tag":181,"props":38299,"children":38301},{"className":38300},[296,155],[38302],{"type":150,"tag":181,"props":38303,"children":38305},{"className":38304},[296],[38306],{"type":155,"value":23188},{"type":150,"tag":181,"props":38308,"children":38310},{"className":38309},[352],[38311],{"type":155,"value":355},{"type":150,"tag":181,"props":38313,"children":38315},{"className":38314},[318],[38316],{"type":150,"tag":181,"props":38317,"children":38319},{"className":38318,"style":8297},[323],[38320],{"type":150,"tag":181,"props":38321,"children":38322},{},[],{"type":150,"tag":181,"props":38324,"children":38326},{"className":38325},[835,2482],[],{"type":150,"tag":181,"props":38328,"children":38330},{"className":38329,"style":658},[377],[],{"type":150,"tag":181,"props":38332,"children":38334},{"className":38333},[663],[38335],{"type":155,"value":5876},{"type":150,"tag":181,"props":38337,"children":38339},{"className":38338,"style":658},[377],[],{"type":150,"tag":181,"props":38341,"children":38343},{"className":38342},[285],[38344,38348,38353,38358],{"type":150,"tag":181,"props":38345,"children":38347},{"className":38346,"style":819},[290],[],{"type":150,"tag":181,"props":38349,"children":38351},{"className":38350},[296],[38352],{"type":155,"value":38151},{"type":150,"tag":181,"props":38354,"children":38356},{"className":38355},[377],[38357],{"type":155,"value":8576},{"type":150,"tag":181,"props":38359,"children":38361},{"className":38360},[296,155],[38362],{"type":150,"tag":181,"props":38363,"children":38365},{"className":38364},[296],[38366],{"type":155,"value":23626},{"type":150,"tag":151,"props":38368,"children":38369},{},[38370],{"type":155,"value":38371},"按 8 TB\u002Fs 的带宽上限，单流大约 900 token\u002Fs。",{"type":150,"tag":151,"props":38373,"children":38374},{},[38375,38380],{"type":150,"tag":158,"props":38376,"children":38377},{},[38378],{"type":155,"value":38379},"3.6 对 1125，差了 300 倍。",{"type":155,"value":38381}," B200 的 FP4 算力有 99.7% 用不上。",{"type":150,"tag":151,"props":38383,"children":38384},{},[38385],{"type":155,"value":38386},"换权重精度结论也一样。KV 那 890 字节 \u002F token 已经是 FP4 下的实测值，所以只动权重：",{"type":150,"tag":22173,"props":38388,"children":38389},{},[38390,38420],{"type":150,"tag":22177,"props":38391,"children":38392},{},[38393],{"type":150,"tag":22181,"props":38394,"children":38395},{},[38396,38401,38406,38411,38415],{"type":150,"tag":22185,"props":38397,"children":38398},{},[38399],{"type":155,"value":38400},"权重精度",{"type":150,"tag":22185,"props":38402,"children":38403},{},[38404],{"type":155,"value":38405},"权重字节 \u002F token",{"type":150,"tag":22185,"props":38407,"children":38408},{},[38409],{"type":155,"value":38410},"拐点",{"type":150,"tag":22185,"props":38412,"children":38413},{},[38414],{"type":155,"value":26889},{"type":150,"tag":22185,"props":38416,"children":38417},{},[38418],{"type":155,"value":38419},"差几倍",{"type":150,"tag":22199,"props":38421,"children":38422},{},[38423,38449,38477],{"type":150,"tag":22181,"props":38424,"children":38425},{},[38426,38430,38435,38439,38444],{"type":150,"tag":22206,"props":38427,"children":38428},{},[38429],{"type":155,"value":37458},{"type":150,"tag":22206,"props":38431,"children":38432},{},[38433],{"type":155,"value":38434},"8.0 GB",{"type":150,"tag":22206,"props":38436,"children":38437},{},[38438],{"type":155,"value":37564},{"type":150,"tag":22206,"props":38440,"children":38441},{},[38442],{"type":155,"value":38443},"3.58",{"type":150,"tag":22206,"props":38445,"children":38446},{},[38447],{"type":155,"value":38448},"314×",{"type":150,"tag":22181,"props":38450,"children":38451},{},[38452,38457,38462,38467,38472],{"type":150,"tag":22206,"props":38453,"children":38454},{},[38455],{"type":155,"value":38456},"FP8",{"type":150,"tag":22206,"props":38458,"children":38459},{},[38460],{"type":155,"value":38461},"16.0 GB",{"type":150,"tag":22206,"props":38463,"children":38464},{},[38465],{"type":155,"value":38466},"562",{"type":150,"tag":22206,"props":38468,"children":38469},{},[38470],{"type":155,"value":38471},"1.89",{"type":150,"tag":22206,"props":38473,"children":38474},{},[38475],{"type":155,"value":38476},"298×",{"type":150,"tag":22181,"props":38478,"children":38479},{},[38480,38485,38490,38495,38500],{"type":150,"tag":22206,"props":38481,"children":38482},{},[38483],{"type":155,"value":38484},"BF16",{"type":150,"tag":22206,"props":38486,"children":38487},{},[38488],{"type":155,"value":38489},"32.0 GB",{"type":150,"tag":22206,"props":38491,"children":38492},{},[38493],{"type":155,"value":38494},"281",{"type":150,"tag":22206,"props":38496,"children":38497},{},[38498],{"type":155,"value":38499},"0.97",{"type":150,"tag":22206,"props":38501,"children":38502},{},[38503],{"type":155,"value":38504},"289×",{"type":150,"tag":151,"props":38506,"children":38507},{},[38508,38510,38759,38761,38766],{"type":155,"value":38509},"精度降一半，拐点翻一倍，但算术强度也跟着掉一半，两边抵消。根源是 B200 上这三个精度的「算力 × 每参字节」都是同一个数：",{"type":150,"tag":181,"props":38511,"children":38513},{"className":38512},[184],[38514,38586],{"type":150,"tag":181,"props":38515,"children":38517},{"className":38516},[189],[38518],{"type":150,"tag":192,"props":38519,"children":38520},{"xmlns":194},[38521],{"type":150,"tag":197,"props":38522,"children":38523},{},[38524,38581],{"type":150,"tag":201,"props":38525,"children":38526},{},[38527,38531,38535,38539,38543,38548,38552,38556,38560,38565,38569,38573,38577],{"type":150,"tag":215,"props":38528,"children":38529},{},[38530],{"type":155,"value":37540},{"type":150,"tag":221,"props":38532,"children":38533},{},[38534],{"type":155,"value":5380},{"type":150,"tag":215,"props":38536,"children":38537},{},[38538],{"type":155,"value":37783},{"type":150,"tag":221,"props":38540,"children":38541},{},[38542],{"type":155,"value":865},{"type":150,"tag":215,"props":38544,"children":38545},{},[38546],{"type":155,"value":38547},"4500",{"type":150,"tag":221,"props":38549,"children":38550},{},[38551],{"type":155,"value":5380},{"type":150,"tag":215,"props":38553,"children":38554},{},[38555],{"type":155,"value":219},{"type":150,"tag":221,"props":38557,"children":38558},{},[38559],{"type":155,"value":865},{"type":150,"tag":215,"props":38561,"children":38562},{},[38563],{"type":155,"value":38564},"2250",{"type":150,"tag":221,"props":38566,"children":38567},{},[38568],{"type":155,"value":5380},{"type":150,"tag":215,"props":38570,"children":38571},{},[38572],{"type":155,"value":238},{"type":150,"tag":221,"props":38574,"children":38575},{},[38576],{"type":155,"value":865},{"type":150,"tag":215,"props":38578,"children":38579},{},[38580],{"type":155,"value":38547},{"type":150,"tag":271,"props":38582,"children":38583},{"encoding":273},[38584],{"type":155,"value":38585},"9000 \\times 0.5 = 4500 \\times 1 = 2250 \\times 2 = 4500",{"type":150,"tag":181,"props":38587,"children":38589},{"className":38588,"ariaHidden":223},[280],[38590,38616,38642,38668,38694,38720,38746],{"type":150,"tag":181,"props":38591,"children":38593},{"className":38592},[285],[38594,38598,38603,38607,38612],{"type":150,"tag":181,"props":38595,"children":38597},{"className":38596,"style":5443},[290],[],{"type":150,"tag":181,"props":38599,"children":38601},{"className":38600},[296],[38602],{"type":155,"value":37540},{"type":150,"tag":181,"props":38604,"children":38606},{"className":38605,"style":4993},[377],[],{"type":150,"tag":181,"props":38608,"children":38610},{"className":38609},[4998],[38611],{"type":155,"value":5380},{"type":150,"tag":181,"props":38613,"children":38615},{"className":38614,"style":4993},[377],[],{"type":150,"tag":181,"props":38617,"children":38619},{"className":38618},[285],[38620,38624,38629,38633,38638],{"type":150,"tag":181,"props":38621,"children":38623},{"className":38622,"style":5012},[290],[],{"type":150,"tag":181,"props":38625,"children":38627},{"className":38626},[296],[38628],{"type":155,"value":37783},{"type":150,"tag":181,"props":38630,"children":38632},{"className":38631,"style":658},[377],[],{"type":150,"tag":181,"props":38634,"children":38636},{"className":38635},[663],[38637],{"type":155,"value":865},{"type":150,"tag":181,"props":38639,"children":38641},{"className":38640,"style":658},[377],[],{"type":150,"tag":181,"props":38643,"children":38645},{"className":38644},[285],[38646,38650,38655,38659,38664],{"type":150,"tag":181,"props":38647,"children":38649},{"className":38648,"style":5443},[290],[],{"type":150,"tag":181,"props":38651,"children":38653},{"className":38652},[296],[38654],{"type":155,"value":38547},{"type":150,"tag":181,"props":38656,"children":38658},{"className":38657,"style":4993},[377],[],{"type":150,"tag":181,"props":38660,"children":38662},{"className":38661},[4998],[38663],{"type":155,"value":5380},{"type":150,"tag":181,"props":38665,"children":38667},{"className":38666,"style":4993},[377],[],{"type":150,"tag":181,"props":38669,"children":38671},{"className":38670},[285],[38672,38676,38681,38685,38690],{"type":150,"tag":181,"props":38673,"children":38675},{"className":38674,"style":5012},[290],[],{"type":150,"tag":181,"props":38677,"children":38679},{"className":38678},[296],[38680],{"type":155,"value":219},{"type":150,"tag":181,"props":38682,"children":38684},{"className":38683,"style":658},[377],[],{"type":150,"tag":181,"props":38686,"children":38688},{"className":38687},[663],[38689],{"type":155,"value":865},{"type":150,"tag":181,"props":38691,"children":38693},{"className":38692,"style":658},[377],[],{"type":150,"tag":181,"props":38695,"children":38697},{"className":38696},[285],[38698,38702,38707,38711,38716],{"type":150,"tag":181,"props":38699,"children":38701},{"className":38700,"style":5443},[290],[],{"type":150,"tag":181,"props":38703,"children":38705},{"className":38704},[296],[38706],{"type":155,"value":38564},{"type":150,"tag":181,"props":38708,"children":38710},{"className":38709,"style":4993},[377],[],{"type":150,"tag":181,"props":38712,"children":38714},{"className":38713},[4998],[38715],{"type":155,"value":5380},{"type":150,"tag":181,"props":38717,"children":38719},{"className":38718,"style":4993},[377],[],{"type":150,"tag":181,"props":38721,"children":38723},{"className":38722},[285],[38724,38728,38733,38737,38742],{"type":150,"tag":181,"props":38725,"children":38727},{"className":38726,"style":5012},[290],[],{"type":150,"tag":181,"props":38729,"children":38731},{"className":38730},[296],[38732],{"type":155,"value":238},{"type":150,"tag":181,"props":38734,"children":38736},{"className":38735,"style":658},[377],[],{"type":150,"tag":181,"props":38738,"children":38740},{"className":38739},[663],[38741],{"type":155,"value":865},{"type":150,"tag":181,"props":38743,"children":38745},{"className":38744,"style":658},[377],[],{"type":150,"tag":181,"props":38747,"children":38749},{"className":38748},[285],[38750,38754],{"type":150,"tag":181,"props":38751,"children":38753},{"className":38752,"style":5012},[290],[],{"type":150,"tag":181,"props":38755,"children":38757},{"className":38756},[296],[38758],{"type":155,"value":38547},{"type":155,"value":38760},"。所以",{"type":150,"tag":158,"props":38762,"children":38763},{},[38764],{"type":155,"value":38765},"差多少倍基本由架构决定，不由精度决定",{"type":155,"value":723},{"type":150,"tag":151,"props":38768,"children":38769},{},[38770],{"type":155,"value":38771},"流量拆开看：",{"type":150,"tag":22173,"props":38773,"children":38774},{},[38775,38795],{"type":150,"tag":22177,"props":38776,"children":38777},{},[38778],{"type":150,"tag":22181,"props":38779,"children":38780},{},[38781,38786,38790],{"type":150,"tag":22185,"props":38782,"children":38783},{},[38784],{"type":155,"value":38785},"1M context 下每 token 的流量",{"type":150,"tag":22185,"props":38787,"children":38788},{},[38789],{"type":155,"value":24838},{"type":150,"tag":22185,"props":38791,"children":38792},{},[38793],{"type":155,"value":38794},"占比",{"type":150,"tag":22199,"props":38796,"children":38797},{},[38798,38815],{"type":150,"tag":22181,"props":38799,"children":38800},{},[38801,38806,38810],{"type":150,"tag":22206,"props":38802,"children":38803},{},[38804],{"type":155,"value":38805},"权重",{"type":150,"tag":22206,"props":38807,"children":38808},{},[38809],{"type":155,"value":38434},{"type":150,"tag":22206,"props":38811,"children":38812},{},[38813],{"type":155,"value":38814},"90%",{"type":150,"tag":22181,"props":38816,"children":38817},{},[38818,38823,38828],{"type":150,"tag":22206,"props":38819,"children":38820},{},[38821],{"type":155,"value":38822},"KV",{"type":150,"tag":22206,"props":38824,"children":38825},{},[38826],{"type":155,"value":38827},"0.93 GB",{"type":150,"tag":22206,"props":38829,"children":38830},{},[38831],{"type":155,"value":38832},"10%",{"type":150,"tag":151,"props":38834,"children":38835},{},[38836,38838,38843],{"type":155,"value":38837},"模型结构和硬件的 co-design 之后，",{"type":150,"tag":158,"props":38839,"children":38840},{},[38841],{"type":155,"value":38842},"KV 已经不是瓶颈",{"type":155,"value":723},{"type":150,"tag":151,"props":38845,"children":38846},{},[38847],{"type":155,"value":38848},"ds4.1f 把 KV 压到 890 字节 \u002F token 之后，就算跑到 1M context，它也只占总流量的 10%。再压一半，总时间只少 6%；压到四分之一，也只少 9%。",{"type":150,"tag":151,"props":38850,"children":38851},{},[38852],{"type":155,"value":38853},"而权重占 90%。把它减半，就是接近一倍的提速。",{"type":150,"tag":151,"props":38855,"children":38856},{},[38857],{"type":155,"value":38858},"字节数直接决定速度。decode 是 memory bound，每个 token 的耗时就是总字节除以带宽：",{"type":150,"tag":22173,"props":38860,"children":38861},{},[38862,38936],{"type":150,"tag":22177,"props":38863,"children":38864},{},[38865],{"type":150,"tag":22181,"props":38866,"children":38867},{},[38868,38913,38917,38921,38926,38931],{"type":150,"tag":22185,"props":38869,"children":38870},{},[38871],{"type":150,"tag":181,"props":38872,"children":38874},{"className":38873},[184],[38875,38896],{"type":150,"tag":181,"props":38876,"children":38878},{"className":38877},[189],[38879],{"type":150,"tag":192,"props":38880,"children":38881},{"xmlns":194},[38882],{"type":150,"tag":197,"props":38883,"children":38884},{},[38885,38892],{"type":150,"tag":201,"props":38886,"children":38887},{},[38888],{"type":150,"tag":209,"props":38889,"children":38890},{},[38891],{"type":155,"value":5417},{"type":150,"tag":271,"props":38893,"children":38894},{"encoding":273},[38895],{"type":155,"value":5417},{"type":150,"tag":181,"props":38897,"children":38899},{"className":38898,"ariaHidden":223},[280],[38900],{"type":150,"tag":181,"props":38901,"children":38903},{"className":38902},[285],[38904,38908],{"type":150,"tag":181,"props":38905,"children":38907},{"className":38906,"style":5680},[290],[],{"type":150,"tag":181,"props":38909,"children":38911},{"className":38910,"style":5606},[296,301],[38912],{"type":155,"value":5417},{"type":150,"tag":22185,"props":38914,"children":38915},{},[38916],{"type":155,"value":38805},{"type":150,"tag":22185,"props":38918,"children":38919},{},[38920],{"type":155,"value":38822},{"type":150,"tag":22185,"props":38922,"children":38923},{},[38924],{"type":155,"value":38925},"合计",{"type":150,"tag":22185,"props":38927,"children":38928},{},[38929],{"type":155,"value":38930},"每 token",{"type":150,"tag":22185,"props":38932,"children":38933},{},[38934],{"type":155,"value":38935},"单流 tok\u002Fs",{"type":150,"tag":22199,"props":38937,"children":38938},{},[38939,38970,39000,39031],{"type":150,"tag":22181,"props":38940,"children":38941},{},[38942,38946,38951,38956,38960,38965],{"type":150,"tag":22206,"props":38943,"children":38944},{},[38945],{"type":155,"value":26915},{"type":150,"tag":22206,"props":38947,"children":38948},{},[38949],{"type":155,"value":38950},"8.00 GB",{"type":150,"tag":22206,"props":38952,"children":38953},{},[38954],{"type":155,"value":38955},"0.004 GB",{"type":150,"tag":22206,"props":38957,"children":38958},{},[38959],{"type":155,"value":38950},{"type":150,"tag":22206,"props":38961,"children":38962},{},[38963],{"type":155,"value":38964},"1.00 ms",{"type":150,"tag":22206,"props":38966,"children":38967},{},[38968],{"type":155,"value":38969},"1000",{"type":150,"tag":22181,"props":38971,"children":38972},{},[38973,38977,38981,38986,38991,38995],{"type":150,"tag":22206,"props":38974,"children":38975},{},[38976],{"type":155,"value":26943},{"type":150,"tag":22206,"props":38978,"children":38979},{},[38980],{"type":155,"value":38950},{"type":150,"tag":22206,"props":38982,"children":38983},{},[38984],{"type":155,"value":38985},"0.029 GB",{"type":150,"tag":22206,"props":38987,"children":38988},{},[38989],{"type":155,"value":38990},"8.03 GB",{"type":150,"tag":22206,"props":38992,"children":38993},{},[38994],{"type":155,"value":38964},{"type":150,"tag":22206,"props":38996,"children":38997},{},[38998],{"type":155,"value":38999},"996",{"type":150,"tag":22181,"props":39001,"children":39002},{},[39003,39007,39011,39016,39021,39026],{"type":150,"tag":22206,"props":39004,"children":39005},{},[39006],{"type":155,"value":26970},{"type":150,"tag":22206,"props":39008,"children":39009},{},[39010],{"type":155,"value":38950},{"type":150,"tag":22206,"props":39012,"children":39013},{},[39014],{"type":155,"value":39015},"0.117 GB",{"type":150,"tag":22206,"props":39017,"children":39018},{},[39019],{"type":155,"value":39020},"8.12 GB",{"type":150,"tag":22206,"props":39022,"children":39023},{},[39024],{"type":155,"value":39025},"1.01 ms",{"type":150,"tag":22206,"props":39027,"children":39028},{},[39029],{"type":155,"value":39030},"986",{"type":150,"tag":22181,"props":39032,"children":39033},{},[39034,39038,39042,39047,39052,39057],{"type":150,"tag":22206,"props":39035,"children":39036},{},[39037],{"type":155,"value":26997},{"type":150,"tag":22206,"props":39039,"children":39040},{},[39041],{"type":155,"value":38950},{"type":150,"tag":22206,"props":39043,"children":39044},{},[39045],{"type":155,"value":39046},"0.933 GB",{"type":150,"tag":22206,"props":39048,"children":39049},{},[39050],{"type":155,"value":39051},"8.93 GB",{"type":150,"tag":22206,"props":39053,"children":39054},{},[39055],{"type":155,"value":39056},"1.12 ms",{"type":150,"tag":22206,"props":39058,"children":39059},{},[39060],{"type":155,"value":39061},"896",{"type":150,"tag":151,"props":39063,"children":39064},{},[39065,39070],{"type":150,"tag":158,"props":39066,"children":39067},{},[39068],{"type":155,"value":39069},"从 4k 到 1M，单流速度只掉 10%。",{"type":155,"value":39071}," KV 压缩买到的是跑长 context 不掉速，能跑起来只是附带的结果。",{"type":150,"tag":151,"props":39073,"children":39074},{},[39075],{"type":155,"value":39076},"再看容量。单卡 172 GB 可用显存，按 890 字节 \u002F token：",{"type":150,"tag":181,"props":39078,"children":39080},{"className":39079},[1003],[39081],{"type":150,"tag":181,"props":39082,"children":39084},{"className":39083},[184],[39085,39144],{"type":150,"tag":181,"props":39086,"children":39088},{"className":39087},[189],[39089],{"type":150,"tag":192,"props":39090,"children":39091},{"xmlns":194,"display":1015},[39092],{"type":150,"tag":197,"props":39093,"children":39094},{},[39095,39139],{"type":150,"tag":201,"props":39096,"children":39097},{},[39098,39125,39129,39134],{"type":150,"tag":1774,"props":39099,"children":39100},{},[39101,39113],{"type":150,"tag":201,"props":39102,"children":39103},{},[39104,39109],{"type":150,"tag":215,"props":39105,"children":39106},{},[39107],{"type":155,"value":39108},"172",{"type":150,"tag":249,"props":39110,"children":39111},{},[39112],{"type":155,"value":29293},{"type":150,"tag":201,"props":39114,"children":39115},{},[39116,39120],{"type":150,"tag":215,"props":39117,"children":39118},{},[39119],{"type":155,"value":37937},{"type":150,"tag":249,"props":39121,"children":39122},{},[39123],{"type":155,"value":39124}," B",{"type":150,"tag":221,"props":39126,"children":39127},{},[39128],{"type":155,"value":5876},{"type":150,"tag":215,"props":39130,"children":39131},{},[39132],{"type":155,"value":39133},"1.93",{"type":150,"tag":249,"props":39135,"children":39136},{},[39137],{"type":155,"value":39138}," 亿 token",{"type":150,"tag":271,"props":39140,"children":39141},{"encoding":273},[39142],{"type":155,"value":39143},"\\frac{172\\ \\text{GB}}{890\\ \\text{B}} \\approx 1.93\\ \\text{亿 token}",{"type":150,"tag":181,"props":39145,"children":39147},{"className":39146,"ariaHidden":223},[280],[39148,39284],{"type":150,"tag":181,"props":39149,"children":39151},{"className":39150},[285],[39152,39156,39271,39275,39280],{"type":150,"tag":181,"props":39153,"children":39155},{"className":39154,"style":31545},[290],[],{"type":150,"tag":181,"props":39157,"children":39159},{"className":39158},[296],[39160,39164,39267],{"type":150,"tag":181,"props":39161,"children":39163},{"className":39162},[824,2482],[],{"type":150,"tag":181,"props":39165,"children":39167},{"className":39166},[1774],[39168],{"type":150,"tag":181,"props":39169,"children":39171},{"className":39170},[312,313],[39172,39256],{"type":150,"tag":181,"props":39173,"children":39175},{"className":39174},[318],[39176,39251],{"type":150,"tag":181,"props":39177,"children":39179},{"className":39178,"style":23490},[323],[39180,39210,39221],{"type":150,"tag":181,"props":39181,"children":39182},{"style":8235},[39183,39187],{"type":150,"tag":181,"props":39184,"children":39186},{"className":39185,"style":8240},[333],[],{"type":150,"tag":181,"props":39188,"children":39190},{"className":39189},[296],[39191,39196,39201],{"type":150,"tag":181,"props":39192,"children":39194},{"className":39193},[296],[39195],{"type":155,"value":37937},{"type":150,"tag":181,"props":39197,"children":39199},{"className":39198},[377],[39200],{"type":155,"value":8576},{"type":150,"tag":181,"props":39202,"children":39204},{"className":39203},[296,155],[39205],{"type":150,"tag":181,"props":39206,"children":39208},{"className":39207},[296],[39209],{"type":155,"value":23188},{"type":150,"tag":181,"props":39211,"children":39212},{"style":8253},[39213,39217],{"type":150,"tag":181,"props":39214,"children":39216},{"className":39215,"style":8240},[333],[],{"type":150,"tag":181,"props":39218,"children":39220},{"className":39219,"style":2804},[2803],[],{"type":150,"tag":181,"props":39222,"children":39223},{"style":8265},[39224,39228],{"type":150,"tag":181,"props":39225,"children":39227},{"className":39226,"style":8240},[333],[],{"type":150,"tag":181,"props":39229,"children":39231},{"className":39230},[296],[39232,39237,39242],{"type":150,"tag":181,"props":39233,"children":39235},{"className":39234},[296],[39236],{"type":155,"value":39108},{"type":150,"tag":181,"props":39238,"children":39240},{"className":39239},[377],[39241],{"type":155,"value":8576},{"type":150,"tag":181,"props":39243,"children":39245},{"className":39244},[296,155],[39246],{"type":150,"tag":181,"props":39247,"children":39249},{"className":39248},[296],[39250],{"type":155,"value":29439},{"type":150,"tag":181,"props":39252,"children":39254},{"className":39253},[352],[39255],{"type":155,"value":355},{"type":150,"tag":181,"props":39257,"children":39259},{"className":39258},[318],[39260],{"type":150,"tag":181,"props":39261,"children":39263},{"className":39262,"style":8297},[323],[39264],{"type":150,"tag":181,"props":39265,"children":39266},{},[],{"type":150,"tag":181,"props":39268,"children":39270},{"className":39269},[835,2482],[],{"type":150,"tag":181,"props":39272,"children":39274},{"className":39273,"style":658},[377],[],{"type":150,"tag":181,"props":39276,"children":39278},{"className":39277},[663],[39279],{"type":155,"value":5876},{"type":150,"tag":181,"props":39281,"children":39283},{"className":39282,"style":658},[377],[],{"type":150,"tag":181,"props":39285,"children":39287},{"className":39286},[285],[39288,39292,39297,39302],{"type":150,"tag":181,"props":39289,"children":39291},{"className":39290,"style":761},[290],[],{"type":150,"tag":181,"props":39293,"children":39295},{"className":39294},[296],[39296],{"type":155,"value":39133},{"type":150,"tag":181,"props":39298,"children":39300},{"className":39299},[377],[39301],{"type":155,"value":8576},{"type":150,"tag":181,"props":39303,"children":39305},{"className":39304},[296,155],[39306,39312],{"type":150,"tag":181,"props":39307,"children":39309},{"className":39308},[296,24835],[39310],{"type":155,"value":39311},"亿",{"type":150,"tag":181,"props":39313,"children":39315},{"className":39314},[296],[39316],{"type":155,"value":39317}," token",{"type":150,"tag":151,"props":39319,"children":39320},{},[39321],{"type":155,"value":39322},"一张卡装得下 1.93 亿 token 的 global KV。",{"type":150,"tag":151,"props":39324,"children":39325},{},[39326],{"type":155,"value":39327},"但容量不是硬上限。装不下就往下沉，host DRAM、SSD 一级一级放；请求再来时命中 prefix cache，把 KV 搬回 HBM 就能接着算。技术报告里那两个数本来就是按这个层级给的：常驻 HBM 的 global KV 是 890 字节 \u002F token，而落在 SSD 或 host memory 的 persistent KV 再压到 V4-Flash 的 1\u002F8。",{"type":150,"tag":151,"props":39329,"children":39330},{},[39331],{"type":150,"tag":158,"props":39332,"children":39333},{},[39334],{"type":155,"value":39335},"KV 这一层已经从\"装不下就崩\"的硬约束，变成了一个分层的存储问题。",{"type":150,"tag":151,"props":39337,"children":39338},{},[39339],{"type":155,"value":39340},"所以这条线走到这里，战场又换了一次位置：从\"压 KV\"换到\"压权重\"。前面几代解决的是历史存成什么样，接下来要解决的是 16B activated 这个数本身。",{"type":150,"tag":151,"props":39342,"children":39343},{},[39344],{"type":155,"value":39345},"batch 能不能救？批处理把权重摊薄，但同时把 KV 按 batch 复制：",{"type":150,"tag":181,"props":39347,"children":39349},{"className":39348},[1003],[39350],{"type":150,"tag":181,"props":39351,"children":39353},{"className":39352},[184],[39354,39465],{"type":150,"tag":181,"props":39355,"children":39357},{"className":39356},[189],[39358],{"type":150,"tag":192,"props":39359,"children":39360},{"xmlns":194,"display":1015},[39361],{"type":150,"tag":197,"props":39362,"children":39363},{},[39364,39460],{"type":150,"tag":201,"props":39365,"children":39366},{},[39367,39371,39375,39379,39383,39387,39391,39395],{"type":150,"tag":209,"props":39368,"children":39369},{},[39370],{"type":155,"value":24937},{"type":150,"tag":221,"props":39372,"children":39373},{"stretchy":789},[39374],{"type":155,"value":792},{"type":150,"tag":209,"props":39376,"children":39377},{},[39378],{"type":155,"value":23188},{"type":150,"tag":221,"props":39380,"children":39381},{"separator":223},[39382],{"type":155,"value":226},{"type":150,"tag":209,"props":39384,"children":39385},{},[39386],{"type":155,"value":5417},{"type":150,"tag":221,"props":39388,"children":39389},{"stretchy":789},[39390],{"type":155,"value":802},{"type":150,"tag":221,"props":39392,"children":39393},{},[39394],{"type":155,"value":865},{"type":150,"tag":1774,"props":39396,"children":39397},{},[39398,39425],{"type":150,"tag":201,"props":39399,"children":39400},{},[39401,39405,39409,39413,39417,39421],{"type":150,"tag":209,"props":39402,"children":39403},{},[39404],{"type":155,"value":23188},{"type":150,"tag":221,"props":39406,"children":39407},{},[39408],{"type":155,"value":5380},{"type":150,"tag":215,"props":39410,"children":39411},{},[39412],{"type":155,"value":238},{"type":150,"tag":221,"props":39414,"children":39415},{},[39416],{"type":155,"value":5380},{"type":150,"tag":215,"props":39418,"children":39419},{},[39420],{"type":155,"value":35393},{"type":150,"tag":249,"props":39422,"children":39423},{},[39424],{"type":155,"value":23188},{"type":150,"tag":201,"props":39426,"children":39427},{},[39428,39432,39436,39440,39444,39448,39452,39456],{"type":150,"tag":215,"props":39429,"children":39430},{},[39431],{"type":155,"value":9202},{"type":150,"tag":249,"props":39433,"children":39434},{},[39435],{"type":155,"value":29293},{"type":150,"tag":221,"props":39437,"children":39438},{},[39439],{"type":155,"value":4962},{"type":150,"tag":209,"props":39441,"children":39442},{},[39443],{"type":155,"value":23188},{"type":150,"tag":221,"props":39445,"children":39446},{},[39447],{"type":155,"value":5380},{"type":150,"tag":215,"props":39449,"children":39450},{},[39451],{"type":155,"value":37937},{"type":150,"tag":221,"props":39453,"children":39454},{},[39455],{"type":155,"value":5380},{"type":150,"tag":209,"props":39457,"children":39458},{},[39459],{"type":155,"value":5417},{"type":150,"tag":271,"props":39461,"children":39462},{"encoding":273},[39463],{"type":155,"value":39464},"I(B, N) = \\frac{B \\times 2 \\times 16\\text{B}}{8\\ \\text{GB} + B \\times 890 \\times N}",{"type":150,"tag":181,"props":39466,"children":39468},{"className":39467,"ariaHidden":223},[280],[39469,39524],{"type":150,"tag":181,"props":39470,"children":39472},{"className":39471},[285],[39473,39477,39482,39487,39492,39497,39501,39506,39511,39515,39520],{"type":150,"tag":181,"props":39474,"children":39476},{"className":39475,"style":819},[290],[],{"type":150,"tag":181,"props":39478,"children":39480},{"className":39479,"style":24957},[296,301],[39481],{"type":155,"value":24937},{"type":150,"tag":181,"props":39483,"children":39485},{"className":39484},[824],[39486],{"type":155,"value":792},{"type":150,"tag":181,"props":39488,"children":39490},{"className":39489,"style":27072},[296,301],[39491],{"type":155,"value":23188},{"type":150,"tag":181,"props":39493,"children":39495},{"className":39494},[371],[39496],{"type":155,"value":226},{"type":150,"tag":181,"props":39498,"children":39500},{"className":39499,"style":378},[377],[],{"type":150,"tag":181,"props":39502,"children":39504},{"className":39503,"style":5606},[296,301],[39505],{"type":155,"value":5417},{"type":150,"tag":181,"props":39507,"children":39509},{"className":39508},[835],[39510],{"type":155,"value":802},{"type":150,"tag":181,"props":39512,"children":39514},{"className":39513,"style":658},[377],[],{"type":150,"tag":181,"props":39516,"children":39518},{"className":39517},[663],[39519],{"type":155,"value":865},{"type":150,"tag":181,"props":39521,"children":39523},{"className":39522,"style":658},[377],[],{"type":150,"tag":181,"props":39525,"children":39527},{"className":39526},[285],[39528,39533],{"type":150,"tag":181,"props":39529,"children":39532},{"className":39530,"style":39531},[290],"height:2.1297em;vertical-align:-0.7693em;",[],{"type":150,"tag":181,"props":39534,"children":39536},{"className":39535},[296],[39537,39541,39729],{"type":150,"tag":181,"props":39538,"children":39540},{"className":39539},[824,2482],[],{"type":150,"tag":181,"props":39542,"children":39544},{"className":39543},[1774],[39545],{"type":150,"tag":181,"props":39546,"children":39548},{"className":39547},[312,313],[39549,39718],{"type":150,"tag":181,"props":39550,"children":39552},{"className":39551},[318],[39553,39713],{"type":150,"tag":181,"props":39554,"children":39556},{"className":39555,"style":23490},[323],[39557,39641,39652],{"type":150,"tag":181,"props":39558,"children":39559},{"style":8235},[39560,39564],{"type":150,"tag":181,"props":39561,"children":39563},{"className":39562,"style":8240},[333],[],{"type":150,"tag":181,"props":39565,"children":39567},{"className":39566},[296],[39568,39573,39578,39587,39591,39596,39600,39605,39609,39614,39618,39623,39627,39632,39636],{"type":150,"tag":181,"props":39569,"children":39571},{"className":39570},[296],[39572],{"type":155,"value":9202},{"type":150,"tag":181,"props":39574,"children":39576},{"className":39575},[377],[39577],{"type":155,"value":8576},{"type":150,"tag":181,"props":39579,"children":39581},{"className":39580},[296,155],[39582],{"type":150,"tag":181,"props":39583,"children":39585},{"className":39584},[296],[39586],{"type":155,"value":29439},{"type":150,"tag":181,"props":39588,"children":39590},{"className":39589,"style":4993},[377],[],{"type":150,"tag":181,"props":39592,"children":39594},{"className":39593},[4998],[39595],{"type":155,"value":4962},{"type":150,"tag":181,"props":39597,"children":39599},{"className":39598,"style":4993},[377],[],{"type":150,"tag":181,"props":39601,"children":39603},{"className":39602,"style":27072},[296,301],[39604],{"type":155,"value":23188},{"type":150,"tag":181,"props":39606,"children":39608},{"className":39607,"style":4993},[377],[],{"type":150,"tag":181,"props":39610,"children":39612},{"className":39611},[4998],[39613],{"type":155,"value":5380},{"type":150,"tag":181,"props":39615,"children":39617},{"className":39616,"style":4993},[377],[],{"type":150,"tag":181,"props":39619,"children":39621},{"className":39620},[296],[39622],{"type":155,"value":37937},{"type":150,"tag":181,"props":39624,"children":39626},{"className":39625,"style":4993},[377],[],{"type":150,"tag":181,"props":39628,"children":39630},{"className":39629},[4998],[39631],{"type":155,"value":5380},{"type":150,"tag":181,"props":39633,"children":39635},{"className":39634,"style":4993},[377],[],{"type":150,"tag":181,"props":39637,"children":39639},{"className":39638,"style":5606},[296,301],[39640],{"type":155,"value":5417},{"type":150,"tag":181,"props":39642,"children":39643},{"style":8253},[39644,39648],{"type":150,"tag":181,"props":39645,"children":39647},{"className":39646,"style":8240},[333],[],{"type":150,"tag":181,"props":39649,"children":39651},{"className":39650,"style":2804},[2803],[],{"type":150,"tag":181,"props":39653,"children":39654},{"style":8265},[39655,39659],{"type":150,"tag":181,"props":39656,"children":39658},{"className":39657,"style":8240},[333],[],{"type":150,"tag":181,"props":39660,"children":39662},{"className":39661},[296],[39663,39668,39672,39677,39681,39686,39690,39695,39699,39704],{"type":150,"tag":181,"props":39664,"children":39666},{"className":39665,"style":27072},[296,301],[39667],{"type":155,"value":23188},{"type":150,"tag":181,"props":39669,"children":39671},{"className":39670,"style":4993},[377],[],{"type":150,"tag":181,"props":39673,"children":39675},{"className":39674},[4998],[39676],{"type":155,"value":5380},{"type":150,"tag":181,"props":39678,"children":39680},{"className":39679,"style":4993},[377],[],{"type":150,"tag":181,"props":39682,"children":39684},{"className":39683},[296],[39685],{"type":155,"value":238},{"type":150,"tag":181,"props":39687,"children":39689},{"className":39688,"style":4993},[377],[],{"type":150,"tag":181,"props":39691,"children":39693},{"className":39692},[4998],[39694],{"type":155,"value":5380},{"type":150,"tag":181,"props":39696,"children":39698},{"className":39697,"style":4993},[377],[],{"type":150,"tag":181,"props":39700,"children":39702},{"className":39701},[296],[39703],{"type":155,"value":35393},{"type":150,"tag":181,"props":39705,"children":39707},{"className":39706},[296,155],[39708],{"type":150,"tag":181,"props":39709,"children":39711},{"className":39710},[296],[39712],{"type":155,"value":23188},{"type":150,"tag":181,"props":39714,"children":39716},{"className":39715},[352],[39717],{"type":155,"value":355},{"type":150,"tag":181,"props":39719,"children":39721},{"className":39720},[318],[39722],{"type":150,"tag":181,"props":39723,"children":39725},{"className":39724,"style":26077},[323],[39726],{"type":150,"tag":181,"props":39727,"children":39728},{},[],{"type":150,"tag":181,"props":39730,"children":39732},{"className":39731},[835,2482],[],{"type":150,"tag":151,"props":39734,"children":39735},{},[39736],{"type":155,"value":39737},"代进不同长度（可以直接运行验证）：",{"type":150,"tag":34045,"props":39739,"children":39743},{"className":39740,"code":39741,"language":39742,"meta":28,"style":28},"language-python shiki shiki-themes github-light github-dark","P = 16e9        # activated 参数量\nW = 8e9         # FP4 权重，每 token 要读的字节数\nkv = 890        # 每 token 的 KV 字节数\ninflection = 1125  # B200 FP4 的算力带宽比（FLOP\u002Fbyte）\n\ndef intensity(B, N):\n    return B * 2 * P \u002F (W + B * kv * N)\n\nfor N in [4e3, 32e3, 128e3, 1e6]:\n    cells = \"  \".join(f\"B={B}:{intensity(B, N):>6.0f}\" for B in [1, 8, 64, 512])\n    mark = \"  \u003C- 越线\" if intensity(512, N) > inflection else \"\"\n    print(f\"N={N:>8.0f}  {cells}{mark}\")\n","python",[39744],{"type":150,"tag":34051,"props":39745,"children":39746},{"__ignoreMap":28},[39747,39773,39795,39817,39839,39848,39867,39935,39943,40005,40131,40188],{"type":150,"tag":181,"props":39748,"children":39749},{"class":34056,"line":34057},[39750,39756,39761,39767],{"type":150,"tag":181,"props":39751,"children":39753},{"style":39752},"--shiki-default:#24292E;--shiki-dark:#E1E4E8",[39754],{"type":155,"value":39755},"P ",{"type":150,"tag":181,"props":39757,"children":39759},{"style":39758},"--shiki-default:#D73A49;--shiki-dark:#F97583",[39760],{"type":155,"value":865},{"type":150,"tag":181,"props":39762,"children":39764},{"style":39763},"--shiki-default:#005CC5;--shiki-dark:#79B8FF",[39765],{"type":155,"value":39766}," 16e9",{"type":150,"tag":181,"props":39768,"children":39770},{"style":39769},"--shiki-default:#6A737D;--shiki-dark:#6A737D",[39771],{"type":155,"value":39772},"        # activated 参数量\n",{"type":150,"tag":181,"props":39774,"children":39775},{"class":34056,"line":34066},[39776,39781,39785,39790],{"type":150,"tag":181,"props":39777,"children":39778},{"style":39752},[39779],{"type":155,"value":39780},"W ",{"type":150,"tag":181,"props":39782,"children":39783},{"style":39758},[39784],{"type":155,"value":865},{"type":150,"tag":181,"props":39786,"children":39787},{"style":39763},[39788],{"type":155,"value":39789}," 8e9",{"type":150,"tag":181,"props":39791,"children":39792},{"style":39769},[39793],{"type":155,"value":39794},"         # FP4 权重，每 token 要读的字节数\n",{"type":150,"tag":181,"props":39796,"children":39797},{"class":34056,"line":34075},[39798,39803,39807,39812],{"type":150,"tag":181,"props":39799,"children":39800},{"style":39752},[39801],{"type":155,"value":39802},"kv ",{"type":150,"tag":181,"props":39804,"children":39805},{"style":39758},[39806],{"type":155,"value":865},{"type":150,"tag":181,"props":39808,"children":39809},{"style":39763},[39810],{"type":155,"value":39811}," 890",{"type":150,"tag":181,"props":39813,"children":39814},{"style":39769},[39815],{"type":155,"value":39816},"        # 每 token 的 KV 字节数\n",{"type":150,"tag":181,"props":39818,"children":39819},{"class":34056,"line":34084},[39820,39825,39829,39834],{"type":150,"tag":181,"props":39821,"children":39822},{"style":39752},[39823],{"type":155,"value":39824},"inflection ",{"type":150,"tag":181,"props":39826,"children":39827},{"style":39758},[39828],{"type":155,"value":865},{"type":150,"tag":181,"props":39830,"children":39831},{"style":39763},[39832],{"type":155,"value":39833}," 1125",{"type":150,"tag":181,"props":39835,"children":39836},{"style":39769},[39837],{"type":155,"value":39838},"  # B200 FP4 的算力带宽比（FLOP\u002Fbyte）\n",{"type":150,"tag":181,"props":39840,"children":39841},{"class":34056,"line":34092},[39842],{"type":150,"tag":181,"props":39843,"children":39845},{"emptyLinePlaceholder":39844},true,[39846],{"type":155,"value":39847},"\n",{"type":150,"tag":181,"props":39849,"children":39850},{"class":34056,"line":34150},[39851,39856,39862],{"type":150,"tag":181,"props":39852,"children":39853},{"style":39758},[39854],{"type":155,"value":39855},"def",{"type":150,"tag":181,"props":39857,"children":39859},{"style":39858},"--shiki-default:#6F42C1;--shiki-dark:#B392F0",[39860],{"type":155,"value":39861}," intensity",{"type":150,"tag":181,"props":39863,"children":39864},{"style":39752},[39865],{"type":155,"value":39866},"(B, N):\n",{"type":150,"tag":181,"props":39868,"children":39869},{"class":34056,"line":34158},[39870,39875,39880,39885,39890,39895,39900,39904,39909,39913,39917,39921,39926,39930],{"type":150,"tag":181,"props":39871,"children":39872},{"style":39758},[39873],{"type":155,"value":39874},"    return",{"type":150,"tag":181,"props":39876,"children":39877},{"style":39752},[39878],{"type":155,"value":39879}," B ",{"type":150,"tag":181,"props":39881,"children":39882},{"style":39758},[39883],{"type":155,"value":39884},"*",{"type":150,"tag":181,"props":39886,"children":39887},{"style":39763},[39888],{"type":155,"value":39889}," 2",{"type":150,"tag":181,"props":39891,"children":39892},{"style":39758},[39893],{"type":155,"value":39894}," *",{"type":150,"tag":181,"props":39896,"children":39897},{"style":39752},[39898],{"type":155,"value":39899}," P ",{"type":150,"tag":181,"props":39901,"children":39902},{"style":39758},[39903],{"type":155,"value":4740},{"type":150,"tag":181,"props":39905,"children":39906},{"style":39752},[39907],{"type":155,"value":39908}," (W ",{"type":150,"tag":181,"props":39910,"children":39911},{"style":39758},[39912],{"type":155,"value":4962},{"type":150,"tag":181,"props":39914,"children":39915},{"style":39752},[39916],{"type":155,"value":39879},{"type":150,"tag":181,"props":39918,"children":39919},{"style":39758},[39920],{"type":155,"value":39884},{"type":150,"tag":181,"props":39922,"children":39923},{"style":39752},[39924],{"type":155,"value":39925}," kv ",{"type":150,"tag":181,"props":39927,"children":39928},{"style":39758},[39929],{"type":155,"value":39884},{"type":150,"tag":181,"props":39931,"children":39932},{"style":39752},[39933],{"type":155,"value":39934}," N)\n",{"type":150,"tag":181,"props":39936,"children":39938},{"class":34056,"line":39937},8,[39939],{"type":150,"tag":181,"props":39940,"children":39941},{"emptyLinePlaceholder":39844},[39942],{"type":155,"value":39847},{"type":150,"tag":181,"props":39944,"children":39946},{"class":34056,"line":39945},9,[39947,39952,39957,39962,39967,39972,39977,39982,39986,39991,39995,40000],{"type":150,"tag":181,"props":39948,"children":39949},{"style":39758},[39950],{"type":155,"value":39951},"for",{"type":150,"tag":181,"props":39953,"children":39954},{"style":39752},[39955],{"type":155,"value":39956}," N ",{"type":150,"tag":181,"props":39958,"children":39959},{"style":39758},[39960],{"type":155,"value":39961},"in",{"type":150,"tag":181,"props":39963,"children":39964},{"style":39752},[39965],{"type":155,"value":39966}," [",{"type":150,"tag":181,"props":39968,"children":39969},{"style":39763},[39970],{"type":155,"value":39971},"4e3",{"type":150,"tag":181,"props":39973,"children":39974},{"style":39752},[39975],{"type":155,"value":39976},", ",{"type":150,"tag":181,"props":39978,"children":39979},{"style":39763},[39980],{"type":155,"value":39981},"32e3",{"type":150,"tag":181,"props":39983,"children":39984},{"style":39752},[39985],{"type":155,"value":39976},{"type":150,"tag":181,"props":39987,"children":39988},{"style":39763},[39989],{"type":155,"value":39990},"128e3",{"type":150,"tag":181,"props":39992,"children":39993},{"style":39752},[39994],{"type":155,"value":39976},{"type":150,"tag":181,"props":39996,"children":39997},{"style":39763},[39998],{"type":155,"value":39999},"1e6",{"type":150,"tag":181,"props":40001,"children":40002},{"style":39752},[40003],{"type":155,"value":40004},"]:\n",{"type":150,"tag":181,"props":40006,"children":40008},{"class":34056,"line":40007},10,[40009,40014,40018,40024,40029,40034,40039,40044,40048,40053,40058,40062,40067,40072,40076,40081,40086,40090,40094,40098,40102,40106,40110,40114,40118,40122,40126],{"type":150,"tag":181,"props":40010,"children":40011},{"style":39752},[40012],{"type":155,"value":40013},"    cells ",{"type":150,"tag":181,"props":40015,"children":40016},{"style":39758},[40017],{"type":155,"value":865},{"type":150,"tag":181,"props":40019,"children":40021},{"style":40020},"--shiki-default:#032F62;--shiki-dark:#9ECBFF",[40022],{"type":155,"value":40023}," \"  \"",{"type":150,"tag":181,"props":40025,"children":40026},{"style":39752},[40027],{"type":155,"value":40028},".join(",{"type":150,"tag":181,"props":40030,"children":40031},{"style":39758},[40032],{"type":155,"value":40033},"f",{"type":150,"tag":181,"props":40035,"children":40036},{"style":40020},[40037],{"type":155,"value":40038},"\"B=",{"type":150,"tag":181,"props":40040,"children":40041},{"style":39763},[40042],{"type":155,"value":40043},"{",{"type":150,"tag":181,"props":40045,"children":40046},{"style":39752},[40047],{"type":155,"value":23188},{"type":150,"tag":181,"props":40049,"children":40050},{"style":39763},[40051],{"type":155,"value":40052},"}",{"type":150,"tag":181,"props":40054,"children":40055},{"style":40020},[40056],{"type":155,"value":40057},":",{"type":150,"tag":181,"props":40059,"children":40060},{"style":39763},[40061],{"type":155,"value":40043},{"type":150,"tag":181,"props":40063,"children":40064},{"style":39752},[40065],{"type":155,"value":40066},"intensity(B, N)",{"type":150,"tag":181,"props":40068,"children":40069},{"style":39758},[40070],{"type":155,"value":40071},":>6.0f",{"type":150,"tag":181,"props":40073,"children":40074},{"style":39763},[40075],{"type":155,"value":40052},{"type":150,"tag":181,"props":40077,"children":40078},{"style":40020},[40079],{"type":155,"value":40080},"\"",{"type":150,"tag":181,"props":40082,"children":40083},{"style":39758},[40084],{"type":155,"value":40085}," for",{"type":150,"tag":181,"props":40087,"children":40088},{"style":39752},[40089],{"type":155,"value":39879},{"type":150,"tag":181,"props":40091,"children":40092},{"style":39758},[40093],{"type":155,"value":39961},{"type":150,"tag":181,"props":40095,"children":40096},{"style":39752},[40097],{"type":155,"value":39966},{"type":150,"tag":181,"props":40099,"children":40100},{"style":39763},[40101],{"type":155,"value":219},{"type":150,"tag":181,"props":40103,"children":40104},{"style":39752},[40105],{"type":155,"value":39976},{"type":150,"tag":181,"props":40107,"children":40108},{"style":39763},[40109],{"type":155,"value":9202},{"type":150,"tag":181,"props":40111,"children":40112},{"style":39752},[40113],{"type":155,"value":39976},{"type":150,"tag":181,"props":40115,"children":40116},{"style":39763},[40117],{"type":155,"value":20641},{"type":150,"tag":181,"props":40119,"children":40120},{"style":39752},[40121],{"type":155,"value":39976},{"type":150,"tag":181,"props":40123,"children":40124},{"style":39763},[40125],{"type":155,"value":20786},{"type":150,"tag":181,"props":40127,"children":40128},{"style":39752},[40129],{"type":155,"value":40130},"])\n",{"type":150,"tag":181,"props":40132,"children":40134},{"class":34056,"line":40133},11,[40135,40140,40144,40149,40154,40159,40163,40168,40173,40178,40183],{"type":150,"tag":181,"props":40136,"children":40137},{"style":39752},[40138],{"type":155,"value":40139},"    mark ",{"type":150,"tag":181,"props":40141,"children":40142},{"style":39758},[40143],{"type":155,"value":865},{"type":150,"tag":181,"props":40145,"children":40146},{"style":40020},[40147],{"type":155,"value":40148}," \"  \u003C- 越线\"",{"type":150,"tag":181,"props":40150,"children":40151},{"style":39758},[40152],{"type":155,"value":40153}," if",{"type":150,"tag":181,"props":40155,"children":40156},{"style":39752},[40157],{"type":155,"value":40158}," intensity(",{"type":150,"tag":181,"props":40160,"children":40161},{"style":39763},[40162],{"type":155,"value":20786},{"type":150,"tag":181,"props":40164,"children":40165},{"style":39752},[40166],{"type":155,"value":40167},", N) ",{"type":150,"tag":181,"props":40169,"children":40170},{"style":39758},[40171],{"type":155,"value":40172},">",{"type":150,"tag":181,"props":40174,"children":40175},{"style":39752},[40176],{"type":155,"value":40177}," inflection ",{"type":150,"tag":181,"props":40179,"children":40180},{"style":39758},[40181],{"type":155,"value":40182},"else",{"type":150,"tag":181,"props":40184,"children":40185},{"style":40020},[40186],{"type":155,"value":40187}," \"\"\n",{"type":150,"tag":181,"props":40189,"children":40191},{"class":34056,"line":40190},12,[40192,40197,40201,40205,40210,40214,40218,40223,40227,40232,40237,40242,40247,40251,40255],{"type":150,"tag":181,"props":40193,"children":40194},{"style":39763},[40195],{"type":155,"value":40196},"    print",{"type":150,"tag":181,"props":40198,"children":40199},{"style":39752},[40200],{"type":155,"value":792},{"type":150,"tag":181,"props":40202,"children":40203},{"style":39758},[40204],{"type":155,"value":40033},{"type":150,"tag":181,"props":40206,"children":40207},{"style":40020},[40208],{"type":155,"value":40209},"\"N=",{"type":150,"tag":181,"props":40211,"children":40212},{"style":39763},[40213],{"type":155,"value":40043},{"type":150,"tag":181,"props":40215,"children":40216},{"style":39752},[40217],{"type":155,"value":5417},{"type":150,"tag":181,"props":40219,"children":40220},{"style":39758},[40221],{"type":155,"value":40222},":>8.0f",{"type":150,"tag":181,"props":40224,"children":40225},{"style":39763},[40226],{"type":155,"value":40052},{"type":150,"tag":181,"props":40228,"children":40229},{"style":39763},[40230],{"type":155,"value":40231},"  {",{"type":150,"tag":181,"props":40233,"children":40234},{"style":39752},[40235],{"type":155,"value":40236},"cells",{"type":150,"tag":181,"props":40238,"children":40239},{"style":39763},[40240],{"type":155,"value":40241},"}{",{"type":150,"tag":181,"props":40243,"children":40244},{"style":39752},[40245],{"type":155,"value":40246},"mark",{"type":150,"tag":181,"props":40248,"children":40249},{"style":39763},[40250],{"type":155,"value":40052},{"type":150,"tag":181,"props":40252,"children":40253},{"style":40020},[40254],{"type":155,"value":40080},{"type":150,"tag":181,"props":40256,"children":40257},{"style":39752},[40258],{"type":155,"value":40259},")\n",{"type":150,"tag":151,"props":40261,"children":40262},{},[40263,40265,40344,40345,40422,40424,40510,40512,40588],{"type":155,"value":40264},"输出里只有两格越线，都在 4k 那一行，而且要 ",{"type":150,"tag":181,"props":40266,"children":40268},{"className":40267},[184],[40269,40300],{"type":150,"tag":181,"props":40270,"children":40272},{"className":40271},[189],[40273],{"type":150,"tag":192,"props":40274,"children":40275},{"xmlns":194},[40276],{"type":150,"tag":197,"props":40277,"children":40278},{},[40279,40295],{"type":150,"tag":201,"props":40280,"children":40281},{},[40282,40286,40291],{"type":150,"tag":209,"props":40283,"children":40284},{},[40285],{"type":155,"value":23188},{"type":150,"tag":221,"props":40287,"children":40288},{},[40289],{"type":155,"value":40290},"≥",{"type":150,"tag":215,"props":40292,"children":40293},{},[40294],{"type":155,"value":20786},{"type":150,"tag":271,"props":40296,"children":40297},{"encoding":273},[40298],{"type":155,"value":40299},"B \\ge 512",{"type":150,"tag":181,"props":40301,"children":40303},{"className":40302,"ariaHidden":223},[280],[40304,40331],{"type":150,"tag":181,"props":40305,"children":40307},{"className":40306},[285],[40308,40313,40318,40322,40327],{"type":150,"tag":181,"props":40309,"children":40312},{"className":40310,"style":40311},[290],"height:0.8193em;vertical-align:-0.136em;",[],{"type":150,"tag":181,"props":40314,"children":40316},{"className":40315,"style":27072},[296,301],[40317],{"type":155,"value":23188},{"type":150,"tag":181,"props":40319,"children":40321},{"className":40320,"style":658},[377],[],{"type":150,"tag":181,"props":40323,"children":40325},{"className":40324},[663],[40326],{"type":155,"value":40290},{"type":150,"tag":181,"props":40328,"children":40330},{"className":40329,"style":658},[377],[],{"type":150,"tag":181,"props":40332,"children":40334},{"className":40333},[285],[40335,40339],{"type":150,"tag":181,"props":40336,"children":40338},{"className":40337,"style":5012},[290],[],{"type":150,"tag":181,"props":40340,"children":40342},{"className":40341},[296],[40343],{"type":155,"value":20786},{"type":155,"value":723},{"type":150,"tag":181,"props":40346,"children":40348},{"className":40347},[184],[40349,40379],{"type":150,"tag":181,"props":40350,"children":40352},{"className":40351},[189],[40353],{"type":150,"tag":192,"props":40354,"children":40355},{"xmlns":194},[40356],{"type":150,"tag":197,"props":40357,"children":40358},{},[40359,40374],{"type":150,"tag":201,"props":40360,"children":40361},{},[40362,40366,40370],{"type":150,"tag":209,"props":40363,"children":40364},{},[40365],{"type":155,"value":5417},{"type":150,"tag":221,"props":40367,"children":40368},{},[40369],{"type":155,"value":12767},{"type":150,"tag":215,"props":40371,"children":40372},{},[40373],{"type":155,"value":25836},{"type":150,"tag":271,"props":40375,"children":40376},{"encoding":273},[40377],{"type":155,"value":40378},"N \\to 0",{"type":150,"tag":181,"props":40380,"children":40382},{"className":40381,"ariaHidden":223},[280],[40383,40409],{"type":150,"tag":181,"props":40384,"children":40386},{"className":40385},[285],[40387,40391,40396,40400,40405],{"type":150,"tag":181,"props":40388,"children":40390},{"className":40389,"style":5680},[290],[],{"type":150,"tag":181,"props":40392,"children":40394},{"className":40393,"style":5606},[296,301],[40395],{"type":155,"value":5417},{"type":150,"tag":181,"props":40397,"children":40399},{"className":40398,"style":658},[377],[],{"type":150,"tag":181,"props":40401,"children":40403},{"className":40402},[663],[40404],{"type":155,"value":12767},{"type":150,"tag":181,"props":40406,"children":40408},{"className":40407,"style":658},[377],[],{"type":150,"tag":181,"props":40410,"children":40412},{"className":40411},[285],[40413,40417],{"type":150,"tag":181,"props":40414,"children":40416},{"className":40415,"style":5012},[290],[],{"type":150,"tag":181,"props":40418,"children":40420},{"className":40419},[296],[40421],{"type":155,"value":25836},{"type":155,"value":40423}," 时 ",{"type":150,"tag":181,"props":40425,"children":40427},{"className":40426},[184],[40428,40462],{"type":150,"tag":181,"props":40429,"children":40431},{"className":40430},[189],[40432],{"type":150,"tag":192,"props":40433,"children":40434},{"xmlns":194},[40435],{"type":150,"tag":197,"props":40436,"children":40437},{},[40438,40457],{"type":150,"tag":201,"props":40439,"children":40440},{},[40441,40445,40449,40453],{"type":150,"tag":209,"props":40442,"children":40443},{},[40444],{"type":155,"value":24937},{"type":150,"tag":221,"props":40446,"children":40447},{},[40448],{"type":155,"value":12767},{"type":150,"tag":215,"props":40450,"children":40451},{},[40452],{"type":155,"value":22376},{"type":150,"tag":209,"props":40454,"children":40455},{},[40456],{"type":155,"value":23188},{"type":150,"tag":271,"props":40458,"children":40459},{"encoding":273},[40460],{"type":155,"value":40461},"I \\to 4B",{"type":150,"tag":181,"props":40463,"children":40465},{"className":40464,"ariaHidden":223},[280],[40466,40492],{"type":150,"tag":181,"props":40467,"children":40469},{"className":40468},[285],[40470,40474,40479,40483,40488],{"type":150,"tag":181,"props":40471,"children":40473},{"className":40472,"style":5680},[290],[],{"type":150,"tag":181,"props":40475,"children":40477},{"className":40476,"style":24957},[296,301],[40478],{"type":155,"value":24937},{"type":150,"tag":181,"props":40480,"children":40482},{"className":40481,"style":658},[377],[],{"type":150,"tag":181,"props":40484,"children":40486},{"className":40485},[663],[40487],{"type":155,"value":12767},{"type":150,"tag":181,"props":40489,"children":40491},{"className":40490,"style":658},[377],[],{"type":150,"tag":181,"props":40493,"children":40495},{"className":40494},[285],[40496,40500,40505],{"type":150,"tag":181,"props":40497,"children":40499},{"className":40498,"style":5680},[290],[],{"type":150,"tag":181,"props":40501,"children":40503},{"className":40502},[296],[40504],{"type":155,"value":22376},{"type":150,"tag":181,"props":40506,"children":40508},{"className":40507,"style":27072},[296,301],[40509],{"type":155,"value":23188},{"type":155,"value":40511},"，所以 batch 要超过 280 才有可能碰到那条线。",{"type":150,"tag":181,"props":40513,"children":40515},{"className":40514},[184],[40516,40545],{"type":150,"tag":181,"props":40517,"children":40519},{"className":40518},[189],[40520],{"type":150,"tag":192,"props":40521,"children":40522},{"xmlns":194},[40523],{"type":150,"tag":197,"props":40524,"children":40525},{},[40526,40541],{"type":150,"tag":201,"props":40527,"children":40528},{},[40529,40533,40537],{"type":150,"tag":209,"props":40530,"children":40531},{},[40532],{"type":155,"value":23188},{"type":150,"tag":221,"props":40534,"children":40535},{},[40536],{"type":155,"value":865},{"type":150,"tag":215,"props":40538,"children":40539},{},[40540],{"type":155,"value":20641},{"type":150,"tag":271,"props":40542,"children":40543},{"encoding":273},[40544],{"type":155,"value":28374},{"type":150,"tag":181,"props":40546,"children":40548},{"className":40547,"ariaHidden":223},[280],[40549,40575],{"type":150,"tag":181,"props":40550,"children":40552},{"className":40551},[285],[40553,40557,40562,40566,40571],{"type":150,"tag":181,"props":40554,"children":40556},{"className":40555,"style":5680},[290],[],{"type":150,"tag":181,"props":40558,"children":40560},{"className":40559,"style":27072},[296,301],[40561],{"type":155,"value":23188},{"type":150,"tag":181,"props":40563,"children":40565},{"className":40564,"style":658},[377],[],{"type":150,"tag":181,"props":40567,"children":40569},{"className":40568},[663],[40570],{"type":155,"value":865},{"type":150,"tag":181,"props":40572,"children":40574},{"className":40573,"style":658},[377],[],{"type":150,"tag":181,"props":40576,"children":40578},{"className":40577},[285],[40579,40583],{"type":150,"tag":181,"props":40580,"children":40582},{"className":40581,"style":5012},[290],[],{"type":150,"tag":181,"props":40584,"children":40586},{"className":40585},[296],[40587],{"type":155,"value":20641},{"type":155,"value":40589}," 的极限是 256，连门槛都摸不到。",{"type":150,"tag":151,"props":40591,"children":40592},{},[40593,40635,40637,40713,40714,40845],{"type":150,"tag":181,"props":40594,"children":40596},{"className":40595},[184],[40597,40618],{"type":150,"tag":181,"props":40598,"children":40600},{"className":40599},[189],[40601],{"type":150,"tag":192,"props":40602,"children":40603},{"xmlns":194},[40604],{"type":150,"tag":197,"props":40605,"children":40606},{},[40607,40614],{"type":150,"tag":201,"props":40608,"children":40609},{},[40610],{"type":150,"tag":209,"props":40611,"children":40612},{},[40613],{"type":155,"value":5417},{"type":150,"tag":271,"props":40615,"children":40616},{"encoding":273},[40617],{"type":155,"value":5417},{"type":150,"tag":181,"props":40619,"children":40621},{"className":40620,"ariaHidden":223},[280],[40622],{"type":150,"tag":181,"props":40623,"children":40625},{"className":40624},[285],[40626,40630],{"type":150,"tag":181,"props":40627,"children":40629},{"className":40628,"style":5680},[290],[],{"type":150,"tag":181,"props":40631,"children":40633},{"className":40632,"style":5606},[296,301],[40634],{"type":155,"value":5417},{"type":155,"value":40636}," 一大就更没戏。",{"type":150,"tag":181,"props":40638,"children":40640},{"className":40639},[184],[40641,40670],{"type":150,"tag":181,"props":40642,"children":40644},{"className":40643},[189],[40645],{"type":150,"tag":192,"props":40646,"children":40647},{"xmlns":194},[40648],{"type":150,"tag":197,"props":40649,"children":40650},{},[40651,40666],{"type":150,"tag":201,"props":40652,"children":40653},{},[40654,40658,40662],{"type":150,"tag":209,"props":40655,"children":40656},{},[40657],{"type":155,"value":23188},{"type":150,"tag":221,"props":40659,"children":40660},{},[40661],{"type":155,"value":12767},{"type":150,"tag":209,"props":40663,"children":40664},{"mathvariant":1618},[40665],{"type":155,"value":26184},{"type":150,"tag":271,"props":40667,"children":40668},{"encoding":273},[40669],{"type":155,"value":28534},{"type":150,"tag":181,"props":40671,"children":40673},{"className":40672,"ariaHidden":223},[280],[40674,40700],{"type":150,"tag":181,"props":40675,"children":40677},{"className":40676},[285],[40678,40682,40687,40691,40696],{"type":150,"tag":181,"props":40679,"children":40681},{"className":40680,"style":5680},[290],[],{"type":150,"tag":181,"props":40683,"children":40685},{"className":40684,"style":27072},[296,301],[40686],{"type":155,"value":23188},{"type":150,"tag":181,"props":40688,"children":40690},{"className":40689,"style":658},[377],[],{"type":150,"tag":181,"props":40692,"children":40694},{"className":40693},[663],[40695],{"type":155,"value":12767},{"type":150,"tag":181,"props":40697,"children":40699},{"className":40698,"style":658},[377],[],{"type":150,"tag":181,"props":40701,"children":40703},{"className":40702},[285],[40704,40708],{"type":150,"tag":181,"props":40705,"children":40707},{"className":40706,"style":914},[290],[],{"type":150,"tag":181,"props":40709,"children":40711},{"className":40710},[296],[40712],{"type":155,"value":26184},{"type":155,"value":40423},{"type":150,"tag":181,"props":40715,"children":40717},{"className":40716},[184],[40718,40772],{"type":150,"tag":181,"props":40719,"children":40721},{"className":40720},[189],[40722],{"type":150,"tag":192,"props":40723,"children":40724},{"xmlns":194},[40725],{"type":150,"tag":197,"props":40726,"children":40727},{},[40728,40767],{"type":150,"tag":201,"props":40729,"children":40730},{},[40731,40735,40739,40743,40747,40751,40755,40759,40763],{"type":150,"tag":209,"props":40732,"children":40733},{},[40734],{"type":155,"value":24937},{"type":150,"tag":221,"props":40736,"children":40737},{},[40738],{"type":155,"value":12767},{"type":150,"tag":215,"props":40740,"children":40741},{},[40742],{"type":155,"value":238},{"type":150,"tag":209,"props":40744,"children":40745},{},[40746],{"type":155,"value":25017},{"type":150,"tag":209,"props":40748,"children":40749},{"mathvariant":1618},[40750],{"type":155,"value":4740},{"type":150,"tag":221,"props":40752,"children":40753},{"stretchy":789},[40754],{"type":155,"value":792},{"type":150,"tag":215,"props":40756,"children":40757},{},[40758],{"type":155,"value":37937},{"type":150,"tag":209,"props":40760,"children":40761},{},[40762],{"type":155,"value":5417},{"type":150,"tag":221,"props":40764,"children":40765},{"stretchy":789},[40766],{"type":155,"value":802},{"type":150,"tag":271,"props":40768,"children":40769},{"encoding":273},[40770],{"type":155,"value":40771},"I \\to 2P \u002F (890N)",{"type":150,"tag":181,"props":40773,"children":40775},{"className":40774,"ariaHidden":223},[280],[40776,40802],{"type":150,"tag":181,"props":40777,"children":40779},{"className":40778},[285],[40780,40784,40789,40793,40798],{"type":150,"tag":181,"props":40781,"children":40783},{"className":40782,"style":5680},[290],[],{"type":150,"tag":181,"props":40785,"children":40787},{"className":40786,"style":24957},[296,301],[40788],{"type":155,"value":24937},{"type":150,"tag":181,"props":40790,"children":40792},{"className":40791,"style":658},[377],[],{"type":150,"tag":181,"props":40794,"children":40796},{"className":40795},[663],[40797],{"type":155,"value":12767},{"type":150,"tag":181,"props":40799,"children":40801},{"className":40800,"style":658},[377],[],{"type":150,"tag":181,"props":40803,"children":40805},{"className":40804},[285],[40806,40810,40815,40820,40825,40830,40835,40840],{"type":150,"tag":181,"props":40807,"children":40809},{"className":40808,"style":819},[290],[],{"type":150,"tag":181,"props":40811,"children":40813},{"className":40812},[296],[40814],{"type":155,"value":238},{"type":150,"tag":181,"props":40816,"children":40818},{"className":40817,"style":3691},[296,301],[40819],{"type":155,"value":25017},{"type":150,"tag":181,"props":40821,"children":40823},{"className":40822},[296],[40824],{"type":155,"value":4740},{"type":150,"tag":181,"props":40826,"children":40828},{"className":40827},[824],[40829],{"type":155,"value":792},{"type":150,"tag":181,"props":40831,"children":40833},{"className":40832},[296],[40834],{"type":155,"value":37937},{"type":150,"tag":181,"props":40836,"children":40838},{"className":40837,"style":5606},[296,301],[40839],{"type":155,"value":5417},{"type":150,"tag":181,"props":40841,"children":40843},{"className":40842},[835],[40844],{"type":155,"value":802},{"type":155,"value":40846},"，1M context 下只有 34，离 1125 差 33 倍。因为 KV 太小，把权重完全摊掉之后剩下的那点 KV 依然喂不饱算力。",{"type":150,"tag":151,"props":40848,"children":40849},{},[40850,40852,40928,40930,40972],{"type":155,"value":40851},"结论比 MLA 那次更极端。MLA 至少留了一个\"大 batch 加短 context 会翻到 compute bound\"的窗口，而且那个窗口落在 4k，恰好是它收益最小的区间。ds4.1f 的窗口更窄：要 ",{"type":150,"tag":181,"props":40853,"children":40855},{"className":40854},[184],[40856,40885],{"type":150,"tag":181,"props":40857,"children":40859},{"className":40858},[189],[40860],{"type":150,"tag":192,"props":40861,"children":40862},{"xmlns":194},[40863],{"type":150,"tag":197,"props":40864,"children":40865},{},[40866,40881],{"type":150,"tag":201,"props":40867,"children":40868},{},[40869,40873,40877],{"type":150,"tag":209,"props":40870,"children":40871},{},[40872],{"type":155,"value":23188},{"type":150,"tag":221,"props":40874,"children":40875},{},[40876],{"type":155,"value":40290},{"type":150,"tag":215,"props":40878,"children":40879},{},[40880],{"type":155,"value":20786},{"type":150,"tag":271,"props":40882,"children":40883},{"encoding":273},[40884],{"type":155,"value":40299},{"type":150,"tag":181,"props":40886,"children":40888},{"className":40887,"ariaHidden":223},[280],[40889,40915],{"type":150,"tag":181,"props":40890,"children":40892},{"className":40891},[285],[40893,40897,40902,40906,40911],{"type":150,"tag":181,"props":40894,"children":40896},{"className":40895,"style":40311},[290],[],{"type":150,"tag":181,"props":40898,"children":40900},{"className":40899,"style":27072},[296,301],[40901],{"type":155,"value":23188},{"type":150,"tag":181,"props":40903,"children":40905},{"className":40904,"style":658},[377],[],{"type":150,"tag":181,"props":40907,"children":40909},{"className":40908},[663],[40910],{"type":155,"value":40290},{"type":150,"tag":181,"props":40912,"children":40914},{"className":40913,"style":658},[377],[],{"type":150,"tag":181,"props":40916,"children":40918},{"className":40917},[285],[40919,40923],{"type":150,"tag":181,"props":40920,"children":40922},{"className":40921,"style":5012},[290],[],{"type":150,"tag":181,"props":40924,"children":40926},{"className":40925},[296],[40927],{"type":155,"value":20786},{"type":155,"value":40929}," 且 ",{"type":150,"tag":181,"props":40931,"children":40933},{"className":40932},[184],[40934,40955],{"type":150,"tag":181,"props":40935,"children":40937},{"className":40936},[189],[40938],{"type":150,"tag":192,"props":40939,"children":40940},{"xmlns":194},[40941],{"type":150,"tag":197,"props":40942,"children":40943},{},[40944,40951],{"type":150,"tag":201,"props":40945,"children":40946},{},[40947],{"type":150,"tag":209,"props":40948,"children":40949},{},[40950],{"type":155,"value":5417},{"type":150,"tag":271,"props":40952,"children":40953},{"encoding":273},[40954],{"type":155,"value":5417},{"type":150,"tag":181,"props":40956,"children":40958},{"className":40957,"ariaHidden":223},[280],[40959],{"type":150,"tag":181,"props":40960,"children":40962},{"className":40961},[285],[40963,40967],{"type":150,"tag":181,"props":40964,"children":40966},{"className":40965,"style":5680},[290],[],{"type":150,"tag":181,"props":40968,"children":40970},{"className":40969,"style":5606},[296,301],[40971],{"type":155,"value":5417},{"type":155,"value":40973}," 压到 1.4 万以内，才算勉强碰到算力那一侧。",{"type":150,"tag":151,"props":40975,"children":40976},{},[40977],{"type":155,"value":40978},"再算一遍吞吐。每步耗时取带宽和算力里较大的那个：",{"type":150,"tag":22173,"props":40980,"children":40981},{},[40982,41048],{"type":150,"tag":22177,"props":40983,"children":40984},{},[40985],{"type":150,"tag":22181,"props":40986,"children":40987},{},[40988,41033,41038,41043],{"type":150,"tag":22185,"props":40989,"children":40990},{},[40991],{"type":150,"tag":181,"props":40992,"children":40994},{"className":40993},[184],[40995,41016],{"type":150,"tag":181,"props":40996,"children":40998},{"className":40997},[189],[40999],{"type":150,"tag":192,"props":41000,"children":41001},{"xmlns":194},[41002],{"type":150,"tag":197,"props":41003,"children":41004},{},[41005,41012],{"type":150,"tag":201,"props":41006,"children":41007},{},[41008],{"type":150,"tag":209,"props":41009,"children":41010},{},[41011],{"type":155,"value":23188},{"type":150,"tag":271,"props":41013,"children":41014},{"encoding":273},[41015],{"type":155,"value":23188},{"type":150,"tag":181,"props":41017,"children":41019},{"className":41018,"ariaHidden":223},[280],[41020],{"type":150,"tag":181,"props":41021,"children":41023},{"className":41022},[285],[41024,41028],{"type":150,"tag":181,"props":41025,"children":41027},{"className":41026,"style":5680},[290],[],{"type":150,"tag":181,"props":41029,"children":41031},{"className":41030,"style":27072},[296,301],[41032],{"type":155,"value":23188},{"type":150,"tag":22185,"props":41034,"children":41035},{},[41036],{"type":155,"value":41037},"1M：单序列 tok\u002Fs",{"type":150,"tag":22185,"props":41039,"children":41040},{},[41041],{"type":155,"value":41042},"1M：总吞吐",{"type":150,"tag":22185,"props":41044,"children":41045},{},[41046],{"type":155,"value":41047},"32k：总吞吐",{"type":150,"tag":22199,"props":41049,"children":41050},{},[41051,41070,41092,41114,41136,41157],{"type":150,"tag":22181,"props":41052,"children":41053},{},[41054,41058,41062,41066],{"type":150,"tag":22206,"props":41055,"children":41056},{},[41057],{"type":155,"value":219},{"type":150,"tag":22206,"props":41059,"children":41060},{},[41061],{"type":155,"value":39061},{"type":150,"tag":22206,"props":41063,"children":41064},{},[41065],{"type":155,"value":39061},{"type":150,"tag":22206,"props":41067,"children":41068},{},[41069],{"type":155,"value":38999},{"type":150,"tag":22181,"props":41071,"children":41072},{},[41073,41077,41082,41087],{"type":150,"tag":22206,"props":41074,"children":41075},{},[41076],{"type":155,"value":9202},{"type":150,"tag":22206,"props":41078,"children":41079},{},[41080],{"type":155,"value":41081},"517",{"type":150,"tag":22206,"props":41083,"children":41084},{},[41085],{"type":155,"value":41086},"4,138",{"type":150,"tag":22206,"props":41088,"children":41089},{},[41090],{"type":155,"value":41091},"7,773",{"type":150,"tag":22181,"props":41093,"children":41094},{},[41095,41099,41104,41109],{"type":150,"tag":22206,"props":41096,"children":41097},{},[41098],{"type":155,"value":35419},{"type":150,"tag":22206,"props":41100,"children":41101},{},[41102],{"type":155,"value":41103},"211",{"type":150,"tag":22206,"props":41105,"children":41106},{},[41107],{"type":155,"value":41108},"6,761",{"type":150,"tag":22206,"props":41110,"children":41111},{},[41112],{"type":155,"value":41113},"28,657",{"type":150,"tag":22181,"props":41115,"children":41116},{},[41117,41121,41126,41131],{"type":150,"tag":22206,"props":41118,"children":41119},{},[41120],{"type":155,"value":5871},{"type":150,"tag":22206,"props":41122,"children":41123},{},[41124],{"type":155,"value":41125},"63",{"type":150,"tag":22206,"props":41127,"children":41128},{},[41129],{"type":155,"value":41130},"8,034",{"type":150,"tag":22206,"props":41132,"children":41133},{},[41134],{"type":155,"value":41135},"87,276",{"type":150,"tag":22181,"props":41137,"children":41138},{},[41139,41143,41147,41152],{"type":150,"tag":22206,"props":41140,"children":41141},{},[41142],{"type":155,"value":20786},{"type":150,"tag":22206,"props":41144,"children":41145},{},[41146],{"type":155,"value":35393},{"type":150,"tag":22206,"props":41148,"children":41149},{},[41150],{"type":155,"value":41151},"8,431",{"type":150,"tag":22206,"props":41153,"children":41154},{},[41155],{"type":155,"value":41156},"178,617",{"type":150,"tag":22181,"props":41158,"children":41159},{},[41160,41228,41233,41238],{"type":150,"tag":22206,"props":41161,"children":41162},{},[41163],{"type":150,"tag":181,"props":41164,"children":41166},{"className":41165},[184],[41167,41193],{"type":150,"tag":181,"props":41168,"children":41170},{"className":41169},[189],[41171],{"type":150,"tag":192,"props":41172,"children":41173},{"xmlns":194},[41174],{"type":150,"tag":197,"props":41175,"children":41176},{},[41177,41188],{"type":150,"tag":201,"props":41178,"children":41179},{},[41180,41184],{"type":150,"tag":221,"props":41181,"children":41182},{},[41183],{"type":155,"value":12767},{"type":150,"tag":209,"props":41185,"children":41186},{"mathvariant":1618},[41187],{"type":155,"value":26184},{"type":150,"tag":271,"props":41189,"children":41190},{"encoding":273},[41191],{"type":155,"value":41192},"\\to \\infty",{"type":150,"tag":181,"props":41194,"children":41196},{"className":41195,"ariaHidden":223},[280],[41197,41215],{"type":150,"tag":181,"props":41198,"children":41200},{"className":41199},[285],[41201,41206,41211],{"type":150,"tag":181,"props":41202,"children":41205},{"className":41203,"style":41204},[290],"height:0.3669em;",[],{"type":150,"tag":181,"props":41207,"children":41209},{"className":41208},[663],[41210],{"type":155,"value":12767},{"type":150,"tag":181,"props":41212,"children":41214},{"className":41213,"style":658},[377],[],{"type":150,"tag":181,"props":41216,"children":41218},{"className":41217},[285],[41219,41223],{"type":150,"tag":181,"props":41220,"children":41222},{"className":41221,"style":914},[290],[],{"type":150,"tag":181,"props":41224,"children":41226},{"className":41225},[296],[41227],{"type":155,"value":26184},{"type":150,"tag":22206,"props":41229,"children":41230},{},[41231],{"type":155,"value":41232},"趋于 0",{"type":150,"tag":22206,"props":41234,"children":41235},{},[41236],{"type":155,"value":41237},"8,572",{"type":150,"tag":22206,"props":41239,"children":41240},{},[41241],{"type":155,"value":41242},"274,315",{"type":150,"tag":151,"props":41244,"children":41245},{},[41246],{"type":150,"tag":158,"props":41247,"children":41248},{},[41249],{"type":155,"value":41250},"总吞吐有上限，因为它最终被 KV 的带宽锁住：",{"type":150,"tag":181,"props":41252,"children":41254},{"className":41253},[1003],[41255],{"type":150,"tag":181,"props":41256,"children":41258},{"className":41257},[184],[41259,41331],{"type":150,"tag":181,"props":41260,"children":41262},{"className":41261},[189],[41263],{"type":150,"tag":192,"props":41264,"children":41265},{"xmlns":194,"display":1015},[41266],{"type":150,"tag":197,"props":41267,"children":41268},{},[41269,41326],{"type":150,"tag":201,"props":41270,"children":41271},{},[41272,41289,41293],{"type":150,"tag":205,"props":41273,"children":41274},{},[41275,41280,41285],{"type":150,"tag":249,"props":41276,"children":41277},{},[41278],{"type":155,"value":41279},"吞吐",{"type":150,"tag":209,"props":41281,"children":41282},{},[41283],{"type":155,"value":41284},"max",{"type":150,"tag":221,"props":41286,"children":41287},{},[41288],{"type":155,"value":1813},{"type":150,"tag":221,"props":41290,"children":41291},{},[41292],{"type":155,"value":12767},{"type":150,"tag":1774,"props":41294,"children":41295},{},[41296,41307],{"type":150,"tag":201,"props":41297,"children":41298},{},[41299,41303],{"type":150,"tag":215,"props":41300,"children":41301},{},[41302],{"type":155,"value":9202},{"type":150,"tag":249,"props":41304,"children":41305},{},[41306],{"type":155,"value":23434},{"type":150,"tag":201,"props":41308,"children":41309},{},[41310,41314,41318,41322],{"type":150,"tag":215,"props":41311,"children":41312},{},[41313],{"type":155,"value":37937},{"type":150,"tag":249,"props":41315,"children":41316},{},[41317],{"type":155,"value":39124},{"type":150,"tag":221,"props":41319,"children":41320},{},[41321],{"type":155,"value":5380},{"type":150,"tag":209,"props":41323,"children":41324},{},[41325],{"type":155,"value":5417},{"type":150,"tag":271,"props":41327,"children":41328},{"encoding":273},[41329],{"type":155,"value":41330},"\\text{吞吐}_{\\max} \\to \\frac{8\\ \\text{TB\u002Fs}}{890\\ \\text{B} \\times N}",{"type":150,"tag":181,"props":41332,"children":41334},{"className":41333,"ariaHidden":223},[280],[41335,41437],{"type":150,"tag":181,"props":41336,"children":41338},{"className":41337},[285],[41339,41343,41424,41428,41433],{"type":150,"tag":181,"props":41340,"children":41342},{"className":41341,"style":6190},[290],[],{"type":150,"tag":181,"props":41344,"children":41346},{"className":41345},[296],[41347,41356],{"type":150,"tag":181,"props":41348,"children":41350},{"className":41349},[296,155],[41351],{"type":150,"tag":181,"props":41352,"children":41354},{"className":41353},[296,24835],[41355],{"type":155,"value":41279},{"type":150,"tag":181,"props":41357,"children":41359},{"className":41358},[307],[41360],{"type":150,"tag":181,"props":41361,"children":41363},{"className":41362},[312,313],[41364,41413],{"type":150,"tag":181,"props":41365,"children":41367},{"className":41366},[318],[41368,41408],{"type":150,"tag":181,"props":41369,"children":41371},{"className":41370,"style":4865},[323],[41372],{"type":150,"tag":181,"props":41373,"children":41375},{"style":41374},"top:-2.55em;margin-right:0.05em;",[41376,41380],{"type":150,"tag":181,"props":41377,"children":41379},{"className":41378,"style":334},[333],[],{"type":150,"tag":181,"props":41381,"children":41383},{"className":41382},[339,340,341,342],[41384],{"type":150,"tag":181,"props":41385,"children":41387},{"className":41386},[296,342],[41388],{"type":150,"tag":181,"props":41389,"children":41391},{"className":41390},[2517,342],[41392,41398,41403],{"type":150,"tag":181,"props":41393,"children":41395},{"className":41394},[342],[41396],{"type":155,"value":41397},"m",{"type":150,"tag":181,"props":41399,"children":41401},{"className":41400},[342],[41402],{"type":155,"value":9663},{"type":150,"tag":181,"props":41404,"children":41406},{"className":41405},[342],[41407],{"type":155,"value":213},{"type":150,"tag":181,"props":41409,"children":41411},{"className":41410},[352],[41412],{"type":155,"value":355},{"type":150,"tag":181,"props":41414,"children":41416},{"className":41415},[318],[41417],{"type":150,"tag":181,"props":41418,"children":41420},{"className":41419,"style":363},[323],[41421],{"type":150,"tag":181,"props":41422,"children":41423},{},[],{"type":150,"tag":181,"props":41425,"children":41427},{"className":41426,"style":658},[377],[],{"type":150,"tag":181,"props":41429,"children":41431},{"className":41430},[663],[41432],{"type":155,"value":12767},{"type":150,"tag":181,"props":41434,"children":41436},{"className":41435,"style":658},[377],[],{"type":150,"tag":181,"props":41438,"children":41440},{"className":41439},[285],[41441,41446],{"type":150,"tag":181,"props":41442,"children":41445},{"className":41443,"style":41444},[290],"height:2.1963em;vertical-align:-0.7693em;",[],{"type":150,"tag":181,"props":41447,"children":41449},{"className":41448},[296],[41450,41454,41575],{"type":150,"tag":181,"props":41451,"children":41453},{"className":41452},[824,2482],[],{"type":150,"tag":181,"props":41455,"children":41457},{"className":41456},[1774],[41458],{"type":150,"tag":181,"props":41459,"children":41461},{"className":41460},[312,313],[41462,41564],{"type":150,"tag":181,"props":41463,"children":41465},{"className":41464},[318],[41466,41559],{"type":150,"tag":181,"props":41467,"children":41469},{"className":41468,"style":27312},[323],[41470,41518,41529],{"type":150,"tag":181,"props":41471,"children":41472},{"style":8235},[41473,41477],{"type":150,"tag":181,"props":41474,"children":41476},{"className":41475,"style":8240},[333],[],{"type":150,"tag":181,"props":41478,"children":41480},{"className":41479},[296],[41481,41486,41491,41500,41504,41509,41513],{"type":150,"tag":181,"props":41482,"children":41484},{"className":41483},[296],[41485],{"type":155,"value":37937},{"type":150,"tag":181,"props":41487,"children":41489},{"className":41488},[377],[41490],{"type":155,"value":8576},{"type":150,"tag":181,"props":41492,"children":41494},{"className":41493},[296,155],[41495],{"type":150,"tag":181,"props":41496,"children":41498},{"className":41497},[296],[41499],{"type":155,"value":23188},{"type":150,"tag":181,"props":41501,"children":41503},{"className":41502,"style":4993},[377],[],{"type":150,"tag":181,"props":41505,"children":41507},{"className":41506},[4998],[41508],{"type":155,"value":5380},{"type":150,"tag":181,"props":41510,"children":41512},{"className":41511,"style":4993},[377],[],{"type":150,"tag":181,"props":41514,"children":41516},{"className":41515,"style":5606},[296,301],[41517],{"type":155,"value":5417},{"type":150,"tag":181,"props":41519,"children":41520},{"style":8253},[41521,41525],{"type":150,"tag":181,"props":41522,"children":41524},{"className":41523,"style":8240},[333],[],{"type":150,"tag":181,"props":41526,"children":41528},{"className":41527,"style":2804},[2803],[],{"type":150,"tag":181,"props":41530,"children":41531},{"style":8265},[41532,41536],{"type":150,"tag":181,"props":41533,"children":41535},{"className":41534,"style":8240},[333],[],{"type":150,"tag":181,"props":41537,"children":41539},{"className":41538},[296],[41540,41545,41550],{"type":150,"tag":181,"props":41541,"children":41543},{"className":41542},[296],[41544],{"type":155,"value":9202},{"type":150,"tag":181,"props":41546,"children":41548},{"className":41547},[377],[41549],{"type":155,"value":8576},{"type":150,"tag":181,"props":41551,"children":41553},{"className":41552},[296,155],[41554],{"type":150,"tag":181,"props":41555,"children":41557},{"className":41556},[296],[41558],{"type":155,"value":23522},{"type":150,"tag":181,"props":41560,"children":41562},{"className":41561},[352],[41563],{"type":155,"value":355},{"type":150,"tag":181,"props":41565,"children":41567},{"className":41566},[318],[41568],{"type":150,"tag":181,"props":41569,"children":41571},{"className":41570,"style":26077},[323],[41572],{"type":150,"tag":181,"props":41573,"children":41574},{},[],{"type":150,"tag":181,"props":41576,"children":41578},{"className":41577},[835,2482],[],{"type":150,"tag":22173,"props":41580,"children":41581},{},[41582,41649],{"type":150,"tag":22177,"props":41583,"children":41584},{},[41585],{"type":150,"tag":22181,"props":41586,"children":41587},{},[41588,41633,41637,41641,41645],{"type":150,"tag":22185,"props":41589,"children":41590},{},[41591],{"type":150,"tag":181,"props":41592,"children":41594},{"className":41593},[184],[41595,41616],{"type":150,"tag":181,"props":41596,"children":41598},{"className":41597},[189],[41599],{"type":150,"tag":192,"props":41600,"children":41601},{"xmlns":194},[41602],{"type":150,"tag":197,"props":41603,"children":41604},{},[41605,41612],{"type":150,"tag":201,"props":41606,"children":41607},{},[41608],{"type":150,"tag":209,"props":41609,"children":41610},{},[41611],{"type":155,"value":5417},{"type":150,"tag":271,"props":41613,"children":41614},{"encoding":273},[41615],{"type":155,"value":5417},{"type":150,"tag":181,"props":41617,"children":41619},{"className":41618,"ariaHidden":223},[280],[41620],{"type":150,"tag":181,"props":41621,"children":41623},{"className":41622},[285],[41624,41628],{"type":150,"tag":181,"props":41625,"children":41627},{"className":41626,"style":5680},[290],[],{"type":150,"tag":181,"props":41629,"children":41631},{"className":41630,"style":5606},[296,301],[41632],{"type":155,"value":5417},{"type":150,"tag":22185,"props":41634,"children":41635},{},[41636],{"type":155,"value":26915},{"type":150,"tag":22185,"props":41638,"children":41639},{},[41640],{"type":155,"value":26943},{"type":150,"tag":22185,"props":41642,"children":41643},{},[41644],{"type":155,"value":26970},{"type":150,"tag":22185,"props":41646,"children":41647},{},[41648],{"type":155,"value":26997},{"type":150,"tag":22199,"props":41650,"children":41651},{},[41652],{"type":150,"tag":22181,"props":41653,"children":41654},{},[41655,41660,41665,41670,41675],{"type":150,"tag":22206,"props":41656,"children":41657},{},[41658],{"type":155,"value":41659},"吞吐上限",{"type":150,"tag":22206,"props":41661,"children":41662},{},[41663],{"type":155,"value":41664},"281k",{"type":150,"tag":22206,"props":41666,"children":41667},{},[41668],{"type":155,"value":41669},"274k",{"type":150,"tag":22206,"props":41671,"children":41672},{},[41673],{"type":155,"value":41674},"68.6k",{"type":150,"tag":22206,"props":41676,"children":41677},{},[41678],{"type":155,"value":41679},"8.6k",{"type":150,"tag":151,"props":41681,"children":41682},{},[41683,41685,41763],{"type":155,"value":41684},"4k 和 32k 那两档其实先被算力封顶（281k），再往上才是 KV 带宽封顶。分界在 ",{"type":150,"tag":181,"props":41686,"children":41688},{"className":41687},[184],[41689,41720],{"type":150,"tag":181,"props":41690,"children":41692},{"className":41691},[189],[41693],{"type":150,"tag":192,"props":41694,"children":41695},{"xmlns":194},[41696],{"type":150,"tag":197,"props":41697,"children":41698},{},[41699,41715],{"type":150,"tag":201,"props":41700,"children":41701},{},[41702,41706,41710],{"type":150,"tag":209,"props":41703,"children":41704},{},[41705],{"type":155,"value":5417},{"type":150,"tag":221,"props":41707,"children":41708},{},[41709],{"type":155,"value":5876},{"type":150,"tag":215,"props":41711,"children":41712},{},[41713],{"type":155,"value":41714},"3.2",{"type":150,"tag":271,"props":41716,"children":41717},{"encoding":273},[41718],{"type":155,"value":41719},"N \\approx 3.2",{"type":150,"tag":181,"props":41721,"children":41723},{"className":41722,"ariaHidden":223},[280],[41724,41750],{"type":150,"tag":181,"props":41725,"children":41727},{"className":41726},[285],[41728,41732,41737,41741,41746],{"type":150,"tag":181,"props":41729,"children":41731},{"className":41730,"style":5680},[290],[],{"type":150,"tag":181,"props":41733,"children":41735},{"className":41734,"style":5606},[296,301],[41736],{"type":155,"value":5417},{"type":150,"tag":181,"props":41738,"children":41740},{"className":41739,"style":658},[377],[],{"type":150,"tag":181,"props":41742,"children":41744},{"className":41743},[663],[41745],{"type":155,"value":5876},{"type":150,"tag":181,"props":41747,"children":41749},{"className":41748,"style":658},[377],[],{"type":150,"tag":181,"props":41751,"children":41753},{"className":41752},[285],[41754,41758],{"type":150,"tag":181,"props":41755,"children":41757},{"className":41756,"style":5012},[290],[],{"type":150,"tag":181,"props":41759,"children":41761},{"className":41760},[296],[41762],{"type":155,"value":41714},{"type":155,"value":41764}," 万，和前面 batch 表里算出来的是同一个数。",{"type":150,"tag":151,"props":41766,"children":41767},{},[41768,41773],{"type":150,"tag":158,"props":41769,"children":41770},{},[41771],{"type":155,"value":41772},"代价在延迟那一列。",{"type":155,"value":41774}," 同一个 1M 的请求，batch 从 1 开到 512，单序列速度从 896 掉到 16 token\u002Fs。长 context 上吞吐和交互速度是直接对立的：要吃满吞吐，就得牺牲单个用户的等待时间。",{"type":150,"tag":151,"props":41776,"children":41777},{},[41778],{"type":155,"value":41779},"反过来，如果要求单序列不低于 200 token\u002Fs，1M context 下 batch 只能开到 30 上下，单卡吞吐就是 6-7k token\u002Fs。这个量级和我实际跑 ds4.1f 的手感对得上。",{"type":150,"tag":151,"props":41781,"children":41782},{},[41783],{"type":155,"value":41784},"换成 TP4 部署，每卡的账是这样：",{"type":150,"tag":22173,"props":41786,"children":41787},{},[41788,41802],{"type":150,"tag":22177,"props":41789,"children":41790},{},[41791],{"type":150,"tag":22181,"props":41792,"children":41793},{},[41794,41797],{"type":150,"tag":22185,"props":41795,"children":41796},{},[],{"type":150,"tag":22185,"props":41798,"children":41799},{},[41800],{"type":155,"value":41801},"每卡",{"type":150,"tag":22199,"props":41803,"children":41804},{},[41805,41818,41831,41847,41863,41875],{"type":150,"tag":22181,"props":41806,"children":41807},{},[41808,41813],{"type":150,"tag":22206,"props":41809,"children":41810},{},[41811],{"type":155,"value":41812},"权重常驻显存（全部 552B 专家）",{"type":150,"tag":22206,"props":41814,"children":41815},{},[41816],{"type":155,"value":41817},"69 GB",{"type":150,"tag":22181,"props":41819,"children":41820},{},[41821,41826],{"type":150,"tag":22206,"props":41822,"children":41823},{},[41824],{"type":155,"value":41825},"留给 KV 的显存",{"type":150,"tag":22206,"props":41827,"children":41828},{},[41829],{"type":155,"value":41830},"111 GB",{"type":150,"tag":22181,"props":41832,"children":41833},{},[41834,41839],{"type":150,"tag":22206,"props":41835,"children":41836},{},[41837],{"type":155,"value":41838},"每步读的权重（16B activated 的四分之一）",{"type":150,"tag":22206,"props":41840,"children":41841},{},[41842],{"type":150,"tag":158,"props":41843,"children":41844},{},[41845],{"type":155,"value":41846},"2.00 GB",{"type":150,"tag":22181,"props":41848,"children":41849},{},[41850,41855],{"type":150,"tag":22206,"props":41851,"children":41852},{},[41853],{"type":155,"value":41854},"每步读的 KV（batch 1，1M context）",{"type":150,"tag":22206,"props":41856,"children":41857},{},[41858],{"type":150,"tag":158,"props":41859,"children":41860},{},[41861],{"type":155,"value":41862},"0.23 GB",{"type":150,"tag":22181,"props":41864,"children":41865},{},[41866,41870],{"type":150,"tag":22206,"props":41867,"children":41868},{},[41869],{"type":155,"value":38925},{"type":150,"tag":22206,"props":41871,"children":41872},{},[41873],{"type":155,"value":41874},"2.23 GB",{"type":150,"tag":22181,"props":41876,"children":41877},{},[41878,41883],{"type":150,"tag":22206,"props":41879,"children":41880},{},[41881],{"type":155,"value":41882},"每步带宽下限",{"type":150,"tag":22206,"props":41884,"children":41885},{},[41886],{"type":155,"value":41887},"0.28 ms",{"type":150,"tag":151,"props":41889,"children":41890},{},[41891],{"type":155,"value":41892},"要分清常驻和每步读，也要交代那个除以 4 是哪来的。",{"type":150,"tag":151,"props":41894,"children":41895},{},[41896,41901],{"type":150,"tag":158,"props":41897,"children":41898},{},[41899],{"type":155,"value":41900},"常驻",{"type":155,"value":41902},"：552B 个专家参数必须都在显存里。TP4 把每个权重矩阵切成四份、每卡持一份，所以是",{"type":150,"tag":181,"props":41904,"children":41906},{"className":41905},[1003],[41907],{"type":150,"tag":181,"props":41908,"children":41910},{"className":41909},[184],[41911,41973],{"type":150,"tag":181,"props":41912,"children":41914},{"className":41913},[189],[41915],{"type":150,"tag":192,"props":41916,"children":41917},{"xmlns":194,"display":1015},[41918],{"type":150,"tag":197,"props":41919,"children":41920},{},[41921,41968],{"type":150,"tag":201,"props":41922,"children":41923},{},[41924,41955,41959,41964],{"type":150,"tag":1774,"props":41925,"children":41926},{},[41927,41951],{"type":150,"tag":201,"props":41928,"children":41929},{},[41930,41935,41939,41943,41947],{"type":150,"tag":215,"props":41931,"children":41932},{},[41933],{"type":155,"value":41934},"552",{"type":150,"tag":249,"props":41936,"children":41937},{},[41938],{"type":155,"value":23188},{"type":150,"tag":221,"props":41940,"children":41941},{},[41942],{"type":155,"value":5380},{"type":150,"tag":215,"props":41944,"children":41945},{},[41946],{"type":155,"value":37783},{"type":150,"tag":249,"props":41948,"children":41949},{},[41950],{"type":155,"value":24730},{"type":150,"tag":215,"props":41952,"children":41953},{},[41954],{"type":155,"value":22376},{"type":150,"tag":221,"props":41956,"children":41957},{},[41958],{"type":155,"value":865},{"type":150,"tag":215,"props":41960,"children":41961},{},[41962],{"type":155,"value":41963},"69",{"type":150,"tag":249,"props":41965,"children":41966},{},[41967],{"type":155,"value":29293},{"type":150,"tag":271,"props":41969,"children":41970},{"encoding":273},[41971],{"type":155,"value":41972},"\\frac{552\\text{B} \\times 0.5\\ \\text{字节}}{4} = 69\\ \\text{GB}",{"type":150,"tag":181,"props":41974,"children":41976},{"className":41975,"ariaHidden":223},[280],[41977,42126],{"type":150,"tag":181,"props":41978,"children":41980},{"className":41979},[285],[41981,41985,42113,42117,42122],{"type":150,"tag":181,"props":41982,"children":41984},{"className":41983,"style":31545},[290],[],{"type":150,"tag":181,"props":41986,"children":41988},{"className":41987},[296],[41989,41993,42109],{"type":150,"tag":181,"props":41990,"children":41992},{"className":41991},[824,2482],[],{"type":150,"tag":181,"props":41994,"children":41996},{"className":41995},[1774],[41997],{"type":150,"tag":181,"props":41998,"children":42000},{"className":41999},[312,313],[42001,42098],{"type":150,"tag":181,"props":42002,"children":42004},{"className":42003},[318],[42005,42093],{"type":150,"tag":181,"props":42006,"children":42008},{"className":42007,"style":23490},[323],[42009,42025,42036],{"type":150,"tag":181,"props":42010,"children":42011},{"style":8235},[42012,42016],{"type":150,"tag":181,"props":42013,"children":42015},{"className":42014,"style":8240},[333],[],{"type":150,"tag":181,"props":42017,"children":42019},{"className":42018},[296],[42020],{"type":150,"tag":181,"props":42021,"children":42023},{"className":42022},[296],[42024],{"type":155,"value":22376},{"type":150,"tag":181,"props":42026,"children":42027},{"style":8253},[42028,42032],{"type":150,"tag":181,"props":42029,"children":42031},{"className":42030,"style":8240},[333],[],{"type":150,"tag":181,"props":42033,"children":42035},{"className":42034,"style":2804},[2803],[],{"type":150,"tag":181,"props":42037,"children":42038},{"style":8265},[42039,42043],{"type":150,"tag":181,"props":42040,"children":42042},{"className":42041,"style":8240},[333],[],{"type":150,"tag":181,"props":42044,"children":42046},{"className":42045},[296],[42047,42052,42061,42065,42070,42074,42079,42084],{"type":150,"tag":181,"props":42048,"children":42050},{"className":42049},[296],[42051],{"type":155,"value":41934},{"type":150,"tag":181,"props":42053,"children":42055},{"className":42054},[296,155],[42056],{"type":150,"tag":181,"props":42057,"children":42059},{"className":42058},[296],[42060],{"type":155,"value":23188},{"type":150,"tag":181,"props":42062,"children":42064},{"className":42063,"style":4993},[377],[],{"type":150,"tag":181,"props":42066,"children":42068},{"className":42067},[4998],[42069],{"type":155,"value":5380},{"type":150,"tag":181,"props":42071,"children":42073},{"className":42072,"style":4993},[377],[],{"type":150,"tag":181,"props":42075,"children":42077},{"className":42076},[296],[42078],{"type":155,"value":37783},{"type":150,"tag":181,"props":42080,"children":42082},{"className":42081},[377],[42083],{"type":155,"value":8576},{"type":150,"tag":181,"props":42085,"children":42087},{"className":42086},[296,155],[42088],{"type":150,"tag":181,"props":42089,"children":42091},{"className":42090},[296,24835],[42092],{"type":155,"value":24838},{"type":150,"tag":181,"props":42094,"children":42096},{"className":42095},[352],[42097],{"type":155,"value":355},{"type":150,"tag":181,"props":42099,"children":42101},{"className":42100},[318],[42102],{"type":150,"tag":181,"props":42103,"children":42105},{"className":42104,"style":8297},[323],[42106],{"type":150,"tag":181,"props":42107,"children":42108},{},[],{"type":150,"tag":181,"props":42110,"children":42112},{"className":42111},[835,2482],[],{"type":150,"tag":181,"props":42114,"children":42116},{"className":42115,"style":658},[377],[],{"type":150,"tag":181,"props":42118,"children":42120},{"className":42119},[663],[42121],{"type":155,"value":865},{"type":150,"tag":181,"props":42123,"children":42125},{"className":42124,"style":658},[377],[],{"type":150,"tag":181,"props":42127,"children":42129},{"className":42128},[285],[42130,42134,42139,42144],{"type":150,"tag":181,"props":42131,"children":42133},{"className":42132,"style":5680},[290],[],{"type":150,"tag":181,"props":42135,"children":42137},{"className":42136},[296],[42138],{"type":155,"value":41963},{"type":150,"tag":181,"props":42140,"children":42142},{"className":42141},[377],[42143],{"type":155,"value":8576},{"type":150,"tag":181,"props":42145,"children":42147},{"className":42146},[296,155],[42148],{"type":150,"tag":181,"props":42149,"children":42151},{"className":42150},[296],[42152],{"type":155,"value":29439},{"type":150,"tag":151,"props":42154,"children":42155},{},[42156,42161],{"type":150,"tag":158,"props":42157,"children":42158},{},[42159],{"type":155,"value":42160},"每步读",{"type":155,"value":42162},"：一个 token 只激活其中 16B。整机每步要读",{"type":150,"tag":181,"props":42164,"children":42166},{"className":42165},[1003],[42167],{"type":150,"tag":181,"props":42168,"children":42170},{"className":42169},[184],[42171,42220],{"type":150,"tag":181,"props":42172,"children":42174},{"className":42173},[189],[42175],{"type":150,"tag":192,"props":42176,"children":42177},{"xmlns":194,"display":1015},[42178],{"type":150,"tag":197,"props":42179,"children":42180},{},[42181,42216],{"type":150,"tag":201,"props":42182,"children":42183},{},[42184,42188,42192,42196,42200,42204,42208,42212],{"type":150,"tag":215,"props":42185,"children":42186},{},[42187],{"type":155,"value":35393},{"type":150,"tag":249,"props":42189,"children":42190},{},[42191],{"type":155,"value":23188},{"type":150,"tag":221,"props":42193,"children":42194},{},[42195],{"type":155,"value":5380},{"type":150,"tag":215,"props":42197,"children":42198},{},[42199],{"type":155,"value":37783},{"type":150,"tag":249,"props":42201,"children":42202},{},[42203],{"type":155,"value":24730},{"type":150,"tag":221,"props":42205,"children":42206},{},[42207],{"type":155,"value":865},{"type":150,"tag":215,"props":42209,"children":42210},{},[42211],{"type":155,"value":9202},{"type":150,"tag":249,"props":42213,"children":42214},{},[42215],{"type":155,"value":29293},{"type":150,"tag":271,"props":42217,"children":42218},{"encoding":273},[42219],{"type":155,"value":37804},{"type":150,"tag":181,"props":42221,"children":42223},{"className":42222,"ariaHidden":223},[280],[42224,42259,42299],{"type":150,"tag":181,"props":42225,"children":42227},{"className":42226},[285],[42228,42232,42237,42246,42250,42255],{"type":150,"tag":181,"props":42229,"children":42231},{"className":42230,"style":5470},[290],[],{"type":150,"tag":181,"props":42233,"children":42235},{"className":42234},[296],[42236],{"type":155,"value":35393},{"type":150,"tag":181,"props":42238,"children":42240},{"className":42239},[296,155],[42241],{"type":150,"tag":181,"props":42242,"children":42244},{"className":42243},[296],[42245],{"type":155,"value":23188},{"type":150,"tag":181,"props":42247,"children":42249},{"className":42248,"style":4993},[377],[],{"type":150,"tag":181,"props":42251,"children":42253},{"className":42252},[4998],[42254],{"type":155,"value":5380},{"type":150,"tag":181,"props":42256,"children":42258},{"className":42257,"style":4993},[377],[],{"type":150,"tag":181,"props":42260,"children":42262},{"className":42261},[285],[42263,42267,42272,42277,42286,42290,42295],{"type":150,"tag":181,"props":42264,"children":42266},{"className":42265,"style":5680},[290],[],{"type":150,"tag":181,"props":42268,"children":42270},{"className":42269},[296],[42271],{"type":155,"value":37783},{"type":150,"tag":181,"props":42273,"children":42275},{"className":42274},[377],[42276],{"type":155,"value":8576},{"type":150,"tag":181,"props":42278,"children":42280},{"className":42279},[296,155],[42281],{"type":150,"tag":181,"props":42282,"children":42284},{"className":42283},[296,24835],[42285],{"type":155,"value":24838},{"type":150,"tag":181,"props":42287,"children":42289},{"className":42288,"style":658},[377],[],{"type":150,"tag":181,"props":42291,"children":42293},{"className":42292},[663],[42294],{"type":155,"value":865},{"type":150,"tag":181,"props":42296,"children":42298},{"className":42297,"style":658},[377],[],{"type":150,"tag":181,"props":42300,"children":42302},{"className":42301},[285],[42303,42307,42312,42317],{"type":150,"tag":181,"props":42304,"children":42306},{"className":42305,"style":5680},[290],[],{"type":150,"tag":181,"props":42308,"children":42310},{"className":42309},[296],[42311],{"type":155,"value":9202},{"type":150,"tag":181,"props":42313,"children":42315},{"className":42314},[377],[42316],{"type":155,"value":8576},{"type":150,"tag":181,"props":42318,"children":42320},{"className":42319},[296,155],[42321],{"type":150,"tag":181,"props":42322,"children":42324},{"className":42323},[296],[42325],{"type":155,"value":29439},{"type":150,"tag":151,"props":42327,"children":42328},{},[42329],{"type":155,"value":42330},"而这 16B 激活参数同样散在四张卡上，各卡只读自己那一份，所以每卡读四分之一：",{"type":150,"tag":181,"props":42332,"children":42334},{"className":42333},[1003],[42335],{"type":150,"tag":181,"props":42336,"children":42338},{"className":42337},[184],[42339,42387],{"type":150,"tag":181,"props":42340,"children":42342},{"className":42341},[189],[42343],{"type":150,"tag":192,"props":42344,"children":42345},{"xmlns":194,"display":1015},[42346],{"type":150,"tag":197,"props":42347,"children":42348},{},[42349,42382],{"type":150,"tag":201,"props":42350,"children":42351},{},[42352,42370,42374,42378],{"type":150,"tag":1774,"props":42353,"children":42354},{},[42355,42366],{"type":150,"tag":201,"props":42356,"children":42357},{},[42358,42362],{"type":150,"tag":215,"props":42359,"children":42360},{},[42361],{"type":155,"value":9202},{"type":150,"tag":249,"props":42363,"children":42364},{},[42365],{"type":155,"value":29293},{"type":150,"tag":215,"props":42367,"children":42368},{},[42369],{"type":155,"value":22376},{"type":150,"tag":221,"props":42371,"children":42372},{},[42373],{"type":155,"value":865},{"type":150,"tag":215,"props":42375,"children":42376},{},[42377],{"type":155,"value":238},{"type":150,"tag":249,"props":42379,"children":42380},{},[42381],{"type":155,"value":29293},{"type":150,"tag":271,"props":42383,"children":42384},{"encoding":273},[42385],{"type":155,"value":42386},"\\frac{8\\ \\text{GB}}{4} = 2\\ \\text{GB}",{"type":150,"tag":181,"props":42388,"children":42390},{"className":42389,"ariaHidden":223},[280],[42391,42513],{"type":150,"tag":181,"props":42392,"children":42394},{"className":42393},[285],[42395,42399,42500,42504,42509],{"type":150,"tag":181,"props":42396,"children":42398},{"className":42397,"style":31545},[290],[],{"type":150,"tag":181,"props":42400,"children":42402},{"className":42401},[296],[42403,42407,42496],{"type":150,"tag":181,"props":42404,"children":42406},{"className":42405},[824,2482],[],{"type":150,"tag":181,"props":42408,"children":42410},{"className":42409},[1774],[42411],{"type":150,"tag":181,"props":42412,"children":42414},{"className":42413},[312,313],[42415,42485],{"type":150,"tag":181,"props":42416,"children":42418},{"className":42417},[318],[42419,42480],{"type":150,"tag":181,"props":42420,"children":42422},{"className":42421,"style":23490},[323],[42423,42439,42450],{"type":150,"tag":181,"props":42424,"children":42425},{"style":8235},[42426,42430],{"type":150,"tag":181,"props":42427,"children":42429},{"className":42428,"style":8240},[333],[],{"type":150,"tag":181,"props":42431,"children":42433},{"className":42432},[296],[42434],{"type":150,"tag":181,"props":42435,"children":42437},{"className":42436},[296],[42438],{"type":155,"value":22376},{"type":150,"tag":181,"props":42440,"children":42441},{"style":8253},[42442,42446],{"type":150,"tag":181,"props":42443,"children":42445},{"className":42444,"style":8240},[333],[],{"type":150,"tag":181,"props":42447,"children":42449},{"className":42448,"style":2804},[2803],[],{"type":150,"tag":181,"props":42451,"children":42452},{"style":8265},[42453,42457],{"type":150,"tag":181,"props":42454,"children":42456},{"className":42455,"style":8240},[333],[],{"type":150,"tag":181,"props":42458,"children":42460},{"className":42459},[296],[42461,42466,42471],{"type":150,"tag":181,"props":42462,"children":42464},{"className":42463},[296],[42465],{"type":155,"value":9202},{"type":150,"tag":181,"props":42467,"children":42469},{"className":42468},[377],[42470],{"type":155,"value":8576},{"type":150,"tag":181,"props":42472,"children":42474},{"className":42473},[296,155],[42475],{"type":150,"tag":181,"props":42476,"children":42478},{"className":42477},[296],[42479],{"type":155,"value":29439},{"type":150,"tag":181,"props":42481,"children":42483},{"className":42482},[352],[42484],{"type":155,"value":355},{"type":150,"tag":181,"props":42486,"children":42488},{"className":42487},[318],[42489],{"type":150,"tag":181,"props":42490,"children":42492},{"className":42491,"style":8297},[323],[42493],{"type":150,"tag":181,"props":42494,"children":42495},{},[],{"type":150,"tag":181,"props":42497,"children":42499},{"className":42498},[835,2482],[],{"type":150,"tag":181,"props":42501,"children":42503},{"className":42502,"style":658},[377],[],{"type":150,"tag":181,"props":42505,"children":42507},{"className":42506},[663],[42508],{"type":155,"value":865},{"type":150,"tag":181,"props":42510,"children":42512},{"className":42511,"style":658},[377],[],{"type":150,"tag":181,"props":42514,"children":42516},{"className":42515},[285],[42517,42521,42526,42531],{"type":150,"tag":181,"props":42518,"children":42520},{"className":42519,"style":5680},[290],[],{"type":150,"tag":181,"props":42522,"children":42524},{"className":42523},[296],[42525],{"type":155,"value":238},{"type":150,"tag":181,"props":42527,"children":42529},{"className":42528},[377],[42530],{"type":155,"value":8576},{"type":150,"tag":181,"props":42532,"children":42534},{"className":42533},[296,155],[42535],{"type":150,"tag":181,"props":42536,"children":42538},{"className":42537},[296],[42539],{"type":155,"value":29439},{"type":150,"tag":151,"props":42541,"children":42542},{},[42543],{"type":155,"value":42544},"KV 是同一个道理。TP 把 KV head 也分到四张卡上，每卡只存自己那部分，也就只读自己那部分：",{"type":150,"tag":181,"props":42546,"children":42548},{"className":42547},[1003],[42549],{"type":150,"tag":181,"props":42550,"children":42552},{"className":42551},[184],[42553,42617],{"type":150,"tag":181,"props":42554,"children":42556},{"className":42555},[189],[42557],{"type":150,"tag":192,"props":42558,"children":42559},{"xmlns":194,"display":1015},[42560],{"type":150,"tag":197,"props":42561,"children":42562},{},[42563,42612],{"type":150,"tag":201,"props":42564,"children":42565},{},[42566,42599,42603,42608],{"type":150,"tag":1774,"props":42567,"children":42568},{},[42569,42595],{"type":150,"tag":201,"props":42570,"children":42571},{},[42572,42576,42580,42584],{"type":150,"tag":215,"props":42573,"children":42574},{},[42575],{"type":155,"value":37937},{"type":150,"tag":249,"props":42577,"children":42578},{},[42579],{"type":155,"value":39124},{"type":150,"tag":221,"props":42581,"children":42582},{},[42583],{"type":155,"value":5380},{"type":150,"tag":565,"props":42585,"children":42586},{},[42587,42591],{"type":150,"tag":215,"props":42588,"children":42589},{},[42590],{"type":155,"value":25633},{"type":150,"tag":215,"props":42592,"children":42593},{},[42594],{"type":155,"value":37953},{"type":150,"tag":215,"props":42596,"children":42597},{},[42598],{"type":155,"value":22376},{"type":150,"tag":221,"props":42600,"children":42601},{},[42602],{"type":155,"value":865},{"type":150,"tag":215,"props":42604,"children":42605},{},[42606],{"type":155,"value":42607},"0.23",{"type":150,"tag":249,"props":42609,"children":42610},{},[42611],{"type":155,"value":29293},{"type":150,"tag":271,"props":42613,"children":42614},{"encoding":273},[42615],{"type":155,"value":42616},"\\frac{890\\ \\text{B} \\times 10^6}{4} = 0.23\\ \\text{GB}",{"type":150,"tag":181,"props":42618,"children":42620},{"className":42619,"ariaHidden":223},[280],[42621,42803],{"type":150,"tag":181,"props":42622,"children":42624},{"className":42623},[285],[42625,42630,42790,42794,42799],{"type":150,"tag":181,"props":42626,"children":42629},{"className":42627,"style":42628},[290],"height:2.1771em;vertical-align:-0.686em;",[],{"type":150,"tag":181,"props":42631,"children":42633},{"className":42632},[296],[42634,42638,42786],{"type":150,"tag":181,"props":42635,"children":42637},{"className":42636},[824,2482],[],{"type":150,"tag":181,"props":42639,"children":42641},{"className":42640},[1774],[42642],{"type":150,"tag":181,"props":42643,"children":42645},{"className":42644},[312,313],[42646,42775],{"type":150,"tag":181,"props":42647,"children":42649},{"className":42648},[318],[42650,42770],{"type":150,"tag":181,"props":42651,"children":42653},{"className":42652,"style":25791},[323],[42654,42670,42681],{"type":150,"tag":181,"props":42655,"children":42656},{"style":8235},[42657,42661],{"type":150,"tag":181,"props":42658,"children":42660},{"className":42659,"style":8240},[333],[],{"type":150,"tag":181,"props":42662,"children":42664},{"className":42663},[296],[42665],{"type":150,"tag":181,"props":42666,"children":42668},{"className":42667},[296],[42669],{"type":155,"value":22376},{"type":150,"tag":181,"props":42671,"children":42672},{"style":8253},[42673,42677],{"type":150,"tag":181,"props":42674,"children":42676},{"className":42675,"style":8240},[333],[],{"type":150,"tag":181,"props":42678,"children":42680},{"className":42679,"style":2804},[2803],[],{"type":150,"tag":181,"props":42682,"children":42683},{"style":8265},[42684,42688],{"type":150,"tag":181,"props":42685,"children":42687},{"className":42686,"style":8240},[333],[],{"type":150,"tag":181,"props":42689,"children":42691},{"className":42690},[296],[42692,42697,42702,42711,42715,42720,42724,42729],{"type":150,"tag":181,"props":42693,"children":42695},{"className":42694},[296],[42696],{"type":155,"value":37937},{"type":150,"tag":181,"props":42698,"children":42700},{"className":42699},[377],[42701],{"type":155,"value":8576},{"type":150,"tag":181,"props":42703,"children":42705},{"className":42704},[296,155],[42706],{"type":150,"tag":181,"props":42707,"children":42709},{"className":42708},[296],[42710],{"type":155,"value":23188},{"type":150,"tag":181,"props":42712,"children":42714},{"className":42713,"style":4993},[377],[],{"type":150,"tag":181,"props":42716,"children":42718},{"className":42717},[4998],[42719],{"type":155,"value":5380},{"type":150,"tag":181,"props":42721,"children":42723},{"className":42722,"style":4993},[377],[],{"type":150,"tag":181,"props":42725,"children":42727},{"className":42726},[296],[42728],{"type":155,"value":219},{"type":150,"tag":181,"props":42730,"children":42732},{"className":42731},[296],[42733,42738],{"type":150,"tag":181,"props":42734,"children":42736},{"className":42735},[296],[42737],{"type":155,"value":25836},{"type":150,"tag":181,"props":42739,"children":42741},{"className":42740},[307],[42742],{"type":150,"tag":181,"props":42743,"children":42745},{"className":42744},[312],[42746],{"type":150,"tag":181,"props":42747,"children":42749},{"className":42748},[318],[42750],{"type":150,"tag":181,"props":42751,"children":42753},{"className":42752,"style":6831},[323],[42754],{"type":150,"tag":181,"props":42755,"children":42756},{"style":707},[42757,42761],{"type":150,"tag":181,"props":42758,"children":42760},{"className":42759,"style":334},[333],[],{"type":150,"tag":181,"props":42762,"children":42764},{"className":42763},[339,340,341,342],[42765],{"type":150,"tag":181,"props":42766,"children":42768},{"className":42767},[296,342],[42769],{"type":155,"value":37953},{"type":150,"tag":181,"props":42771,"children":42773},{"className":42772},[352],[42774],{"type":155,"value":355},{"type":150,"tag":181,"props":42776,"children":42778},{"className":42777},[318],[42779],{"type":150,"tag":181,"props":42780,"children":42782},{"className":42781,"style":8297},[323],[42783],{"type":150,"tag":181,"props":42784,"children":42785},{},[],{"type":150,"tag":181,"props":42787,"children":42789},{"className":42788},[835,2482],[],{"type":150,"tag":181,"props":42791,"children":42793},{"className":42792,"style":658},[377],[],{"type":150,"tag":181,"props":42795,"children":42797},{"className":42796},[663],[42798],{"type":155,"value":865},{"type":150,"tag":181,"props":42800,"children":42802},{"className":42801,"style":658},[377],[],{"type":150,"tag":181,"props":42804,"children":42806},{"className":42805},[285],[42807,42811,42816,42821],{"type":150,"tag":181,"props":42808,"children":42810},{"className":42809,"style":5680},[290],[],{"type":150,"tag":181,"props":42812,"children":42814},{"className":42813},[296],[42815],{"type":155,"value":42607},{"type":150,"tag":181,"props":42817,"children":42819},{"className":42818},[377],[42820],{"type":155,"value":8576},{"type":150,"tag":181,"props":42822,"children":42824},{"className":42823},[296,155],[42825],{"type":150,"tag":181,"props":42826,"children":42828},{"className":42827},[296],[42829],{"type":155,"value":29439},{"type":150,"tag":151,"props":42831,"children":42832},{},[42833,42838],{"type":150,"tag":158,"props":42834,"children":42835},{},[42836],{"type":155,"value":42837},"两项加起来，KV 只占 10%。",{"type":155,"value":42839}," 和前面 batch 1 的结论一致，TP 不改变这个比例。",{"type":150,"tag":151,"props":42841,"children":42842},{},[42843],{"type":155,"value":42844},"四张卡合起来，节点级 KV 容量是 444 GB，按 890 字节 \u002F token 算就是 4.99 亿 token。单序列的理论上限也从 896 提到约 3,600 token\u002Fs。",{"type":150,"tag":151,"props":42846,"children":42847},{},[42848],{"type":155,"value":42849},"这就是 ds4.1f-FP4 在 B200 上的 roofline。",{"type":150,"tag":151,"props":42851,"children":42852},{},[42853,42855,42860],{"type":155,"value":42854},"拐点 1125 FLOP\u002Fbyte 换一种读法：",{"type":150,"tag":158,"props":42856,"children":42857},{},[42858],{"type":155,"value":42859},"搬运 1 个字节的时间，够做 1125 次浮点运算",{"type":155,"value":42861},"。于是一个机制只要能用不到 1125 次运算省下 1 个字节的搬运，就净赚。以这个比值衡量，ds4.1f 里的几个设计都在做同一笔交易：",{"type":150,"tag":23633,"props":42863,"children":42864},{},[42865,42987,42997],{"type":150,"tag":23637,"props":42866,"children":42867},{},[42868,42873,42875,42985],{"type":150,"tag":158,"props":42869,"children":42870},{},[42871],{"type":155,"value":42872},"SWA Bounded Replay",{"type":155,"value":42874},"：请求恢复时不重放整条序列，只重算最近 ",{"type":150,"tag":181,"props":42876,"children":42878},{"className":42877},[184],[42879,42908],{"type":150,"tag":181,"props":42880,"children":42882},{"className":42881},[189],[42883],{"type":150,"tag":192,"props":42884,"children":42885},{"xmlns":194},[42886],{"type":150,"tag":197,"props":42887,"children":42888},{},[42889,42903],{"type":150,"tag":201,"props":42890,"children":42891},{},[42892],{"type":150,"tag":205,"props":42893,"children":42894},{},[42895,42899],{"type":150,"tag":209,"props":42896,"children":42897},{},[42898],{"type":155,"value":1639},{"type":150,"tag":249,"props":42900,"children":42901},{},[42902],{"type":155,"value":36996},{"type":150,"tag":271,"props":42904,"children":42905},{"encoding":273},[42906],{"type":155,"value":42907},"n_{\\text{win}}",{"type":150,"tag":181,"props":42909,"children":42911},{"className":42910,"ariaHidden":223},[280],[42912],{"type":150,"tag":181,"props":42913,"children":42915},{"className":42914},[285],[42916,42920],{"type":150,"tag":181,"props":42917,"children":42919},{"className":42918,"style":1165},[290],[],{"type":150,"tag":181,"props":42921,"children":42923},{"className":42922},[296],[42924,42929],{"type":150,"tag":181,"props":42925,"children":42927},{"className":42926},[296,301],[42928],{"type":155,"value":1639},{"type":150,"tag":181,"props":42930,"children":42932},{"className":42931},[307],[42933],{"type":150,"tag":181,"props":42934,"children":42936},{"className":42935},[312,313],[42937,42974],{"type":150,"tag":181,"props":42938,"children":42940},{"className":42939},[318],[42941,42969],{"type":150,"tag":181,"props":42942,"children":42944},{"className":42943,"style":6304},[323],[42945],{"type":150,"tag":181,"props":42946,"children":42947},{"style":328},[42948,42952],{"type":150,"tag":181,"props":42949,"children":42951},{"className":42950,"style":334},[333],[],{"type":150,"tag":181,"props":42953,"children":42955},{"className":42954},[339,340,341,342],[42956],{"type":150,"tag":181,"props":42957,"children":42959},{"className":42958},[296,342],[42960],{"type":150,"tag":181,"props":42961,"children":42963},{"className":42962},[296,155,342],[42964],{"type":150,"tag":181,"props":42965,"children":42967},{"className":42966},[296,342],[42968],{"type":155,"value":36996},{"type":150,"tag":181,"props":42970,"children":42972},{"className":42971},[352],[42973],{"type":155,"value":355},{"type":150,"tag":181,"props":42975,"children":42977},{"className":42976},[318],[42978],{"type":150,"tag":181,"props":42979,"children":42981},{"className":42980,"style":363},[323],[42982],{"type":150,"tag":181,"props":42983,"children":42984},{},[],{"type":155,"value":42986}," 个 token 来重建局部状态。用一段计算，换掉局部窗口 KV 的持久化。",{"type":150,"tag":23637,"props":42988,"children":42989},{},[42990,42995],{"type":150,"tag":158,"props":42991,"children":42992},{},[42993],{"type":155,"value":42994},"FP4 KV",{"type":155,"value":42996},"：存储和带宽都降到四分之一，代价是读进来要反量化。用一点计算，换掉四分之三的字节。",{"type":150,"tag":23637,"props":42998,"children":42999},{},[43000,43005],{"type":150,"tag":158,"props":43001,"children":43002},{},[43003],{"type":155,"value":43004},"Hierarchical Indexer",{"type":155,"value":43006},"：先用一层打分粗筛出候选池，后续层只在这个池子里挑。用一层索引，换掉对全部候选的扫描。",{"type":150,"tag":151,"props":43008,"children":43009},{},[43010],{"type":155,"value":43011},"共同点是把成本从 memory 一侧挪到 compute 一侧。",{"type":150,"tag":151,"props":43013,"children":43014},{},[43015,43017],{"type":155,"value":43016},"前面几代的动作一直是压缩存储。现在存储已经压到接近下限，规则反过来：",{"type":150,"tag":158,"props":43018,"children":43019},{},[43020],{"type":155,"value":43021},"用富余的算力，去重建那些不值得搬过来的字节。",{"type":150,"tag":151,"props":43023,"children":43024},{},[43025],{"type":155,"value":43026},"这些省下来的资源最终可以被重新花到：更多 tokens、更好的数据过滤、更长的 RL rollout、更多 environment interaction、更多 rejection sampling、更多 verifier compute、更大的 batch、更复杂的 agent tasks。到这里，可以回到最初的问题了。ds4.1f 省掉的，是账单上几乎每一项。",{"type":150,"tag":169,"props":43028,"children":43030},{"id":43029},"capability-与-efficiency",[43031],{"type":155,"value":43032},"Capability 与 Efficiency",{"type":150,"tag":151,"props":43034,"children":43035},{},[43036],{"type":155,"value":43037},"业界基本有共识：能力里很大的一部分来自后训练，而不是架构。三个例子。",{"type":150,"tag":151,"props":43039,"children":43040},{},[43041,43043,43050],{"type":155,"value":43042},"GLM-5.3 的发布说明只有一句话，",{"type":150,"tag":9663,"props":43044,"children":43047},{"href":43045,"rel":43046},"https:\u002F\u002Fz.ai\u002Fblog\u002Fglm-5.3",[9667],[43048],{"type":155,"value":43049},"Scaling post-training is all we did",{"type":155,"value":43051},"。基座和 GLM-5.2 一样，没有新增一个预训练 token，Terminal-Bench 3.0 从 4.6 涨到 28.3，六倍。",{"type":150,"tag":151,"props":43053,"children":43054},{},[43055,43057,43064],{"type":155,"value":43056},"Cognition 的 ",{"type":150,"tag":9663,"props":43058,"children":43061},{"href":43059,"rel":43060},"https:\u002F\u002Fcognition.com\u002Fblog\u002Fswe-2",[9667],[43062],{"type":155,"value":43063},"SWE-2",{"type":155,"value":43065}," 更能说明分工。它是一个美国实验室从 Kimi K3 的开源权重上后训练出来的，而 Kimi K3 是 Moonshot 放出来的 2.8T 参数模型。Cognition 报的成绩是编码能力离前沿只差一点，成本低 64%；首次真正动手改代码的中位步数从 48 降到 18。",{"type":150,"tag":151,"props":43067,"children":43068},{},[43069],{"type":155,"value":43070},"开源权重到了这个水平，做一款有竞争力的编码产品，贵的环节已经不是预训练，而是后训练和 harness。",{"type":150,"tag":151,"props":43072,"children":43073},{},[43074],{"type":155,"value":43075},"第三个例子是 DeepSeek V4-Flash-0731。它的模型结构和尺寸与 Preview 保持一致，只重新做了后训练，但 Agent benchmark 大幅提升：",{"type":150,"tag":151,"props":43077,"children":43078},{},[43079,43081,43086],{"type":155,"value":43080},"DeepSWE 从 7.3 涨到 54.4，CyberGym 从 38.7 到 76.7，TerminalBench 2.1 从 61.8 到 82.7。模型 backbone 没换。Attention 没换。参数量没换。官方更新日志直接写明结构不变、仅重新后训练。这个例子比任何口号都说明问题：",{"type":150,"tag":158,"props":43082,"children":43083},{},[43084],{"type":155,"value":43085},"同一个 backbone 的 post-training 差异，完全可能比两代 architecture 的差异更大。",{"type":155,"value":43087}," 所以把\"模型进步\"拆成四个层次更合适：",{"type":150,"tag":22173,"props":43089,"children":43090},{},[43091,43112],{"type":150,"tag":22177,"props":43092,"children":43093},{},[43094],{"type":150,"tag":22181,"props":43095,"children":43096},{},[43097,43102,43107],{"type":150,"tag":22185,"props":43098,"children":43099},{},[43100],{"type":155,"value":43101},"层次",{"type":150,"tag":22185,"props":43103,"children":43104},{},[43105],{"type":155,"value":43106},"解决什么",{"type":150,"tag":22185,"props":43108,"children":43109},{},[43110],{"type":155,"value":43111},"对能力的作用",{"type":150,"tag":22199,"props":43113,"children":43114},{},[43115,43133,43151,43169],{"type":150,"tag":22181,"props":43116,"children":43117},{},[43118,43123,43128],{"type":150,"tag":22206,"props":43119,"children":43120},{},[43121],{"type":155,"value":43122},"Architecture",{"type":150,"tag":22206,"props":43124,"children":43125},{},[43126],{"type":155,"value":43127},"每 FLOP \u002F 每 GB 能做多少事",{"type":150,"tag":22206,"props":43129,"children":43130},{},[43131],{"type":155,"value":43132},"提高上限和效率",{"type":150,"tag":22181,"props":43134,"children":43135},{},[43136,43141,43146],{"type":150,"tag":22206,"props":43137,"children":43138},{},[43139],{"type":155,"value":43140},"Data",{"type":150,"tag":22206,"props":43142,"children":43143},{},[43144],{"type":155,"value":43145},"模型到底看到了什么分布",{"type":150,"tag":22206,"props":43147,"children":43148},{},[43149],{"type":155,"value":43150},"决定知识、能力覆盖",{"type":150,"tag":22181,"props":43152,"children":43153},{},[43154,43159,43164],{"type":150,"tag":22206,"props":43155,"children":43156},{},[43157],{"type":155,"value":43158},"Post-training \u002F RL",{"type":150,"tag":22206,"props":43160,"children":43161},{},[43162],{"type":155,"value":43163},"学会怎么推理、用工具、完成任务",{"type":150,"tag":22206,"props":43165,"children":43166},{},[43167],{"type":155,"value":43168},"直接决定最终行为能力",{"type":150,"tag":22181,"props":43170,"children":43171},{},[43172,43177,43182],{"type":150,"tag":22206,"props":43173,"children":43174},{},[43175],{"type":155,"value":43176},"Infra",{"type":150,"tag":22206,"props":43178,"children":43179},{},[43180],{"type":155,"value":43181},"上面三件事是否真的按设计运行",{"type":150,"tag":22206,"props":43183,"children":43184},{},[43185],{"type":155,"value":43186},"防止几个月训练被 bug 偷走",{"type":150,"tag":151,"props":43188,"children":43189},{},[43190,43192,43198,43200,43205],{"type":155,"value":43191},"Infra 这一层最容易被低估。RL 的产出取决于一条很长的系统，公式只占其中一环：",{"type":150,"tag":34051,"props":43193,"children":43195},{"className":43194},[],[43196],{"type":155,"value":43197},"prompt → rollout → tool environment → reward → sampling → advantage → gradient → model update",{"type":155,"value":43199},"。任何地方一个小 bug，最后都会变成同一句话：\"奇怪，我们这个新的 RL 算法怎么效果一般？\" 而那个 bug 可能是 reward 算错、tool state 泄漏、rollout truncation 错、mask 错、sampling distribution 不一致、tokenizer offset 错、advantage normalization 错、stale policy 太严重，或者 sandbox reset 有问题。实际上不是算法一般，是代码在偷偷犯罪。所以更准的因果链是架构先降低成本，省下的成本换来更有效的训练，训练再沉淀成能力；\"架构直接产生智能\"是另一种叙事。架构对能力的作用经常是",{"type":150,"tag":158,"props":43201,"children":43202},{},[43203],{"type":155,"value":43204},"间接的",{"type":155,"value":43206},"。如果问\"这一代模型为什么明显更好用了\"，重要性排序大致是 Data > Post-training\u002FRL > Eval\u002FVerifier > Training infra > Architecture，答案多半落在前面几项，而不在\"我们把 attention 又改了一个 acronym\"。旁边还有一组更好记的说法，把每一层对应到一个\"前沿\"：",{"type":150,"tag":23633,"props":43208,"children":43209},{},[43210,43222,43234,43246],{"type":150,"tag":23637,"props":43211,"children":43212},{},[43213,43215,43220],{"type":155,"value":43214},"Architecture 决定 ",{"type":150,"tag":158,"props":43216,"children":43217},{},[43218],{"type":155,"value":43219},"efficiency frontier",{"type":155,"value":43221},"：每单位算力能处理多少有效训练信号",{"type":150,"tag":23637,"props":43223,"children":43224},{},[43225,43227,43232],{"type":155,"value":43226},"Data 决定 ",{"type":150,"tag":158,"props":43228,"children":43229},{},[43230],{"type":155,"value":43231},"learning frontier",{"type":155,"value":43233},"：模型有机会学到什么",{"type":150,"tag":23637,"props":43235,"children":43236},{},[43237,43239,43244],{"type":155,"value":43238},"RL \u002F post-training 决定 ",{"type":150,"tag":158,"props":43240,"children":43241},{},[43242],{"type":155,"value":43243},"behavior frontier",{"type":155,"value":43245},"：模型会不会把能力真正用出来",{"type":150,"tag":23637,"props":43247,"children":43248},{},[43249,43251,43256],{"type":155,"value":43250},"Infra 决定",{"type":150,"tag":158,"props":43252,"children":43253},{},[43254],{"type":155,"value":43255},"你是否真的获得前三者",{"type":155,"value":43257},"：实验结果到底是不是你以为的实验",{"type":150,"tag":151,"props":43259,"children":43260},{},[43261],{"type":155,"value":43262},"Infra 这一条听起来最不起眼，出问题时损失最大。一个漂亮的新 loss 写在论文里只需要半页；一个可靠的 million-token agent rollout system，可能需要几十个人跟各种分布式 bug、CUDA bug、环境 bug、数据 bug 打几个月架。论文不会把凌晨三点的 NCCL timeout 写进公式里，但模型能力很可能就死在那里。KDA 和 CSA\u002FHCA 的作用前面已经说过：让 1M context 和 long-horizon RL rollout 在计算上变得现实。放进 infra 的语境再看一层：这套系统跑不起来，再好的 loss 也只是半页论文。",{"type":150,"tag":151,"props":43264,"children":43265},{},[43266],{"type":150,"tag":158,"props":43267,"children":43268},{},[43269],{"type":155,"value":43270},"架构是 compute multiplier，乘出来的是训练预算，不是智能本身。",{"type":150,"tag":169,"props":43272,"children":43274},{"id":43273},"算力预算",[43275],{"type":155,"value":43273},{"type":150,"tag":151,"props":43277,"children":43278},{},[43279],{"type":155,"value":43280},"如果架构主要在省钱，那\"该不该冒架构的险\"就变成了一个纯粹的预算问题。假设已验证的 Transformer scaling recipe 能拿到 95 分。新架构理论上能到 100，但有 20% 概率训练炸掉，只剩 70。如果算力富裕，为什么要冒险？把已验证的架构再 scale 10 倍，可能直接拿到 98，而且更可靠。这其实是一种\"富人的保守\"。而算力紧张的实验室没有 10× compute 可以挥霍，只能想办法让 1× compute 做出 3× compute 的效果，架构创新的 ROI 因此变得非常高。从优化目标看更直白。算力富裕的一方更接近：",{"type":150,"tag":181,"props":43282,"children":43284},{"className":43283},[1003],[43285],{"type":150,"tag":181,"props":43286,"children":43288},{"className":43287},[184],[43289,43332],{"type":150,"tag":181,"props":43290,"children":43292},{"className":43291},[189],[43293],{"type":150,"tag":192,"props":43294,"children":43295},{"xmlns":194,"display":1015},[43296],{"type":150,"tag":197,"props":43297,"children":43298},{},[43299,43327],{"type":150,"tag":201,"props":43300,"children":43301},{},[43302,43306,43310,43315,43319,43322],{"type":150,"tag":209,"props":43303,"children":43304},{},[43305],{"type":155,"value":41284},{"type":150,"tag":221,"props":43307,"children":43308},{},[43309],{"type":155,"value":1813},{"type":150,"tag":249,"props":43311,"children":43312},{},[43313],{"type":155,"value":43314},"Capability",{"type":150,"tag":221,"props":43316,"children":43317},{"separator":223},[43318],{"type":155,"value":226},{"type":150,"tag":377,"props":43320,"children":43321},{"width":3313},[],{"type":150,"tag":249,"props":43323,"children":43324},{},[43325],{"type":155,"value":43326},"s.t. Compute 很大",{"type":150,"tag":271,"props":43328,"children":43329},{"encoding":273},[43330],{"type":155,"value":43331},"\\max \\text{Capability}, \\quad \\text{s.t. Compute 很大}",{"type":150,"tag":181,"props":43333,"children":43335},{"className":43334,"ariaHidden":223},[280],[43336],{"type":150,"tag":181,"props":43337,"children":43339},{"className":43338},[285],[43340,43344,43349,43353,43362,43367,43371,43375],{"type":150,"tag":181,"props":43341,"children":43343},{"className":43342,"style":941},[290],[],{"type":150,"tag":181,"props":43345,"children":43347},{"className":43346},[2517],[43348],{"type":155,"value":41284},{"type":150,"tag":181,"props":43350,"children":43352},{"className":43351,"style":378},[377],[],{"type":150,"tag":181,"props":43354,"children":43356},{"className":43355},[296,155],[43357],{"type":150,"tag":181,"props":43358,"children":43360},{"className":43359},[296],[43361],{"type":155,"value":43314},{"type":150,"tag":181,"props":43363,"children":43365},{"className":43364},[371],[43366],{"type":155,"value":226},{"type":150,"tag":181,"props":43368,"children":43370},{"className":43369,"style":3909},[377],[],{"type":150,"tag":181,"props":43372,"children":43374},{"className":43373,"style":378},[377],[],{"type":150,"tag":181,"props":43376,"children":43378},{"className":43377},[296,155],[43379,43385],{"type":150,"tag":181,"props":43380,"children":43382},{"className":43381},[296],[43383],{"type":155,"value":43384},"s.t. Compute ",{"type":150,"tag":181,"props":43386,"children":43388},{"className":43387},[296,24835],[43389],{"type":155,"value":43390},"很大",{"type":150,"tag":151,"props":43392,"children":43393},{},[43394],{"type":155,"value":43395},"所以可以接受较低的 architecture efficiency，只要 scaling 稳。而算力受限的一方更像：",{"type":150,"tag":181,"props":43397,"children":43399},{"className":43398},[1003],[43400],{"type":150,"tag":181,"props":43401,"children":43403},{"className":43402},[184],[43404,43442],{"type":150,"tag":181,"props":43405,"children":43407},{"className":43406},[189],[43408],{"type":150,"tag":192,"props":43409,"children":43410},{"xmlns":194,"display":1015},[43411],{"type":150,"tag":197,"props":43412,"children":43413},{},[43414,43437],{"type":150,"tag":201,"props":43415,"children":43416},{},[43417,43421,43425],{"type":150,"tag":209,"props":43418,"children":43419},{},[43420],{"type":155,"value":41284},{"type":150,"tag":221,"props":43422,"children":43423},{},[43424],{"type":155,"value":1813},{"type":150,"tag":1774,"props":43426,"children":43427},{},[43428,43432],{"type":150,"tag":249,"props":43429,"children":43430},{},[43431],{"type":155,"value":43314},{"type":150,"tag":249,"props":43433,"children":43434},{},[43435],{"type":155,"value":43436},"Compute",{"type":150,"tag":271,"props":43438,"children":43439},{"encoding":273},[43440],{"type":155,"value":43441},"\\max \\frac{\\text{Capability}}{\\text{Compute}}",{"type":150,"tag":181,"props":43443,"children":43445},{"className":43444,"ariaHidden":223},[280],[43446],{"type":150,"tag":181,"props":43447,"children":43449},{"className":43448},[285],[43450,43455,43460,43464],{"type":150,"tag":181,"props":43451,"children":43454},{"className":43452,"style":43453},[290],"height:2.2519em;vertical-align:-0.8804em;",[],{"type":150,"tag":181,"props":43456,"children":43458},{"className":43457},[2517],[43459],{"type":155,"value":41284},{"type":150,"tag":181,"props":43461,"children":43463},{"className":43462,"style":378},[377],[],{"type":150,"tag":181,"props":43465,"children":43467},{"className":43466},[296],[43468,43472,43557],{"type":150,"tag":181,"props":43469,"children":43471},{"className":43470},[824,2482],[],{"type":150,"tag":181,"props":43473,"children":43475},{"className":43474},[1774],[43476],{"type":150,"tag":181,"props":43477,"children":43479},{"className":43478},[312,313],[43480,43545],{"type":150,"tag":181,"props":43481,"children":43483},{"className":43482},[318],[43484,43540],{"type":150,"tag":181,"props":43485,"children":43488},{"className":43486,"style":43487},[323],"height:1.3714em;",[43489,43509,43520],{"type":150,"tag":181,"props":43490,"children":43491},{"style":8235},[43492,43496],{"type":150,"tag":181,"props":43493,"children":43495},{"className":43494,"style":8240},[333],[],{"type":150,"tag":181,"props":43497,"children":43499},{"className":43498},[296],[43500],{"type":150,"tag":181,"props":43501,"children":43503},{"className":43502},[296,155],[43504],{"type":150,"tag":181,"props":43505,"children":43507},{"className":43506},[296],[43508],{"type":155,"value":43436},{"type":150,"tag":181,"props":43510,"children":43511},{"style":8253},[43512,43516],{"type":150,"tag":181,"props":43513,"children":43515},{"className":43514,"style":8240},[333],[],{"type":150,"tag":181,"props":43517,"children":43519},{"className":43518,"style":2804},[2803],[],{"type":150,"tag":181,"props":43521,"children":43522},{"style":8265},[43523,43527],{"type":150,"tag":181,"props":43524,"children":43526},{"className":43525,"style":8240},[333],[],{"type":150,"tag":181,"props":43528,"children":43530},{"className":43529},[296],[43531],{"type":150,"tag":181,"props":43532,"children":43534},{"className":43533},[296,155],[43535],{"type":150,"tag":181,"props":43536,"children":43538},{"className":43537},[296],[43539],{"type":155,"value":43314},{"type":150,"tag":181,"props":43541,"children":43543},{"className":43542},[352],[43544],{"type":155,"value":355},{"type":150,"tag":181,"props":43546,"children":43548},{"className":43547},[318],[43549],{"type":150,"tag":181,"props":43550,"children":43553},{"className":43551,"style":43552},[323],"height:0.8804em;",[43554],{"type":150,"tag":181,"props":43555,"children":43556},{},[],{"type":150,"tag":181,"props":43558,"children":43560},{"className":43559},[835,2482],[],{"type":150,"tag":151,"props":43562,"children":43563},{},[43564],{"type":155,"value":43565},"于是被逼出 MLA、MoE、Sparse、Linear、各种 compression。这和芯片行业很像。资源充裕时加晶体管，资源受限时改 architecture。所以很多所谓架构路线之争，底层其实是不同算力预算下的经济学。把这件事再拆细一点，会发现竞争其实发生在四种资本上：",{"type":150,"tag":23633,"props":43567,"children":43568},{},[43569,43579,43589,43599],{"type":150,"tag":23637,"props":43570,"children":43571},{},[43572,43577],{"type":150,"tag":158,"props":43573,"children":43574},{},[43575],{"type":155,"value":43576},"Compute Capital",{"type":155,"value":43578},"：GPU、HBM、network、datacenter。",{"type":150,"tag":23637,"props":43580,"children":43581},{},[43582,43587],{"type":150,"tag":158,"props":43583,"children":43584},{},[43585],{"type":155,"value":43586},"Data Capital",{"type":155,"value":43588},"：真实用户、agent trajectories、enterprise workflow、feedback。",{"type":150,"tag":23637,"props":43590,"children":43591},{},[43592,43597],{"type":150,"tag":158,"props":43593,"children":43594},{},[43595],{"type":155,"value":43596},"Algorithmic Capital",{"type":155,"value":43598},"：architecture、training recipe、RL、verifier。",{"type":150,"tag":23637,"props":43600,"children":43601},{},[43602,43607],{"type":150,"tag":158,"props":43603,"children":43604},{},[43605],{"type":155,"value":43606},"Infrastructure Capital",{"type":155,"value":43608},"：稳定地把几千几万张 GPU、RL environment、rollout 全跑起来。",{"type":150,"tag":151,"props":43610,"children":43611},{},[43612,43614],{"type":155,"value":43613},"算力富裕的一方在前两项和第四项上有巨大积累；算力受限的一方被迫在第三项特别凶。这也是为什么\"美国靠暴力 scaling、中国靠聪明架构\"这种说法太简化了。更接近事实的写法是：",{"type":150,"tag":158,"props":43615,"children":43616},{},[43617],{"type":155,"value":43618},"一边在用资源优势降低研究风险，另一边在用资源约束提高创新收益。",{"type":150,"tag":151,"props":43620,"children":43621},{},[43622],{"type":155,"value":43623},"而且\"数据\"这一项也没法简单说成谁多谁少。真正稀缺的早就不是互联网文本了。Wikipedia、Common Crawl、GitHub 这些 public corpus，大家拿到的量级差不多。现在值钱的是高质量 reasoning traces、真实 agent trajectories、用户交互反馈、专业任务数据、tool-use traces、可验证环境。这些东西不是爬虫能给你的，是产品规模给的。",{"type":150,"tag":151,"props":43625,"children":43626},{},[43627],{"type":155,"value":43628},"所以更要命的不是\"谁有更多互联网 token\"，而是\"谁有更多高价值的 interaction token\"。但这里有个反转。",{"type":150,"tag":151,"props":43630,"children":43631},{},[43632],{"type":155,"value":43633},"一旦某个架构优化被证明有效，情况就变了。DeepSeek 做出 MLA，或者 Kimi 做出成熟的 Linear Attention，算力多的一方并不会因为自己算力多就拒绝使用。它们完全可以拿\"更多算力 × 更高效架构\"叠加使用。于是原本弱者为了生存开发出来的 efficiency innovation，最后反而可能让强者 scale 得更狠。问题最后又回到：省下来的钱，值不值得花。",{"type":150,"tag":169,"props":43635,"children":43637},{"id":43636},"小结",[43638],{"type":155,"value":43636},{"type":150,"tag":151,"props":43640,"children":43641},{},[43642],{"type":155,"value":43643},"开头说 ds4.1f 是模型架构与硬件 co-design 的杰作。看完整条路线，我觉得这个说法可以讲得更具体一点：它省掉的不是某一项。是账单上的好几项，而且是分几代攒下来的。把整条路线画成一张图是这样：",{"type":150,"tag":34045,"props":43645,"children":43647},{"className":34047,"code":43646,"language":155,"meta":28,"style":28},"MHA              每个 head 一份 K、V\n │\n ▼\nMQA \u002F GQA        压 head 数：离散地共享\n │\n ▼\nMLA              压 D 轴：连续压缩成 latent\n │\n ▼\nCSA \u002F HCA        压 N 轴：压成 block，再挑着看\n │\n ▼\nLinear \u002F SSM     压掉保存本身：只留一个 state\n │\n ▼\nV4.1             压 L 轴和计算阶段：不再每层每步都付同样的钱\n",[43648],{"type":150,"tag":34051,"props":43649,"children":43650},{"__ignoreMap":28},[43651,43659,43667,43675,43683,43690,43697,43705,43712,43719,43727,43734,43741,43750,43758,43766],{"type":150,"tag":181,"props":43652,"children":43653},{"class":34056,"line":34057},[43654],{"type":150,"tag":181,"props":43655,"children":43656},{},[43657],{"type":155,"value":43658},"MHA              每个 head 一份 K、V\n",{"type":150,"tag":181,"props":43660,"children":43661},{"class":34056,"line":34066},[43662],{"type":150,"tag":181,"props":43663,"children":43664},{},[43665],{"type":155,"value":43666}," │\n",{"type":150,"tag":181,"props":43668,"children":43669},{"class":34056,"line":34075},[43670],{"type":150,"tag":181,"props":43671,"children":43672},{},[43673],{"type":155,"value":43674}," ▼\n",{"type":150,"tag":181,"props":43676,"children":43677},{"class":34056,"line":34084},[43678],{"type":150,"tag":181,"props":43679,"children":43680},{},[43681],{"type":155,"value":43682},"MQA \u002F GQA        压 head 数：离散地共享\n",{"type":150,"tag":181,"props":43684,"children":43685},{"class":34056,"line":34092},[43686],{"type":150,"tag":181,"props":43687,"children":43688},{},[43689],{"type":155,"value":43666},{"type":150,"tag":181,"props":43691,"children":43692},{"class":34056,"line":34150},[43693],{"type":150,"tag":181,"props":43694,"children":43695},{},[43696],{"type":155,"value":43674},{"type":150,"tag":181,"props":43698,"children":43699},{"class":34056,"line":34158},[43700],{"type":150,"tag":181,"props":43701,"children":43702},{},[43703],{"type":155,"value":43704},"MLA              压 D 轴：连续压缩成 latent\n",{"type":150,"tag":181,"props":43706,"children":43707},{"class":34056,"line":39937},[43708],{"type":150,"tag":181,"props":43709,"children":43710},{},[43711],{"type":155,"value":43666},{"type":150,"tag":181,"props":43713,"children":43714},{"class":34056,"line":39945},[43715],{"type":150,"tag":181,"props":43716,"children":43717},{},[43718],{"type":155,"value":43674},{"type":150,"tag":181,"props":43720,"children":43721},{"class":34056,"line":40007},[43722],{"type":150,"tag":181,"props":43723,"children":43724},{},[43725],{"type":155,"value":43726},"CSA \u002F HCA        压 N 轴：压成 block，再挑着看\n",{"type":150,"tag":181,"props":43728,"children":43729},{"class":34056,"line":40133},[43730],{"type":150,"tag":181,"props":43731,"children":43732},{},[43733],{"type":155,"value":43666},{"type":150,"tag":181,"props":43735,"children":43736},{"class":34056,"line":40190},[43737],{"type":150,"tag":181,"props":43738,"children":43739},{},[43740],{"type":155,"value":43674},{"type":150,"tag":181,"props":43742,"children":43744},{"class":34056,"line":43743},13,[43745],{"type":150,"tag":181,"props":43746,"children":43747},{},[43748],{"type":155,"value":43749},"Linear \u002F SSM     压掉保存本身：只留一个 state\n",{"type":150,"tag":181,"props":43751,"children":43753},{"class":34056,"line":43752},14,[43754],{"type":150,"tag":181,"props":43755,"children":43756},{},[43757],{"type":155,"value":43666},{"type":150,"tag":181,"props":43759,"children":43761},{"class":34056,"line":43760},15,[43762],{"type":150,"tag":181,"props":43763,"children":43764},{},[43765],{"type":155,"value":43674},{"type":150,"tag":181,"props":43767,"children":43769},{"class":34056,"line":43768},16,[43770],{"type":150,"tag":181,"props":43771,"children":43772},{},[43773],{"type":155,"value":43774},"V4.1             压 L 轴和计算阶段：不再每层每步都付同样的钱\n",{"type":150,"tag":151,"props":43776,"children":43777},{},[43778,43780,43785],{"type":155,"value":43779},"同样的路线可以读成一连串追问：每个 head 都需要独立 KV 吗——MQA\u002FGQA 说不用；每个 token 都需要完整 KV 吗——MLA 说不用；每个 query 都需要看所有 token 吗——CSA\u002FHCA 说不用；每个 token 都需要被永久保存吗——Linear\u002FSSM 说不用；每个 layer 都需要独立 memory、prefill 和 decode 必须用同样计算吗——V4.1 说不用。每一代都在放松一个假设，",{"type":150,"tag":158,"props":43781,"children":43782},{},[43783],{"type":155,"value":43784},"保存信息、压缩信息、找回信息",{"type":155,"value":43786},"三个动作在其中循环。",{"type":150,"tag":151,"props":43788,"children":43789},{},[43790],{"type":155,"value":43791},"前面一直把架构、数据、算力当成三样不同的东西比轻重，这个切法可能一开始就偏了。它们互相兑换：",{"type":150,"tag":151,"props":43793,"children":43794},{},[43795,43800],{"type":150,"tag":158,"props":43796,"children":43797},{},[43798],{"type":155,"value":43799},"数据是架构。",{"type":155,"value":43801}," 用什么数据、怎么组织、怎么采样，直接决定模型长成什么样。同一套 attention，换一个数据分布就是另一个模型。数据管线里每个决定，效果上和改架构是同一个量级。",{"type":150,"tag":151,"props":43803,"children":43804},{},[43805,43810],{"type":150,"tag":158,"props":43806,"children":43807},{},[43808],{"type":155,"value":43809},"架构是算力。",{"type":155,"value":43811}," 这条前面已经推过一遍。MLA 把 KV 压到 1\u002F57，ds4.1f 把 decode 的算术强度压到离拐点 300 倍，省下来的都是算力。架构不改模型会做什么，只改做同一件事要花多少钱。它是算力的另一种记账方式。",{"type":150,"tag":151,"props":43813,"children":43814},{},[43815,43820],{"type":150,"tag":158,"props":43816,"children":43817},{},[43818],{"type":155,"value":43819},"算力是数据。",{"type":155,"value":43821}," 省下来的算力不会消失，会被重新花到更多 tokens、更长的 rollout、更多 verifier compute。算力的下游就是数据。",{"type":150,"tag":151,"props":43823,"children":43824},{},[43825],{"type":155,"value":43826},"三个词绕成一个环。能力和架构谁更重要，这个问题问错了对象：三者是同一笔资源的三种形态，换名字不改变总量，变的只是兑换率。成本压到足够低之后，架构这条线上剩下的竞争，只是决定那笔资源能兑换多少次。",{"type":150,"tag":43828,"props":43829,"children":43830},"style",{},[43831],{"type":155,"value":43832},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":28,"searchDepth":34066,"depth":34075,"links":43834},[43835,43836,43837,43838,43839,43843,43844,43847,43848,43849,43850],{"id":171,"depth":34066,"text":174},{"id":6034,"depth":34066,"text":6037},{"id":7636,"depth":34066,"text":7639},{"id":9822,"depth":34066,"text":13},{"id":30431,"depth":34066,"text":30434,"children":43840},[43841,43842],{"id":30438,"depth":34075,"text":30441},{"id":31661,"depth":34075,"text":31664},{"id":32050,"depth":34066,"text":32053},{"id":34018,"depth":34066,"text":34021,"children":43845},[43846],{"id":34035,"depth":34075,"text":34038},{"id":34444,"depth":34066,"text":34447},{"id":43029,"depth":34066,"text":43032},{"id":43273,"depth":34066,"text":43273},{"id":43636,"depth":34066,"text":43636},"markdown","content:posts:llm-memory-compute-tradeoff.md","content","posts\u002Fllm-memory-compute-tradeoff.md","posts\u002Fllm-memory-compute-tradeoff","md",1789855750446]