[{"data":1,"prerenderedAt":2184},["ShallowReactive",2],{"content-\u002Fposts\u002Fspinning-up-rl-intro-2":3,"blog-posts":1933},{"id":4,"title":5,"body":6,"categories":1917,"date":1918,"description":1919,"draft":1920,"extension":1921,"meta":1922,"navigation":1923,"path":1924,"seo":1925,"stem":1926,"tags":1927,"__hash__":1932},"posts\u002Fposts\u002Fspinning-up-rl-intro-2.md","[译] 深度强化学习入门（二）：强化学习算法的分类",{"type":7,"value":8,"toc":1896},"minimark",[9,37,40,44,51,57,60,73,77,80,89,92,104,107,110,127,131,134,412,591,609,818,1044,1274,1277,1354,1358,1438,1442,1451,1469,1472,1475,1479,1488,1491,1501,1504,1605,1608,1624,1627,1630,1639,1657,1660,1663,1666,1676,1679,1686,1715,1720],[10,11,13],"note",{"title":12},"译者说明",[14,15,16,17,24,25,30,31,36],"p",{},"本文是 OpenAI ",[18,19,23],"a",{"href":20,"rel":21},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002F",[22],"nofollow","Spinning Up in Deep RL"," 教程第二部分 ",[18,26,29],{"href":27,"rel":28},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Frl_intro2.html",[22],"Part 2: Kinds of RL Algorithms"," 的全文翻译。原仓库采用 ",[18,32,35],{"href":33,"rel":34},"https:\u002F\u002Fgithub.com\u002Fopenai\u002Fspinningup\u002Fblob\u002Fmaster\u002FLICENSE",[22],"MIT 协议","。分类图保留官方英文原图，正文保留算法缩写与外链；术语首次出现时附英文原词，后文沿用同一译名。",[14,38,39],{},"术语和记号打好基础后，接下来可以看更丰富的内容：现代 RL 算法的整体版图，以及算法设计中各种取舍。",[41,42,43],"h2",{"id":43},"强化学习算法的分类",[14,45,46],{},[47,48],"img",{"alt":49,"src":50},"现代强化学习算法的一种分类图（官方英文原图）","https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002F_images\u002Frl_algorithms_9_15.svg",[14,52,53],{},[54,55,56],"em",{},"图 1：现代 RL 算法的一种实用分类，并未穷尽所有方法。算法引用见文末。",[14,58,59],{},"先说明一点：现代 RL 算法的模块化特征很难用树形结构表示，因此要画出准确且包罗万象的算法分类图相当困难。为了让分类图能放在一页内，并使这篇导论便于阅读，我们省略了不少进阶主题，例如探索、迁移学习和元学习。本文的目标是：",[61,62,63,67,70],"ul",{},[64,65,66],"li",{},"突出深度 RL 算法中最基础的设计选择：要学习什么，以及怎样学习；",[64,68,69],{},"展示这些选择各自的取舍；",[64,71,72],{},"将几种重要的现代算法放进这些选择构成的脉络中。",[74,75,76],"h3",{"id":76},"无模型与基于模型的强化学习",[14,78,79],{},"强化学习算法面临的一个重要分岔是：智能体能否访问环境模型，或者能否学到一个环境模型。这里的环境模型指预测状态转移和奖励的函数。",[14,81,82,83,88],{},"使用模型的主要好处，是智能体可以向前规划：设想若干种选择分别会带来什么结果，再明确比较这些选择。随后，智能体还可以把规划所得提炼成一个学到的策略。一个著名例子是 ",[18,84,87],{"href":85,"rel":86},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1712.01815",[22],"AlphaZero","。模型方法若能奏效，相较于不使用模型的方法，样本效率（sample efficiency）可能会大幅提高。",[14,90,91],{},"主要困难在于，智能体通常拿不到环境的真实模型。此时若想使用模型，就只能从经验中学习它；这会带来一系列挑战。最突出的一点是，智能体可能利用模型中的偏差，在学到的模型里表现良好，到了真实环境中却表现欠佳，甚至极差。模型学习本身就很难；即使投入大量时间和算力，也可能得不到相应收益。",[14,93,94,95,99,100,103],{},"使用环境模型的算法称为",[96,97,98],"strong",{},"基于模型的方法（model-based methods）","，不使用环境模型的算法称为",[96,101,102],{},"无模型方法（model-free methods）","。无模型方法放弃了借助模型提高样本效率的潜在收益，但通常更容易实现和调参。截至这篇导论写作时（2018 年 9 月），无模型方法更受欢迎，也经过了更多开发和测试。",[74,105,106],{"id":106},"要学习什么",[14,108,109],{},"另一个关键分岔是算法要学习什么。常见的学习对象包括：",[61,111,112,115,118,121,124],{},[64,113,114],{},"策略，可以是随机型策略，也可以是确定型策略；",[64,116,117],{},"动作价值函数（Q 函数）；",[64,119,120],{},"价值函数；",[64,122,123],{},"环境模型；",[64,125,126],{},"或以上对象的组合。",[128,129,130],"h4",{"id":130},"无模型强化学习要学习什么",[14,132,133],{},"无模型 RL 中，表征和训练智能体主要有两种方法。",[14,135,136,139,140,285,286,316,317,407,408,411],{},[96,137,138],{},"策略优化（policy optimization）","。这类方法显式地用策略 ",[141,142,145,191],"span",{"className":143},[144],"katex",[141,146,149],{"className":147},[148],"katex-mathml",[150,151,153],"math",{"xmlns":152},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[154,155,156,186],"semantics",{},[157,158,159,169,174,176,180,183],"mrow",{},[160,161,162,166],"msub",{},[163,164,165],"mi",{},"π",[163,167,168],{},"θ",[170,171,173],"mo",{"stretchy":172},"false","(",[163,175,18],{},[163,177,179],{"mathvariant":178},"normal","∣",[163,181,182],{},"s",[170,184,185],{"stretchy":172},")",[187,188,190],"annotation",{"encoding":189},"application\u002Fx-tex","\\pi_{\\theta}(a|s)",[141,192,196],{"className":193,"ariaHidden":195},[194],"katex-html","true",[141,197,200,205,268,272,275,278,281],{"className":198},[199],"base",[141,201],{"className":202,"style":204},[203],"strut","height:1em;vertical-align:-0.25em;",[141,206,209,214],{"className":207},[208],"mord",[141,210,165],{"className":211,"style":213},[208,212],"mathnormal","margin-right:0.0359em;",[141,215,218],{"className":216},[217],"msupsub",[141,219,223,259],{"className":220},[221,222],"vlist-t","vlist-t2",[141,224,227,254],{"className":225},[226],"vlist-r",[141,228,232],{"className":229,"style":231},[230],"vlist","height:0.3361em;",[141,233,235,240],{"style":234},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[141,236],{"className":237,"style":239},[238],"pstrut","height:2.7em;",[141,241,247],{"className":242},[243,244,245,246],"sizing","reset-size6","size3","mtight",[141,248,250],{"className":249},[208,246],[141,251,168],{"className":252,"style":253},[208,212,246],"margin-right:0.0278em;",[141,255,258],{"className":256},[257],"vlist-s","​",[141,260,262],{"className":261},[226],[141,263,266],{"className":264,"style":265},[230],"height:0.15em;",[141,267],{},[141,269,173],{"className":270},[271],"mopen",[141,273,18],{"className":274},[208,212],[141,276,179],{"className":277},[208],[141,279,182],{"className":280},[208,212],[141,282,185],{"className":283},[284],"mclose"," 表示智能体。它们通过两种方式优化参数 ",[141,287,289,303],{"className":288},[144],[141,290,292],{"className":291},[148],[150,293,294],{"xmlns":152},[154,295,296,300],{},[157,297,298],{},[163,299,168],{},[187,301,302],{"encoding":189},"\\theta",[141,304,306],{"className":305,"ariaHidden":195},[194],[141,307,309,313],{"className":308},[199],[141,310],{"className":311,"style":312},[203],"height:0.6944em;",[141,314,168],{"className":315,"style":253},[208,212],"：直接对性能目标 ",[141,318,320,345],{"className":319},[144],[141,321,323],{"className":322},[148],[150,324,325],{"xmlns":152},[154,326,327,342],{},[157,328,329,332,334,340],{},[163,330,331],{},"J",[170,333,173],{"stretchy":172},[160,335,336,338],{},[163,337,165],{},[163,339,168],{},[170,341,185],{"stretchy":172},[187,343,344],{"encoding":189},"J(\\pi_{\\theta})",[141,346,348],{"className":347,"ariaHidden":195},[194],[141,349,351,354,358,361,404],{"className":350},[199],[141,352],{"className":353,"style":204},[203],[141,355,331],{"className":356,"style":357},[208,212],"margin-right:0.0962em;",[141,359,173],{"className":360},[271],[141,362,364,367],{"className":363},[208],[141,365,165],{"className":366,"style":213},[208,212],[141,368,370],{"className":369},[217],[141,371,373,396],{"className":372},[221,222],[141,374,376,393],{"className":375},[226],[141,377,379],{"className":378,"style":231},[230],[141,380,381,384],{"style":234},[141,382],{"className":383,"style":239},[238],[141,385,387],{"className":386},[243,244,245,246],[141,388,390],{"className":389},[208,246],[141,391,168],{"className":392,"style":253},[208,212,246],[141,394,258],{"className":395},[257],[141,397,399],{"className":398},[226],[141,400,402],{"className":401,"style":265},[230],[141,403],{},[141,405,185],{"className":406},[284]," 做梯度上升，或者间接地最大化该目标的局部代理（surrogate）。优化过程几乎总是采用",[96,409,410],{},"同策略（on-policy）"," 方式：每次更新只使用智能体按照当前最新策略与环境交互时收集的数据。",[14,413,414,415,509,510,590],{},"策略优化通常还会学习一个函数 ",[141,416,418,444],{"className":417},[144],[141,419,421],{"className":420},[148],[150,422,423],{"xmlns":152},[154,424,425,441],{},[157,426,427,435,437,439],{},[160,428,429,432],{},[163,430,431],{},"V",[163,433,434],{},"ϕ",[170,436,173],{"stretchy":172},[163,438,182],{},[170,440,185],{"stretchy":172},[187,442,443],{"encoding":189},"V_{\\phi}(s)",[141,445,447],{"className":446,"ariaHidden":195},[194],[141,448,450,454,500,503,506],{"className":449},[199],[141,451],{"className":452,"style":453},[203],"height:1.0361em;vertical-align:-0.2861em;",[141,455,457,461],{"className":456},[208],[141,458,431],{"className":459,"style":460},[208,212],"margin-right:0.2222em;",[141,462,464],{"className":463},[217],[141,465,467,491],{"className":466},[221,222],[141,468,470,488],{"className":469},[226],[141,471,473],{"className":472,"style":231},[230],[141,474,476,479],{"style":475},"top:-2.55em;margin-left:-0.2222em;margin-right:0.05em;",[141,477],{"className":478,"style":239},[238],[141,480,482],{"className":481},[243,244,245,246],[141,483,485],{"className":484},[208,246],[141,486,434],{"className":487},[208,212,246],[141,489,258],{"className":490},[257],[141,492,494],{"className":493},[226],[141,495,498],{"className":496,"style":497},[230],"height:0.2861em;",[141,499],{},[141,501,173],{"className":502},[271],[141,504,182],{"className":505},[208,212],[141,507,185],{"className":508},[284],"，用来近似该策略下的价值函数 ",[141,511,513,538],{"className":512},[144],[141,514,516],{"className":515},[148],[150,517,518],{"xmlns":152},[154,519,520,535],{},[157,521,522,529,531,533],{},[523,524,525,527],"msup",{},[163,526,431],{},[163,528,165],{},[170,530,173],{"stretchy":172},[163,532,182],{},[170,534,185],{"stretchy":172},[187,536,537],{"encoding":189},"V^{\\pi}(s)",[141,539,541],{"className":540,"ariaHidden":195},[194],[141,542,544,547,581,584,587],{"className":543},[199],[141,545],{"className":546,"style":204},[203],[141,548,550,553],{"className":549},[208],[141,551,431],{"className":552,"style":460},[208,212],[141,554,556],{"className":555},[217],[141,557,559],{"className":558},[221],[141,560,562],{"className":561},[226],[141,563,566],{"className":564,"style":565},[230],"height:0.6644em;",[141,567,569,572],{"style":568},"top:-3.063em;margin-right:0.05em;",[141,570],{"className":571,"style":239},[238],[141,573,575],{"className":574},[243,244,245,246],[141,576,578],{"className":577},[208,246],[141,579,165],{"className":580,"style":213},[208,212,246],[141,582,173],{"className":583},[271],[141,585,182],{"className":586},[208,212],[141,588,185],{"className":589},[284],"，并据此计算如何更新策略。策略优化方法的例子包括：",[61,592,593,601],{},[64,594,595,600],{},[18,596,599],{"href":597,"rel":598},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1602.01783",[22],"A2C \u002F A3C","：通过梯度上升直接最大化性能；",[64,602,603,608],{},[18,604,607],{"href":605,"rel":606},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1707.06347",[22],"PPO","：通过最大化一个代理目标函数，间接推动性能目标上升。这个代理目标会保守估计这次策略更新可能引起的性能目标变化。",[14,610,611,614,615,722,723,813,814,817],{},[96,612,613],{},"Q-learning","。这类方法学习一个函数 ",[141,616,618,648],{"className":617},[144],[141,619,621],{"className":620},[148],[150,622,623],{"xmlns":152},[154,624,625,645],{},[157,626,627,634,636,638,641,643],{},[160,628,629,632],{},[163,630,631],{},"Q",[163,633,168],{},[170,635,173],{"stretchy":172},[163,637,182],{},[170,639,640],{"separator":195},",",[163,642,18],{},[170,644,185],{"stretchy":172},[187,646,647],{"encoding":189},"Q_{\\theta}(s,a)",[141,649,651],{"className":650,"ariaHidden":195},[194],[141,652,654,657,701,704,707,711,716,719],{"className":653},[199],[141,655],{"className":656,"style":204},[203],[141,658,660,663],{"className":659},[208],[141,661,631],{"className":662},[208,212],[141,664,666],{"className":665},[217],[141,667,669,693],{"className":668},[221,222],[141,670,672,690],{"className":671},[226],[141,673,675],{"className":674,"style":231},[230],[141,676,678,681],{"style":677},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[141,679],{"className":680,"style":239},[238],[141,682,684],{"className":683},[243,244,245,246],[141,685,687],{"className":686},[208,246],[141,688,168],{"className":689,"style":253},[208,212,246],[141,691,258],{"className":692},[257],[141,694,696],{"className":695},[226],[141,697,699],{"className":698,"style":265},[230],[141,700],{},[141,702,173],{"className":703},[271],[141,705,182],{"className":706},[208,212],[141,708,640],{"className":709},[710],"mpunct",[141,712],{"className":713,"style":715},[714],"mspace","margin-right:0.1667em;",[141,717,18],{"className":718},[208,212],[141,720,185],{"className":721},[284],"，用来近似最优动作价值函数 ",[141,724,726,755],{"className":725},[144],[141,727,729],{"className":728},[148],[150,730,731],{"xmlns":152},[154,732,733,752],{},[157,734,735,742,744,746,748,750],{},[523,736,737,739],{},[163,738,631],{},[170,740,741],{},"∗",[170,743,173],{"stretchy":172},[163,745,182],{},[170,747,640],{"separator":195},[163,749,18],{},[170,751,185],{"stretchy":172},[187,753,754],{"encoding":189},"Q^*(s,a)",[141,756,758],{"className":757,"ariaHidden":195},[194],[141,759,761,764,795,798,801,804,807,810],{"className":760},[199],[141,762],{"className":763,"style":204},[203],[141,765,767,770],{"className":766},[208],[141,768,631],{"className":769},[208,212],[141,771,773],{"className":772},[217],[141,774,776],{"className":775},[221],[141,777,779],{"className":778},[226],[141,780,783],{"className":781,"style":782},[230],"height:0.6887em;",[141,784,785,788],{"style":568},[141,786],{"className":787,"style":239},[238],[141,789,791],{"className":790},[243,244,245,246],[141,792,741],{"className":793},[794,246],"mbin",[141,796,173],{"className":797},[271],[141,799,182],{"className":800},[208,212],[141,802,640],{"className":803},[710],[141,805],{"className":806,"style":715},[714],[141,808,18],{"className":809},[208,212],[141,811,185],{"className":812},[284],"。它们通常使用基于贝尔曼方程的目标函数。优化过程几乎总是采用",[96,815,816],{},"异策略（off-policy）"," 方式：每次更新可以使用训练过程中任意时刻收集的数据，而不受智能体采集这些数据时采用何种探索策略的限制。",[14,819,820,821,907,908,982,983,1043],{},"最优策略会选择使 ",[141,822,824,851],{"className":823},[144],[141,825,827],{"className":826},[148],[150,828,829],{"xmlns":152},[154,830,831,849],{},[157,832,833,839,841,843,845,847],{},[523,834,835,837],{},[163,836,631],{},[170,838,741],{},[170,840,173],{"stretchy":172},[163,842,182],{},[170,844,640],{"separator":195},[163,846,18],{},[170,848,185],{"stretchy":172},[187,850,754],{"encoding":189},[141,852,854],{"className":853,"ariaHidden":195},[194],[141,855,857,860,889,892,895,898,901,904],{"className":856},[199],[141,858],{"className":859,"style":204},[203],[141,861,863,866],{"className":862},[208],[141,864,631],{"className":865},[208,212],[141,867,869],{"className":868},[217],[141,870,872],{"className":871},[221],[141,873,875],{"className":874},[226],[141,876,878],{"className":877,"style":782},[230],[141,879,880,883],{"style":568},[141,881],{"className":882,"style":239},[238],[141,884,886],{"className":885},[243,244,245,246],[141,887,741],{"className":888},[794,246],[141,890,173],{"className":891},[271],[141,893,182],{"className":894},[208,212],[141,896,640],{"className":897},[710],[141,899],{"className":900,"style":715},[714],[141,902,18],{"className":903},[208,212],[141,905,185],{"className":906},[284]," 最大的动作；Q-learning 则用学得的 ",[141,909,911,929],{"className":910},[144],[141,912,914],{"className":913},[148],[150,915,916],{"xmlns":152},[154,917,918,926],{},[157,919,920],{},[160,921,922,924],{},[163,923,631],{},[163,925,168],{},[187,927,928],{"encoding":189},"Q_{\\theta}",[141,930,932],{"className":931,"ariaHidden":195},[194],[141,933,935,939],{"className":934},[199],[141,936],{"className":937,"style":938},[203],"height:0.8778em;vertical-align:-0.1944em;",[141,940,942,945],{"className":941},[208],[141,943,631],{"className":944},[208,212],[141,946,948],{"className":947},[217],[141,949,951,974],{"className":950},[221,222],[141,952,954,971],{"className":953},[226],[141,955,957],{"className":956,"style":231},[230],[141,958,959,962],{"style":677},[141,960],{"className":961,"style":239},[238],[141,963,965],{"className":964},[243,244,245,246],[141,966,968],{"className":967},[208,246],[141,969,168],{"className":970,"style":253},[208,212,246],[141,972,258],{"className":973},[257],[141,975,977],{"className":976},[226],[141,978,980],{"className":979,"style":265},[230],[141,981],{}," 近似 ",[141,984,986,1004],{"className":985},[144],[141,987,989],{"className":988},[148],[150,990,991],{"xmlns":152},[154,992,993,1001],{},[157,994,995],{},[523,996,997,999],{},[163,998,631],{},[170,1000,741],{},[187,1002,1003],{"encoding":189},"Q^*",[141,1005,1007],{"className":1006,"ariaHidden":195},[194],[141,1008,1010,1014],{"className":1009},[199],[141,1011],{"className":1012,"style":1013},[203],"height:0.8831em;vertical-align:-0.1944em;",[141,1015,1017,1020],{"className":1016},[208],[141,1018,631],{"className":1019},[208,212],[141,1021,1023],{"className":1022},[217],[141,1024,1026],{"className":1025},[221],[141,1027,1029],{"className":1028},[226],[141,1030,1032],{"className":1031,"style":782},[230],[141,1033,1034,1037],{"style":568},[141,1035],{"className":1036,"style":239},[238],[141,1038,1040],{"className":1039},[243,244,245,246],[141,1041,741],{"className":1042},[794,246],"，据此选取动作：",[141,1045,1048],{"className":1046},[1047],"katex-display",[141,1049,1051,1109],{"className":1050},[144],[141,1052,1054],{"className":1053},[148],[150,1055,1057],{"xmlns":152,"display":1056},"block",[154,1058,1059,1106],{},[157,1060,1061,1063,1065,1067,1069,1072,1075,1078,1090,1096,1098,1100,1102,1104],{},[163,1062,18],{},[170,1064,173],{"stretchy":172},[163,1066,182],{},[170,1068,185],{"stretchy":172},[170,1070,1071],{},"=",[163,1073,1074],{},"arg",[170,1076,1077],{},"⁡",[1079,1080,1081,1088],"munder",{},[157,1082,1083,1086],{},[163,1084,1085],{},"max",[170,1087,1077],{},[163,1089,18],{},[160,1091,1092,1094],{},[163,1093,631],{},[163,1095,168],{},[170,1097,173],{"stretchy":172},[163,1099,182],{},[170,1101,640],{"separator":195},[163,1103,18],{},[170,1105,185],{"stretchy":172},[187,1107,1108],{"encoding":189},"a(s) = \\arg\\max_a Q_{\\theta}(s,a)",[141,1110,1112,1141],{"className":1111,"ariaHidden":195},[194],[141,1113,1115,1118,1121,1124,1127,1130,1134,1138],{"className":1114},[199],[141,1116],{"className":1117,"style":204},[203],[141,1119,18],{"className":1120},[208,212],[141,1122,173],{"className":1123},[271],[141,1125,182],{"className":1126},[208,212],[141,1128,185],{"className":1129},[284],[141,1131],{"className":1132,"style":1133},[714],"margin-right:0.2778em;",[141,1135,1071],{"className":1136},[1137],"mrel",[141,1139],{"className":1140,"style":1133},[714],[141,1142,1144,1148,1157,1160,1210,1213,1256,1259,1262,1265,1268,1271],{"className":1143},[199],[141,1145],{"className":1146,"style":1147},[203],"height:1.45em;vertical-align:-0.7em;",[141,1149,1152,1153],{"className":1150},[1151],"mop","ar",[141,1154,1156],{"style":1155},"margin-right:0.0139em;","g",[141,1158],{"className":1159,"style":715},[714],[141,1161,1164],{"className":1162},[1151,1163],"op-limits",[141,1165,1167,1201],{"className":1166},[221,222],[141,1168,1170,1198],{"className":1169},[226],[141,1171,1174,1187],{"className":1172,"style":1173},[230],"height:0.4306em;",[141,1175,1177,1181],{"style":1176},"top:-2.4em;margin-left:0em;",[141,1178],{"className":1179,"style":1180},[238],"height:3em;",[141,1182,1184],{"className":1183},[243,244,245,246],[141,1185,18],{"className":1186},[208,212,246],[141,1188,1190,1193],{"style":1189},"top:-3em;",[141,1191],{"className":1192,"style":1180},[238],[141,1194,1195],{},[141,1196,1085],{"className":1197},[1151],[141,1199,258],{"className":1200},[257],[141,1202,1204],{"className":1203},[226],[141,1205,1208],{"className":1206,"style":1207},[230],"height:0.7em;",[141,1209],{},[141,1211],{"className":1212,"style":715},[714],[141,1214,1216,1219],{"className":1215},[208],[141,1217,631],{"className":1218},[208,212],[141,1220,1222],{"className":1221},[217],[141,1223,1225,1248],{"className":1224},[221,222],[141,1226,1228,1245],{"className":1227},[226],[141,1229,1231],{"className":1230,"style":231},[230],[141,1232,1233,1236],{"style":677},[141,1234],{"className":1235,"style":239},[238],[141,1237,1239],{"className":1238},[243,244,245,246],[141,1240,1242],{"className":1241},[208,246],[141,1243,168],{"className":1244,"style":253},[208,212,246],[141,1246,258],{"className":1247},[257],[141,1249,1251],{"className":1250},[226],[141,1252,1254],{"className":1253,"style":265},[230],[141,1255],{},[141,1257,173],{"className":1258},[271],[141,1260,182],{"className":1261},[208,212],[141,1263,640],{"className":1264},[710],[141,1266],{"className":1267,"style":715},[714],[141,1269,18],{"className":1270},[208,212],[141,1272,185],{"className":1273},[284],[14,1275,1276],{},"Q-learning 方法的例子包括：",[61,1278,1279,1287],{},[64,1280,1281,1286],{},[18,1282,1285],{"href":1283,"rel":1284},"https:\u002F\u002Fwww.cs.toronto.edu\u002F~vmnih\u002Fdocs\u002Fdqn.pdf",[22],"DQN","：推动深度 RL 领域发展的经典方法；",[64,1288,1289,1294,1295,1353],{},[18,1290,1293],{"href":1291,"rel":1292},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1707.06887",[22],"C51","：一种学习回报分布的变体，其期望就是 ",[141,1296,1298,1315],{"className":1297},[144],[141,1299,1301],{"className":1300},[148],[150,1302,1303],{"xmlns":152},[154,1304,1305,1313],{},[157,1306,1307],{},[523,1308,1309,1311],{},[163,1310,631],{},[170,1312,741],{},[187,1314,1003],{"encoding":189},[141,1316,1318],{"className":1317,"ariaHidden":195},[194],[141,1319,1321,1324],{"className":1320},[199],[141,1322],{"className":1323,"style":1013},[203],[141,1325,1327,1330],{"className":1326},[208],[141,1328,631],{"className":1329},[208,212],[141,1331,1333],{"className":1332},[217],[141,1334,1336],{"className":1335},[221],[141,1337,1339],{"className":1338},[226],[141,1340,1342],{"className":1341,"style":782},[230],[141,1343,1344,1347],{"style":568},[141,1345],{"className":1346,"style":239},[238],[141,1348,1350],{"className":1349},[243,244,245,246],[141,1351,741],{"className":1352},[794,246],"。",[128,1355,1357],{"id":1356},"策略优化与-q-learning-的取舍","策略优化与 Q-learning 的取舍",[14,1359,1360,1361,1433,1434,1437],{},"策略优化方法的主要优点是目标明确：它直接优化我们真正关心的对象，因此往往稳定、可靠。相比之下，Q-learning 通过训练 ",[141,1362,1364,1381],{"className":1363},[144],[141,1365,1367],{"className":1366},[148],[150,1368,1369],{"xmlns":152},[154,1370,1371,1379],{},[157,1372,1373],{},[160,1374,1375,1377],{},[163,1376,631],{},[163,1378,168],{},[187,1380,928],{"encoding":189},[141,1382,1384],{"className":1383,"ariaHidden":195},[194],[141,1385,1387,1390],{"className":1386},[199],[141,1388],{"className":1389,"style":938},[203],[141,1391,1393,1396],{"className":1392},[208],[141,1394,631],{"className":1395},[208,212],[141,1397,1399],{"className":1398},[217],[141,1400,1402,1425],{"className":1401},[221,222],[141,1403,1405,1422],{"className":1404},[226],[141,1406,1408],{"className":1407,"style":231},[230],[141,1409,1410,1413],{"style":677},[141,1411],{"className":1412,"style":239},[238],[141,1414,1416],{"className":1415},[243,244,245,246],[141,1417,1419],{"className":1418},[208,246],[141,1420,168],{"className":1421,"style":253},[208,212,246],[141,1423,258],{"className":1424},[257],[141,1426,1428],{"className":1427},[226],[141,1429,1431],{"className":1430,"style":265},[230],[141,1432],{}," 满足自洽方程，间接推动智能体表现变好。这类学习存在多种失效模式，因此通常稳定性较差（相关讨论见文末参考资料 ",[141,1435,1436],{},"1","）。不过，在能够奏效时，Q-learning 的样本效率明显更高，因为它比策略优化更充分地复用数据。",[128,1439,1441],{"id":1440},"介于策略优化与-q-learning-之间的方法","介于策略优化与 Q-learning 之间的方法",[14,1443,1444,1445,1450],{},"策略优化与 Q-learning 并不互斥，在",[18,1446,1449],{"href":1447,"rel":1448},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1704.06440",[22],"某些条件下甚至可以等价","。处于两者之间的一系列算法，可以在两端方法的优缺点之间作出折中。例如：",[61,1452,1453,1461],{},[64,1454,1455,1460],{},[18,1456,1459],{"href":1457,"rel":1458},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1509.02971",[22],"DDPG"," 同时学习一个确定型策略和一个 Q 函数，并利用两者相互改进；",[64,1462,1463,1468],{},[18,1464,1467],{"href":1465,"rel":1466},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1801.01290",[22],"SAC"," 使用随机策略、熵正则化和其他技巧来稳定学习，在标准基准上的得分高于 DDPG。",[74,1470,1471],{"id":1471},"基于模型的强化学习要学习什么",[14,1473,1474],{},"与无模型 RL 不同，基于模型的方法很难归纳成少数几类边界清晰的算法；模型的使用方式有许多彼此独立的维度。下面只举几个例子，远未穷尽。每种方法使用的模型都可能是直接给定的，也可能是从经验中学到的。",[128,1476,1478],{"id":1477},"背景纯规划","背景：纯规划",[14,1480,1481,1482,1487],{},"最基础的做法是不显式表示策略，而是使用 ",[18,1483,1486],{"href":1484,"rel":1485},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FModel_predictive_control",[22],"模型预测控制（model-predictive control，MPC）"," 这样的纯规划方法来选择动作。MPC 中，智能体每次观测环境时，都会根据模型计算一个最优计划；计划规定从当前时刻起，在一个固定时间窗口内要采取哪些动作。",[14,1489,1490],{},"规划算法也可以借助学到的价值函数，考虑规划视界之外的未来奖励。随后，智能体只执行计划中的第一个动作，并立即丢弃计划的其余部分。每次准备与环境交互时，智能体都会重新计算计划，以免继续执行旧计划时规划视界已短于预期。",[61,1492,1493],{},[64,1494,1495,1500],{},[18,1496,1499],{"href":1497,"rel":1498},"https:\u002F\u002Fsites.google.com\u002Fview\u002Fmbmf",[22],"MBMF"," 探索了在深度 RL 的标准基准任务中，使用学到的环境模型进行 MPC 的做法。",[128,1502,1503],{"id":1503},"专家迭代",[14,1505,1506,1507,1604],{},"在纯规划的基础上，一个直接的扩展是显式表示并学习策略 ",[141,1508,1510,1537],{"className":1509},[144],[141,1511,1513],{"className":1512},[148],[150,1514,1515],{"xmlns":152},[154,1516,1517,1535],{},[157,1518,1519,1525,1527,1529,1531,1533],{},[160,1520,1521,1523],{},[163,1522,165],{},[163,1524,168],{},[170,1526,173],{"stretchy":172},[163,1528,18],{},[163,1530,179],{"mathvariant":178},[163,1532,182],{},[170,1534,185],{"stretchy":172},[187,1536,190],{"encoding":189},[141,1538,1540],{"className":1539,"ariaHidden":195},[194],[141,1541,1543,1546,1589,1592,1595,1598,1601],{"className":1542},[199],[141,1544],{"className":1545,"style":204},[203],[141,1547,1549,1552],{"className":1548},[208],[141,1550,165],{"className":1551,"style":213},[208,212],[141,1553,1555],{"className":1554},[217],[141,1556,1558,1581],{"className":1557},[221,222],[141,1559,1561,1578],{"className":1560},[226],[141,1562,1564],{"className":1563,"style":231},[230],[141,1565,1566,1569],{"style":234},[141,1567],{"className":1568,"style":239},[238],[141,1570,1572],{"className":1571},[243,244,245,246],[141,1573,1575],{"className":1574},[208,246],[141,1576,168],{"className":1577,"style":253},[208,212,246],[141,1579,258],{"className":1580},[257],[141,1582,1584],{"className":1583},[226],[141,1585,1587],{"className":1586,"style":265},[230],[141,1588],{},[141,1590,173],{"className":1591},[271],[141,1593,18],{"className":1594},[208,212],[141,1596,179],{"className":1597},[208],[141,1599,182],{"className":1600},[208,212],[141,1602,185],{"className":1603},[284],"。智能体在模型中使用规划算法（例如蒙特卡洛树搜索）来规划，并从当前策略中采样候选动作。规划算法给出的动作比策略单独给出的动作更好，因此，规划算法可以充当相对于当前策略的“专家”。",[14,1606,1607],{},"之后，智能体更新策略，使它更倾向于给出与规划算法相似的动作。",[61,1609,1610,1618],{},[64,1611,1612,1617],{},[18,1613,1616],{"href":1614,"rel":1615},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1705.08439",[22],"ExIt"," 使用这种方法训练深度神经网络来玩 Hex。",[64,1619,1620,1623],{},[18,1621,87],{"href":85,"rel":1622},[22]," 也是这种方法的例子。",[128,1625,1626],{"id":1626},"用模型生成的经验增强无模型方法",[14,1628,1629],{},"使用无模型 RL 算法训练策略或 Q 函数，同时在更新智能体时采取以下做法之一：",[1631,1632,1633,1636],"ol",{},[64,1634,1635],{},"用模型生成的虚拟经验补充真实经验；",[64,1637,1638],{},"只使用模型生成的虚拟经验来更新智能体。",[61,1640,1641,1649],{},[64,1642,1643,1648],{},[18,1644,1647],{"href":1645,"rel":1646},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1803.00101",[22],"MBVE"," 展示了如何用虚拟经验补充真实经验。",[64,1650,1651,1656],{},[18,1652,1655],{"href":1653,"rel":1654},"https:\u002F\u002Fworldmodels.github.io\u002F",[22],"World Models"," 展示了如何只用虚拟经验训练智能体，并把这种方式称作“在梦境中训练”（training in the dream）。",[128,1658,1659],{"id":1659},"在策略中嵌入规划过程",[14,1661,1662],{},"另一种方法是把规划过程直接嵌入策略，作为策略调用的一个子程序；完整计划作为辅助信息提供给策略。再用任意标准的无模型算法训练策略。这里的关键是，策略可以学会选择在什么时候、以什么方式使用规划结果。",[14,1664,1665],{},"这样可以减轻模型偏差的影响：如果模型在某些状态下给出的规划不可靠，策略可以学会忽略它。",[61,1667,1668],{},[64,1669,1670,1675],{},[18,1671,1674],{"href":1672,"rel":1673},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1707.06203",[22],"I2A"," 是让智能体具备这类“想象”能力的一种方法。",[41,1677,1678],{"id":1678},"参考资料",[14,1680,1681],{},[96,1682,1683,1685],{},[141,1684,1436],{}," Q-learning 方法为何会失效",[61,1687,1688,1697,1706],{},[64,1689,1690,1691,1696],{},"Tsitsiklis 和 Van Roy 的",[18,1692,1695],{"href":1693,"rel":1694},"https:\u002F\u002Fweb.mit.edu\u002Fjnt\u002Fwww\u002FPapers\u002FJ063-97-bvr-td.pdf",[22],"经典论文","；",[64,1698,1699,1700,1705],{},"Szepesvári 的",[18,1701,1704],{"href":1702,"rel":1703},"https:\u002F\u002Fsites.ualberta.ca\u002F~szepesva\u002Fpapers\u002FRLAlgsInMDPs.pdf",[22],"综述","，第 4.3.2 节；",[64,1707,1708,1709,1714],{},"Sutton 与 Barto 的",[18,1710,1713],{"href":1711,"rel":1712},"http:\u002F\u002Fincompleteideas.net\u002Fbook\u002Fthe-book-2nd.html",[22],"《强化学习：导论》","第 11 章，尤其是第 11.3 节关于“致命三元组”（deadly triad）的讨论：函数逼近、自举更新和异策略数据共同导致价值学习算法不稳定。",[14,1716,1717],{},[96,1718,1719],{},"分类图中的算法引用",[61,1721,1722,1734,1745,1758,1769,1782,1793,1804,1815,1828,1841,1852,1863,1874,1885],{},[64,1723,1724,1733],{},[96,1725,1726,1729,1730],{},[141,1727,1728],{},"2"," ",[18,1731,599],{"href":597,"rel":1732},[22],"（Asynchronous Advantage Actor-Critic）：Mnih 等，2016。",[64,1735,1736,1744],{},[96,1737,1738,1729,1741],{},[141,1739,1740],{},"3",[18,1742,607],{"href":605,"rel":1743},[22],"（Proximal Policy Optimization）：Schulman 等，2017。",[64,1746,1747,1757],{},[96,1748,1749,1729,1752],{},[141,1750,1751],{},"4",[18,1753,1756],{"href":1754,"rel":1755},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1502.05477",[22],"TRPO","（Trust Region Policy Optimization）：Schulman 等，2015。",[64,1759,1760,1768],{},[96,1761,1762,1729,1765],{},[141,1763,1764],{},"5",[18,1766,1459],{"href":1457,"rel":1767},[22],"（Deep Deterministic Policy Gradient）：Lillicrap 等，2015。",[64,1770,1771,1781],{},[96,1772,1773,1729,1776],{},[141,1774,1775],{},"6",[18,1777,1780],{"href":1778,"rel":1779},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1802.09477",[22],"TD3","（Twin Delayed DDPG）：Fujimoto 等，2018。",[64,1783,1784,1792],{},[96,1785,1786,1729,1789],{},[141,1787,1788],{},"7",[18,1790,1467],{"href":1465,"rel":1791},[22],"（Soft Actor-Critic）：Haarnoja 等，2018。",[64,1794,1795,1803],{},[96,1796,1797,1729,1800],{},[141,1798,1799],{},"8",[18,1801,1285],{"href":1283,"rel":1802},[22],"（Deep Q-Networks）：Mnih 等，2013。",[64,1805,1806,1814],{},[96,1807,1808,1729,1811],{},[141,1809,1810],{},"9",[18,1812,1293],{"href":1291,"rel":1813},[22],"（Categorical 51-Atom DQN）：Bellemare 等，2017。",[64,1816,1817,1827],{},[96,1818,1819,1729,1822],{},[141,1820,1821],{},"10",[18,1823,1826],{"href":1824,"rel":1825},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1710.10044",[22],"QR-DQN","（Quantile Regression DQN）：Dabney 等，2017。",[64,1829,1830,1840],{},[96,1831,1832,1729,1835],{},[141,1833,1834],{},"11",[18,1836,1839],{"href":1837,"rel":1838},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1707.01495",[22],"HER","（Hindsight Experience Replay）：Andrychowicz 等，2017。",[64,1842,1843,1851],{},[96,1844,1845,1729,1848],{},[141,1846,1847],{},"12",[18,1849,1655],{"href":1653,"rel":1850},[22],"：Ha 和 Schmidhuber，2018。",[64,1853,1854,1862],{},[96,1855,1856,1729,1859],{},[141,1857,1858],{},"13",[18,1860,1674],{"href":1672,"rel":1861},[22],"（Imagination-Augmented Agents）：Weber 等，2017。",[64,1864,1865,1873],{},[96,1866,1867,1729,1870],{},[141,1868,1869],{},"14",[18,1871,1499],{"href":1497,"rel":1872},[22],"（Model-Based RL with Model-Free Fine-Tuning）：Nagabandi 等，2017。",[64,1875,1876,1884],{},[96,1877,1878,1729,1881],{},[141,1879,1880],{},"15",[18,1882,1647],{"href":1645,"rel":1883},[22],"（Model-Based Value Expansion）：Feinberg 等，2018。",[64,1886,1887,1895],{},[96,1888,1889,1729,1892],{},[141,1890,1891],{},"16",[18,1893,87],{"href":85,"rel":1894},[22],"：Silver 等，2017。",{"title":1897,"searchDepth":1898,"depth":1899,"links":1900},"",2,3,[1901,1916],{"id":43,"depth":1898,"text":43,"children":1902},[1903,1904,1910],{"id":76,"depth":1899,"text":76},{"id":106,"depth":1899,"text":106,"children":1905},[1906,1908,1909],{"id":130,"depth":1907,"text":130},4,{"id":1356,"depth":1907,"text":1357},{"id":1440,"depth":1907,"text":1441},{"id":1471,"depth":1899,"text":1471,"children":1911},[1912,1913,1914,1915],{"id":1477,"depth":1907,"text":1478},{"id":1503,"depth":1907,"text":1503},{"id":1626,"depth":1907,"text":1626},{"id":1659,"depth":1907,"text":1659},{"id":1678,"depth":1898,"text":1678},"技术","2026-09-28 10:20:21","OpenAI Spinning Up 第二部分的全文翻译：无模型与基于模型的 RL、策略优化与 Q-learning 的取舍，以及规划、专家迭代和经验增强等方法。",false,"md",{},true,"\u002Fposts\u002Fspinning-up-rl-intro-2",{"title":5,"description":1919},"posts\u002Fspinning-up-rl-intro-2",[1928,1929,1930,1931],"强化学习","RL","Spinning Up","翻译","USpdmPhp2_ws_i3xJLsbQaECC0pEcKXnNx7FN8osFJg",[1934,1944,1949,1953,1961,1967,1971,1981,1987,1989,1995,2004,2008,2012,2016,2026,2033,2039,2043,2048,2056,2063,2071,2076,2082,2088,2093,2098,2105,2112,2119,2125,2132,2140,2146,2152,2158,2164,2169,2177],{"path":1935,"title":1936,"description":1937,"date":1938,"categories":1939,"tags":1940,"draft":1920},"\u002Fposts\u002Fsurvival-welcome-to-university","生存手册——欢迎来到大学","大学同时生产三样东西：可验证技能、认证信号和驯化资本。你付了四年的钱，真正拿到手的是哪一样，取决于你毕业时要进的那个市场在为什么付费。","2026-10-12 20:00:00","生活杂谈",[1941,1942,1943],"大学","求职","方法论",{"path":1945,"title":1946,"description":1897,"date":1947,"categories":1897,"tags":1948,"draft":1920},"\u002Fposts\u002Fgp-tab-测试","gp-tab-测试","2026-10-12 10:51:22",null,{"path":1950,"title":1951,"description":1897,"date":1952,"categories":1897,"tags":1948,"draft":1923},"\u002Fposts\u002Fnetwork-clean","怎么让网络变得干净","2026-10-11 19:40:02",{"path":1954,"title":1955,"description":1956,"date":1957,"categories":1939,"tags":1958,"draft":1920},"\u002Fposts\u002Fthe-age-of-hesitation","[搬运] 不敢投入的时代","为什么我们现在越来越不敢投入了。很多时候我们以为的选择，比如你有两个工作offer，现在选择好像是我到底应该考公还是考编。明明这两个选项都不是真正的选项，因为你还没有开始，也不知道最后能不能成。这个时候我们就已经开始犹豫了。","2026-10-10 11:44:12",[1931,1959,1960],"搬运","心理学",{"path":1962,"title":1963,"description":1964,"date":1965,"categories":1939,"tags":1966,"draft":1920},"\u002Fposts\u002Fwhy-happiness-look-like-same","[搬运] 为什么大家的幸福都长一个样？","为什么大家的幸福都长一个样？一种认知被广泛传播后似乎变得不断扁平，这种感觉很奇怪。","2026-10-09 13:57:15",[1931,1959,1960],{"path":1968,"title":1969,"description":1897,"date":1970,"categories":1939,"tags":1948,"draft":1920},"\u002Fposts\u002Feverything-is-paid-twice","任何事情都要被消费两次","2026-10-06 12:30:06",{"path":1972,"title":1973,"description":1974,"date":1975,"categories":1976,"tags":1977,"draft":1920},"\u002Fposts\u002Fwhy-llm-seems-worse-after-launch","为什么模型上线后，体感像是“降智”了？","权重与 KV cache 的量化都能降低 serving 成本，也可能改变输出；但体感变弱不足以证明服务后来切换了精度。","2026-09-29 21:44:08","技术杂谈",[1978,1979,1980],"LLMs","Serving","量化",{"path":1982,"title":1983,"description":1984,"date":1985,"categories":1917,"tags":1986,"draft":1920},"\u002Fposts\u002Fspinning-up-policy-optimization","[译] 深度强化学习入门（三）：策略优化入门","OpenAI Spinning Up 教程第三部分的全文翻译：推导最简单的策略梯度、log-derivative trick、EGLP 引理、reward-to-go 与基线，配 PyTorch 示例代码。","2026-09-28 17:49:25",[1928,1929,1930,1931],{"path":1924,"title":5,"description":1919,"date":1918,"categories":1917,"tags":1988,"draft":1920},[1928,1929,1930,1931],{"path":1990,"title":1991,"description":1992,"date":1993,"categories":1917,"tags":1994,"draft":1920},"\u002Fposts\u002Fspinning-up-rl-intro","[译] 深度强化学习入门（一）：RL 的关键概念","OpenAI Spinning Up 教程第一部分的全文翻译：智能体与环境、状态与观测、动作空间、策略、轨迹、奖励与回报、价值函数、贝尔曼方程、优势函数，附 MDP 形式化。","2026-09-28 09:29:40",[1928,1929,1930,1931],{"path":1996,"title":1997,"description":1998,"date":1999,"categories":1939,"tags":2000,"draft":1923},"\u002Fposts\u002Fwhen-machines-come-alive","冰箱之后，生活还会怎样改变？","从厨房里的果蝇想到冰箱、移动互联网与 AI 的多轮调用：AGI 已经到来，只是昂贵的智能会怎样改变生活？","2026-09-25 06:36:22",[2001,2002,2003],"AI","随笔","科技",{"path":2005,"title":2006,"description":1897,"date":2007,"categories":1897,"tags":1948,"draft":1923},"\u002Fposts\u002Ftest","test","2026-09-20 06:37:44",{"path":2009,"title":2010,"description":1897,"date":2011,"categories":1897,"tags":1948,"draft":1923},"\u002Fposts\u002Fmathematical-foundations-of-llms-1","LLM的数学基础（1）：","2026-09-19 22:59:55",{"path":2013,"title":2014,"description":1897,"date":2015,"categories":1897,"tags":1948,"draft":1923},"\u002Fposts\u002Ftowards-maximize-serving-capatity","如何在 offline bench 里最大化 LLM serving system 的吞吐","2026-09-19 05:45:05",{"path":2017,"title":2018,"description":2019,"date":2020,"categories":1976,"tags":2021,"draft":1920},"\u002Fposts\u002Fllm-memory-compute-tradeoff","LLM 的 Memory-Compute Trade-off：从 MHA 到 MLA、Sparse 与 Linear Attention","ds4.1f 上线了。它的架构 novel 在哪，便宜和快又是从哪省出来的。","2026-09-11 12:00:00",[1978,2022,2023,2024,2025],"Attention","MLA","DeepSeek","长上下文",{"path":2027,"title":2028,"description":2029,"date":2030,"categories":1976,"tags":2031,"draft":1923},"\u002Fposts\u002Fmarkdown-extensions","Markdown 扩展语法速查","本站 Markdown 支持的扩展语法演示与速查：admonition 提示块、GitHub 仓库卡片、spoiler 遮罩、ABC 乐谱、GP 乐谱、音频播放器、浏览器可运行代码块。","2026-09-01 15:30:00",[2032],"博客",{"path":2034,"title":2035,"description":2036,"date":2037,"categories":1939,"tags":2038,"draft":1920},"\u002Fposts\u002Fwhy-we-still-need-to-write-a-blog","为什么还需要手写博客？","AI这么强，还要古法写博客吗？","2026-09-01 13:17:53",[2032],{"path":2040,"title":2041,"description":1897,"date":2042,"categories":1897,"tags":1948,"draft":1923},"\u002Fposts\u002Fllm-serving-1","LLM Serving（一）:以在H200上部署GLM-5.2为例","2026-07-21 06:15:44",{"path":2044,"title":2045,"description":1897,"date":2046,"categories":2047,"tags":1948,"draft":1920},"\u002Fposts\u002Finteresting-music-rhythm","有趣的音乐节奏","2026-07-16 12:47:24","音乐",{"path":2049,"title":2050,"description":2051,"date":2052,"categories":1917,"tags":2053,"draft":1920},"\u002Fposts\u002Ftowards-monosemanticity","LLM 可解释性（一）：SAE与单语义","读 Anthropic《Towards Monosemanticity》。从难以解释的神经元出发, 讲清superposition为什么让单个神经元混成一团, SAE怎么把多语义的激活分解成单语义特征。","2026-07-12 22:00:00",[1978,2054,2055],"可解释性","SAE",{"path":2057,"title":2058,"description":2059,"date":2060,"categories":1939,"tags":2061,"draft":1923},"\u002Fposts\u002Fpiketty-ai-china-k-shaped","用皮凯蒂的镜头看 AI 与中国 K 型社会","一次凌晨的对话整理。从 r>g 出发，看 AI 时代的资本分配、中国的世袭化趋势、体制内的二元劳动力市场，以及一个普通 00 后在一线打拼的真实牌面。","2026-06-24 03:00:00",[2002,2062,2001],"经济",{"path":2064,"title":2065,"description":1897,"date":2066,"categories":1897,"tags":2067,"draft":1923},"\u002Fposts\u002Fclaude-code-compact-behavior","cluade code compact 行为","2026-06-23 20:17:42",[2068,2069,2070],"Agent","AI Agent","Claude Code",{"path":2072,"title":2073,"description":1897,"date":2074,"categories":1917,"tags":2075,"draft":1920},"\u002Fposts\u002Fclaude-code-settings","Claude Code 配置分享","2026-06-23 14:22:12",[1978,2069,2068,2070],{"path":2077,"title":2078,"description":2079,"date":2080,"categories":1976,"tags":2081,"draft":1920},"\u002Fposts\u002Fmy-thoughts-on-the-ai-industry","简单聊聊对 AI Agent 行业的理解","写这篇文章的初衷，是我发觉逐渐忘了前两个月的 AI\u002FLLMs 是什么样的能力和情况了。意识中似乎某些在突然加速，但理性和直觉告诉我这不对，作为行内人，我知道这是一个渐进的、累积的过程。我想自己回忆一下，Agent的发展。","2026-06-20 13:14:25",[1978,2069,2068],{"path":2083,"title":2084,"description":1897,"date":2085,"categories":1917,"tags":2086,"draft":1920},"\u002Fposts\u002Frotational-positional-encoding","RoPE","2026-06-17 18:31:40",[1978,2087,2084],"Transformer",{"path":2089,"title":2090,"description":1897,"date":2091,"categories":1917,"tags":2092,"draft":1920},"\u002Fposts\u002Fintroduction-to-large-language-model","大语言模型入门基础","2026-06-16 22:36:11",[2087,1978],{"path":2094,"title":2095,"description":1897,"date":2096,"categories":1976,"tags":2097,"draft":1920},"\u002Fposts\u002Fthought-about-coding-agent","关于 Coding Agent 使用的遐思","2026-06-03 17:54:20",[2068],{"path":2099,"title":2100,"description":2101,"date":2102,"categories":1917,"tags":2103,"draft":1920},"\u002Fposts\u002Fabout-ssh-usage","关于SSH的使用——客户端篇","介绍SSH客户端的用法。","2025-06-17 02:44:57",[2104],"工具",{"path":2106,"title":2107,"description":2108,"date":2109,"categories":1976,"tags":2110,"draft":1920},"\u002Fposts\u002Fopensourcesoftwarestarttutorial","如何开始一个开源项目？","浅谈我对开源项目的认知。","2025-04-17 18:00:00",[2111],"开源",{"path":2113,"title":2114,"description":2115,"date":2116,"categories":2117,"tags":2118,"draft":1920},"\u002Fposts\u002Fmotivation","渴求动力","有意识的生活并主动地行动会使得生活更美好。","2025-03-05 00:44:57","生活-情感",[2002],{"path":2120,"title":2121,"description":2122,"date":2123,"categories":2117,"tags":2124,"draft":1920},"\u002Fposts\u002F2024summary","2024年终总结","啊，痛彻的时光；啊，悲怆的人！","2025-01-01 01:00:00",[2002],{"path":2126,"title":2127,"description":2128,"date":2129,"categories":1917,"tags":2130,"draft":1920},"\u002Fposts\u002Fvps-crash-startup","快速搭建个人VPS","如何快速搭建个人VPS。","2024-12-31 19:00:00",[2131],"VPS",{"path":2133,"title":2134,"description":2135,"date":2136,"categories":1917,"tags":2137,"draft":1920},"\u002Fposts\u002Fdata-label-software","AI数据（自动）标注软件概述","对市面上目前的AI数据（自动）标注软件进行调研。","2024-12-20 14:00:00",[2138,2139],"数据标注","深度学习",{"path":2141,"title":2142,"description":2143,"date":2144,"categories":2117,"tags":2145,"draft":1920},"\u002Fposts\u002Fself-destruction","生活的崩坏与自我毁灭","是什么在牵引着我？我的行为，我的思想？我的感受？我所处的地方？它们在何处？","2024-12-19 12:00:00",[2002],{"path":2147,"title":2148,"description":2149,"date":2150,"categories":1917,"tags":2151,"draft":1920},"\u002Fposts\u002Fcs194-196_llmagents","LLM Agents 课程笔记","UCB CS294\u002F194-196 Large Language Model Agents 的课程笔记。","2024-11-30 20:00:00",[2068],{"path":2153,"title":2154,"description":2155,"date":2156,"categories":2117,"tags":2157,"draft":1920},"\u002Fposts\u002Fwhatsyourworth","价值、位置","你们的生存究竟有何价值？如果毫无价值，你们究竟为何存在？","2024-11-29 02:00:00",[2002],{"path":2159,"title":2160,"description":2161,"date":2162,"categories":2117,"tags":2163,"draft":1920},"\u002Fposts\u002Fblogrelifever2","重启博客v2","忙里偷闲学习前端，还是决定结合想法手搓一个自己的博客。","2024-11-25 00:00:00",[2032],{"path":2165,"title":2166,"description":2167,"date":2168,"categories":2117,"tags":1948,"draft":1920},"\u002Fposts\u002Fblogrelife","重启博客","在我的表达欲与现实所需(问题解决记录)驱动下，我决定重写博客。","2024-10-21 20:44:57",{"path":2170,"title":2171,"description":2172,"date":2173,"categories":2117,"tags":2174,"draft":1920},"\u002Fposts\u002Frevival","240205 复活","好吧，断更了许久，我又终于回来了，我又终于回来了。","2024-02-05 20:00:00",[2002,2175,2176],"生活","返乡",{"path":2178,"title":2179,"description":2180,"date":2181,"categories":2117,"tags":2182,"draft":1920},"\u002Fposts\u002Fclouds","「230717」云","我已经很久忘记了不我已经很少把自己的想法在lofter上发布的。","2023-07-17 20:00:00",[2002,2183],"散文",1791871062125]