[{"data":1,"prerenderedAt":21479},["ShallowReactive",2],{"content-\u002Fposts\u002Fspinning-up-policy-optimization":3,"blog-posts":21230},{"id":4,"title":5,"body":6,"categories":21215,"date":21216,"description":21217,"draft":21218,"extension":21219,"meta":21220,"navigation":8548,"path":21221,"seo":21222,"stem":21223,"tags":21224,"__hash__":21229},"posts\u002Fposts\u002Fspinning-up-policy-optimization.md","[译] 深度强化学习入门（三）：策略优化入门",{"type":7,"value":8,"toc":21205},"minimark",[9,47,55,83,92,96,549,552,968,1116,1119,1122,1125,1458,2073,2196,2443,2449,3151,3564,3570,5016,5019,7335,7607,8193,8295,8461,8464,8478,8490,8495,8626,8658,8675,9123,9128,9174,9177,9336,9403,9408,9978,9986,9991,9995,10001,10111,10427,12008,12014,12017,12020,12641,12692,12707,12710,13725,13732,13735,14266,14273,14281,14289,14293,14303,14310,14422,14425,14448,14451,14473,14481,14484,14517,15025,15028,16128,16163,16394,16594,17799,17802,17805,18453,18526,18636,18639,19132,19134,19705,19778,19784,20069,20077,20538,20820,21102,21107,21191,21194,21201],[10,11,13],"note",{"title":12},"译者说明",[14,15,16,17,24,25,30,31,36,37,41,42,46],"p",{},"本文是 OpenAI ",[18,19,23],"a",{"href":20,"rel":21},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002F",[22],"nofollow","Spinning Up in Deep RL"," 教程第三部分 ",[18,26,29],{"href":27,"rel":28},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Frl_intro3.html",[22],"Part 3: Intro to Policy Optimization"," 的全文翻译，原仓库采用 ",[18,32,35],{"href":33,"rel":34},"https:\u002F\u002Fgithub.com\u002Fopenai\u002Fspinningup\u002Fblob\u002Fmaster\u002FLICENSE",[22],"MIT 协议","。术语约定与",[18,38,40],{"href":39},"\u002Fposts\u002Fspinning-up-rl-intro","第一篇","、",[18,43,45],{"href":44},"\u002Fposts\u002Fspinning-up-rl-intro-2","第二篇","一致；文中提到的行号指 GitHub 上对应源文件的行号；选读证明与旧版 Tensorflow 内容链接到官方站点。",[14,48,49,50,54],{},"本节讨论策略优化算法的数学基础，并把内容对接到示例代码上。我们会讲",[51,52,53],"strong",{},"策略梯度","（policy gradient）理论里的三个关键结果：",[56,57,58,67,75],"ul",{},[59,60,61,62,66],"li",{},"描述「策略性能对策略参数的梯度」的",[18,63,65],{"href":64},"#%E6%8E%A8%E5%AF%BC%E6%9C%80%E7%AE%80%E5%8D%95%E7%9A%84%E7%AD%96%E7%95%A5%E6%A2%AF%E5%BA%A6","最简单的方程","；",[59,68,69,70,74],{},"一条允许我们从该表达式里",[18,71,73],{"href":72},"#%E5%88%AB%E8%AE%A9%E8%BF%87%E5%8E%BB%E5%B9%B2%E6%89%B0%E4%BD%A0","去掉无用项","的规则；",[59,76,77,78,82],{},"一条允许我们",[18,79,81],{"href":80},"#%E7%AD%96%E7%95%A5%E6%A2%AF%E5%BA%A6%E4%B8%AD%E7%9A%84%E5%9F%BA%E7%BA%BF","加入有用项","的规则。",[14,84,85,86,91],{},"最后把这三个结果拼在一起，给出策略梯度的优势函数形式，也就是我们的 ",[18,87,90],{"href":88,"rel":89},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Falgorithms\u002Fvpg.html",[22],"Vanilla Policy Gradient","（VPG）实现里用的那个版本。",[93,94,95],"h2",{"id":95},"推导最简单的策略梯度",[14,97,98,99,210,211,498,499,543,544,548],{},"这里考虑随机型参数化策略 ",[100,101,104,133],"span",{"className":102},[103],"katex",[100,105,108],{"className":106},[107],"katex-mathml",[109,110,112],"math",{"xmlns":111},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[113,114,115,128],"semantics",{},[116,117,118],"mrow",{},[119,120,121,125],"msub",{},[122,123,124],"mi",{},"π",[122,126,127],{},"θ",[129,130,132],"annotation",{"encoding":131},"application\u002Fx-tex","\\pi_{\\theta}",[100,134,138],{"className":135,"ariaHidden":137},[136],"katex-html","true",[100,139,142,147],{"className":140},[141],"base",[100,143],{"className":144,"style":146},[145],"strut","height:0.5806em;vertical-align:-0.15em;",[100,148,151,156],{"className":149},[150],"mord",[100,152,124],{"className":153,"style":155},[150,154],"mathnormal","margin-right:0.0359em;",[100,157,160],{"className":158},[159],"msupsub",[100,161,165,201],{"className":162},[163,164],"vlist-t","vlist-t2",[100,166,169,196],{"className":167},[168],"vlist-r",[100,170,174],{"className":171,"style":173},[172],"vlist","height:0.3361em;",[100,175,177,182],{"style":176},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[100,178],{"className":179,"style":181},[180],"pstrut","height:2.7em;",[100,183,189],{"className":184},[185,186,187,188],"sizing","reset-size6","size3","mtight",[100,190,192],{"className":191},[150,188],[100,193,127],{"className":194,"style":195},[150,154,188],"margin-right:0.0278em;",[100,197,200],{"className":198},[199],"vlist-s","​",[100,202,204],{"className":203},[168],[100,205,208],{"className":206,"style":207},[172],"height:0.15em;",[100,209],{},"，目标是最大化期望回报 ",[100,212,214,286],{"className":213},[103],[100,215,217],{"className":216},[107],[109,218,219],{"xmlns":111},[113,220,221,283],{},[116,222,223,226,231,237,240,243,268,271,274,276,278,280],{},[122,224,225],{},"J",[227,228,230],"mo",{"stretchy":229},"false","(",[119,232,233,235],{},[122,234,124],{},[122,236,127],{},[227,238,239],{"stretchy":229},")",[227,241,242],{},"=",[122,244,245],{},[246,247,248,254],"munder",{},[227,249,250],{},[122,251,253],{"mathvariant":252},"double-struck","E",[116,255,256,259,262],{},[122,257,258],{},"τ",[227,260,261],{},"∼",[119,263,264,266],{},[122,265,124],{},[122,267,127],{},[227,269,270],{"stretchy":229},"[",[122,272,273],{},"R",[227,275,230],{"stretchy":229},[122,277,258],{},[227,279,239],{"stretchy":229},[227,281,282],{"stretchy":229},"]",[129,284,285],{"encoding":131},"J(\\pi_{\\theta}) = \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}[R(\\tau)]",[100,287,289,363],{"className":288,"ariaHidden":137},[136],[100,290,292,296,300,304,347,351,356,360],{"className":291},[141],[100,293],{"className":294,"style":295},[145],"height:1em;vertical-align:-0.25em;",[100,297,225],{"className":298,"style":299},[150,154],"margin-right:0.0962em;",[100,301,230],{"className":302},[303],"mopen",[100,305,307,310],{"className":306},[150],[100,308,124],{"className":309,"style":155},[150,154],[100,311,313],{"className":312},[159],[100,314,316,339],{"className":315},[163,164],[100,317,319,336],{"className":318},[168],[100,320,322],{"className":321,"style":173},[172],[100,323,324,327],{"style":176},[100,325],{"className":326,"style":181},[180],[100,328,330],{"className":329},[185,186,187,188],[100,331,333],{"className":332},[150,188],[100,334,127],{"className":335,"style":195},[150,154,188],[100,337,200],{"className":338},[199],[100,340,342],{"className":341},[168],[100,343,345],{"className":344,"style":207},[172],[100,346],{},[100,348,239],{"className":349},[350],"mclose",[100,352],{"className":353,"style":355},[354],"mspace","margin-right:0.2778em;",[100,357,242],{"className":358},[359],"mrel",[100,361],{"className":362,"style":355},[354],[100,364,366,370,481,484,488,491,494],{"className":365},[141],[100,367],{"className":368,"style":369},[145],"height:1.5559em;vertical-align:-0.8059em;",[100,371,373],{"className":372},[150],[100,374,378],{"className":375},[376,377],"mop","op-limits",[100,379,381,472],{"className":380},[163,164],[100,382,384,469],{"className":383},[168],[100,385,388,457],{"className":386,"style":387},[172],"height:0.6889em;",[100,389,391,395],{"style":390},"top:-2.4em;margin-left:0em;",[100,392],{"className":393,"style":394},[180],"height:3em;",[100,396,398],{"className":397},[185,186,187,188],[100,399,401,405,408],{"className":400},[150,188],[100,402,258],{"className":403,"style":404},[150,154,188],"margin-right:0.1132em;",[100,406,261],{"className":407},[359,188],[100,409,411,414],{"className":410},[150,188],[100,412,124],{"className":413,"style":155},[150,154,188],[100,415,417],{"className":416},[159],[100,418,420,448],{"className":419},[163,164],[100,421,423,445],{"className":422},[168],[100,424,427],{"className":425,"style":426},[172],"height:0.3448em;",[100,428,430,434],{"style":429},"top:-2.3488em;margin-left:-0.0359em;margin-right:0.0714em;",[100,431],{"className":432,"style":433},[180],"height:2.5em;",[100,435,439],{"className":436},[185,437,438,188],"reset-size3","size1",[100,440,442],{"className":441},[150,188],[100,443,127],{"className":444,"style":195},[150,154,188],[100,446,200],{"className":447},[199],[100,449,451],{"className":450},[168],[100,452,455],{"className":453,"style":454},[172],"height:0.1512em;",[100,456],{},[100,458,460,463],{"style":459},"top:-3em;",[100,461],{"className":462,"style":394},[180],[100,464,465],{},[100,466,253],{"className":467},[376,468],"mathbb",[100,470,200],{"className":471},[199],[100,473,475],{"className":474},[168],[100,476,479],{"className":477,"style":478},[172],"height:0.8059em;",[100,480],{},[100,482,270],{"className":483},[303],[100,485,273],{"className":486,"style":487},[150,154],"margin-right:0.0077em;",[100,489,230],{"className":490},[303],[100,492,258],{"className":493,"style":404},[150,154],[100,495,497],{"className":496},[350],")]","。为推导方便，取 ",[100,500,502,522],{"className":501},[103],[100,503,505],{"className":504},[107],[109,506,507],{"xmlns":111},[113,508,509,519],{},[116,510,511,513,515,517],{},[122,512,273],{},[227,514,230],{"stretchy":229},[122,516,258],{},[227,518,239],{"stretchy":229},[129,520,521],{"encoding":131},"R(\\tau)",[100,523,525],{"className":524,"ariaHidden":137},[136],[100,526,528,531,534,537,540],{"className":527},[141],[100,529],{"className":530,"style":295},[145],[100,532,273],{"className":533,"style":487},[150,154],[100,535,230],{"className":536},[303],[100,538,258],{"className":539,"style":404},[150,154],[100,541,239],{"className":542},[350]," 为",[18,545,547],{"href":546},"\u002Fposts\u002Fspinning-up-rl-intro#%E5%A5%96%E5%8A%B1%E4%B8%8E%E5%9B%9E%E6%8A%A5","有限视界不折扣回报","；换成无限视界折扣回报的情形，推导几乎完全一样。",[14,550,551],{},"我们想用梯度上升来优化策略，比如：",[100,553,556],{"className":554},[555],"katex-display",[100,557,559,634],{"className":558},[103],[100,560,562],{"className":561},[107],[109,563,565],{"xmlns":111,"display":564},"block",[113,566,567,631],{},[116,568,569,585,587,593,595,598],{},[119,570,571,573],{},[122,572,127],{},[116,574,575,578,581],{},[122,576,577],{},"k",[227,579,580],{},"+",[582,583,584],"mn",{},"1",[227,586,242],{},[119,588,589,591],{},[122,590,127],{},[122,592,577],{},[227,594,580],{},[122,596,597],{},"α",[119,599,600,625],{},[116,601,602,610,612,614,620,622],{},[119,603,604,608],{},[122,605,607],{"mathvariant":606},"normal","∇",[122,609,127],{},[122,611,225],{},[227,613,230],{"stretchy":229},[119,615,616,618],{},[122,617,124],{},[122,619,127],{},[227,621,239],{"stretchy":229},[227,623,624],{"fence":137},"∣",[119,626,627,629],{},[122,628,127],{},[122,630,577],{},[129,632,633],{"encoding":131},"\\theta_{k+1} = \\theta_k + \\alpha \\left. \\nabla_{\\theta} J(\\pi_{\\theta}) \\right|_{\\theta_k}",[100,635,637,706,763],{"className":636,"ariaHidden":137},[136],[100,638,640,644,697,700,703],{"className":639},[141],[100,641],{"className":642,"style":643},[145],"height:0.9028em;vertical-align:-0.2083em;",[100,645,647,650],{"className":646},[150],[100,648,127],{"className":649,"style":195},[150,154],[100,651,653],{"className":652},[159],[100,654,656,688],{"className":655},[163,164],[100,657,659,685],{"className":658},[168],[100,660,662],{"className":661,"style":173},[172],[100,663,665,668],{"style":664},"top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;",[100,666],{"className":667,"style":181},[180],[100,669,671],{"className":670},[185,186,187,188],[100,672,674,678,682],{"className":673},[150,188],[100,675,577],{"className":676,"style":677},[150,154,188],"margin-right:0.0315em;",[100,679,580],{"className":680},[681,188],"mbin",[100,683,584],{"className":684},[150,188],[100,686,200],{"className":687},[199],[100,689,691],{"className":690},[168],[100,692,695],{"className":693,"style":694},[172],"height:0.2083em;",[100,696],{},[100,698],{"className":699,"style":355},[354],[100,701,242],{"className":702},[359],[100,704],{"className":705,"style":355},[354],[100,707,709,713,753,757,760],{"className":708},[141],[100,710],{"className":711,"style":712},[145],"height:0.8444em;vertical-align:-0.15em;",[100,714,716,719],{"className":715},[150],[100,717,127],{"className":718,"style":195},[150,154],[100,720,722],{"className":721},[159],[100,723,725,745],{"className":724},[163,164],[100,726,728,742],{"className":727},[168],[100,729,731],{"className":730,"style":173},[172],[100,732,733,736],{"style":664},[100,734],{"className":735,"style":181},[180],[100,737,739],{"className":738},[185,186,187,188],[100,740,577],{"className":741,"style":677},[150,154,188],[100,743,200],{"className":744},[199],[100,746,748],{"className":747},[168],[100,749,751],{"className":750,"style":207},[172],[100,752],{},[100,754],{"className":755,"style":756},[354],"margin-right:0.2222em;",[100,758,580],{"className":759},[681],[100,761],{"className":762,"style":756},[354],[100,764,766,770,774,778],{"className":765},[141],[100,767],{"className":768,"style":769},[145],"height:1.1556em;vertical-align:-0.4056em;",[100,771,597],{"className":772,"style":773},[150,154],"margin-right:0.0037em;",[100,775],{"className":776,"style":777},[354],"margin-right:0.1667em;",[100,779,782,890],{"className":780},[781],"minner",[100,783,785,789,833,836,839,882,885],{"className":784},[781],[100,786],{"className":787},[303,788],"nulldelimiter",[100,790,792,795],{"className":791},[150],[100,793,607],{"className":794},[150],[100,796,798],{"className":797},[159],[100,799,801,825],{"className":800},[163,164],[100,802,804,822],{"className":803},[168],[100,805,807],{"className":806,"style":173},[172],[100,808,810,813],{"style":809},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[100,811],{"className":812,"style":181},[180],[100,814,816],{"className":815},[185,186,187,188],[100,817,819],{"className":818},[150,188],[100,820,127],{"className":821,"style":195},[150,154,188],[100,823,200],{"className":824},[199],[100,826,828],{"className":827},[168],[100,829,831],{"className":830,"style":207},[172],[100,832],{},[100,834,225],{"className":835,"style":299},[150,154],[100,837,230],{"className":838},[303],[100,840,842,845],{"className":841},[150],[100,843,124],{"className":844,"style":155},[150,154],[100,846,848],{"className":847},[159],[100,849,851,874],{"className":850},[163,164],[100,852,854,871],{"className":853},[168],[100,855,857],{"className":856,"style":173},[172],[100,858,859,862],{"style":176},[100,860],{"className":861,"style":181},[180],[100,863,865],{"className":864},[185,186,187,188],[100,866,868],{"className":867},[150,188],[100,869,127],{"className":870,"style":195},[150,154,188],[100,872,200],{"className":873},[199],[100,875,877],{"className":876},[168],[100,878,880],{"className":879,"style":207},[172],[100,881],{},[100,883,239],{"className":884},[350],[100,886,624],{"className":887,"style":889},[350,888],"delimcenter","top:0em;",[100,891,893],{"className":892},[159],[100,894,896,959],{"className":895},[163,164],[100,897,899,956],{"className":898},[168],[100,900,903],{"className":901,"style":902},[172],"height:0.1864em;",[100,904,906,909],{"style":905},"top:-2.4003em;margin-right:0.05em;",[100,907],{"className":908,"style":181},[180],[100,910,912],{"className":911},[185,186,187,188],[100,913,915],{"className":914},[150,188],[100,916,918,921],{"className":917},[150,188],[100,919,127],{"className":920,"style":195},[150,154,188],[100,922,924],{"className":923},[159],[100,925,927,948],{"className":926},[163,164],[100,928,930,945],{"className":929},[168],[100,931,933],{"className":932,"style":426},[172],[100,934,936,939],{"style":935},"top:-2.3488em;margin-left:-0.0278em;margin-right:0.0714em;",[100,937],{"className":938,"style":433},[180],[100,940,942],{"className":941},[185,437,438,188],[100,943,577],{"className":944,"style":677},[150,154,188],[100,946,200],{"className":947},[199],[100,949,951],{"className":950},[168],[100,952,954],{"className":953,"style":454},[172],[100,955],{},[100,957,200],{"className":958},[199],[100,960,962],{"className":961},[168],[100,963,966],{"className":964,"style":965},[172],"height:0.4056em;",[100,967],{},[14,969,970,971,1108,1109,1111,1112,1115],{},"策略性能的梯度 ",[100,972,974,1004],{"className":973},[103],[100,975,977],{"className":976},[107],[109,978,979],{"xmlns":111},[113,980,981,1001],{},[116,982,983,989,991,993,999],{},[119,984,985,987],{},[122,986,607],{"mathvariant":606},[122,988,127],{},[122,990,225],{},[227,992,230],{"stretchy":229},[119,994,995,997],{},[122,996,124],{},[122,998,127],{},[227,1000,239],{"stretchy":229},[129,1002,1003],{"encoding":131},"\\nabla_{\\theta} J(\\pi_{\\theta})",[100,1005,1007],{"className":1006,"ariaHidden":137},[136],[100,1008,1010,1013,1056,1059,1062,1105],{"className":1009},[141],[100,1011],{"className":1012,"style":295},[145],[100,1014,1016,1019],{"className":1015},[150],[100,1017,607],{"className":1018},[150],[100,1020,1022],{"className":1021},[159],[100,1023,1025,1048],{"className":1024},[163,164],[100,1026,1028,1045],{"className":1027},[168],[100,1029,1031],{"className":1030,"style":173},[172],[100,1032,1033,1036],{"style":809},[100,1034],{"className":1035,"style":181},[180],[100,1037,1039],{"className":1038},[185,186,187,188],[100,1040,1042],{"className":1041},[150,188],[100,1043,127],{"className":1044,"style":195},[150,154,188],[100,1046,200],{"className":1047},[199],[100,1049,1051],{"className":1050},[168],[100,1052,1054],{"className":1053,"style":207},[172],[100,1055],{},[100,1057,225],{"className":1058,"style":299},[150,154],[100,1060,230],{"className":1061},[303],[100,1063,1065,1068],{"className":1064},[150],[100,1066,124],{"className":1067,"style":155},[150,154],[100,1069,1071],{"className":1070},[159],[100,1072,1074,1097],{"className":1073},[163,164],[100,1075,1077,1094],{"className":1076},[168],[100,1078,1080],{"className":1079,"style":173},[172],[100,1081,1082,1085],{"style":176},[100,1083],{"className":1084,"style":181},[180],[100,1086,1088],{"className":1087},[185,186,187,188],[100,1089,1091],{"className":1090},[150,188],[100,1092,127],{"className":1093,"style":195},[150,154,188],[100,1095,200],{"className":1096},[199],[100,1098,1100],{"className":1099},[168],[100,1101,1103],{"className":1102,"style":207},[172],[100,1104],{},[100,1106,239],{"className":1107},[350]," 叫作",[51,1110,53],{},"，按这个方式优化策略的算法叫",[51,1113,1114],{},"策略梯度算法","（例子有 Vanilla Policy Gradient 和 TRPO。PPO 常被叫作策略梯度算法，不过严格说稍有不准）。",[14,1117,1118],{},"要真正用上这个算法，得先有一个能数值计算的策略梯度表达式。这分两步：1）推导策略性能的解析梯度，它最后会化成一个期望的形式；2）给这个期望做样本估计，用有限步「智能体-环境交互」的数据就能算出来。",[14,1120,1121],{},"本小节先找出这个表达式最简单的形式；后面几小节再展示如何在最简单的形式上改进，得到标准策略梯度实现里真正使用的版本。",[14,1123,1124],{},"先摆出几个对推导解析梯度有用的事实。",[14,1126,1127,1130,1131,1203,1204,1457],{},[51,1128,1129],{},"1. 轨迹的概率","。动作来自 ",[100,1132,1134,1151],{"className":1133},[103],[100,1135,1137],{"className":1136},[107],[109,1138,1139],{"xmlns":111},[113,1140,1141,1149],{},[116,1142,1143],{},[119,1144,1145,1147],{},[122,1146,124],{},[122,1148,127],{},[129,1150,132],{"encoding":131},[100,1152,1154],{"className":1153,"ariaHidden":137},[136],[100,1155,1157,1160],{"className":1156},[141],[100,1158],{"className":1159,"style":146},[145],[100,1161,1163,1166],{"className":1162},[150],[100,1164,124],{"className":1165,"style":155},[150,154],[100,1167,1169],{"className":1168},[159],[100,1170,1172,1195],{"className":1171},[163,164],[100,1173,1175,1192],{"className":1174},[168],[100,1176,1178],{"className":1177,"style":173},[172],[100,1179,1180,1183],{"style":176},[100,1181],{"className":1182,"style":181},[180],[100,1184,1186],{"className":1185},[185,186,187,188],[100,1187,1189],{"className":1188},[150,188],[100,1190,127],{"className":1191,"style":195},[150,154,188],[100,1193,200],{"className":1194},[199],[100,1196,1198],{"className":1197},[168],[100,1199,1201],{"className":1200,"style":207},[172],[100,1202],{}," 时，轨迹 ",[100,1205,1207,1268],{"className":1206},[103],[100,1208,1210],{"className":1209},[107],[109,1211,1212],{"xmlns":111},[113,1213,1214,1265],{},[116,1215,1216,1218,1220,1222,1230,1233,1239,1241,1244,1246,1248,1250,1263],{},[122,1217,258],{},[227,1219,242],{},[227,1221,230],{"stretchy":229},[119,1223,1224,1227],{},[122,1225,1226],{},"s",[582,1228,1229],{},"0",[227,1231,1232],{"separator":137},",",[119,1234,1235,1237],{},[122,1236,18],{},[582,1238,1229],{},[227,1240,1232],{"separator":137},[122,1242,1243],{"mathvariant":606},".",[122,1245,1243],{"mathvariant":606},[122,1247,1243],{"mathvariant":606},[227,1249,1232],{"separator":137},[119,1251,1252,1254],{},[122,1253,1226],{},[116,1255,1256,1259,1261],{},[122,1257,1258],{},"T",[227,1260,580],{},[582,1262,584],{},[227,1264,239],{"stretchy":229},[129,1266,1267],{"encoding":131},"\\tau = (s_0, a_0, ..., s_{T+1})",[100,1269,1271,1290],{"className":1270,"ariaHidden":137},[136],[100,1272,1274,1278,1281,1284,1287],{"className":1273},[141],[100,1275],{"className":1276,"style":1277},[145],"height:0.4306em;",[100,1279,258],{"className":1280,"style":404},[150,154],[100,1282],{"className":1283,"style":355},[354],[100,1285,242],{"className":1286},[359],[100,1288],{"className":1289,"style":355},[354],[100,1291,1293,1296,1299,1340,1344,1347,1387,1390,1393,1397,1400,1403,1454],{"className":1292},[141],[100,1294],{"className":1295,"style":295},[145],[100,1297,230],{"className":1298},[303],[100,1300,1302,1305],{"className":1301},[150],[100,1303,1226],{"className":1304},[150,154],[100,1306,1308],{"className":1307},[159],[100,1309,1311,1332],{"className":1310},[163,164],[100,1312,1314,1329],{"className":1313},[168],[100,1315,1318],{"className":1316,"style":1317},[172],"height:0.3011em;",[100,1319,1320,1323],{"style":809},[100,1321],{"className":1322,"style":181},[180],[100,1324,1326],{"className":1325},[185,186,187,188],[100,1327,1229],{"className":1328},[150,188],[100,1330,200],{"className":1331},[199],[100,1333,1335],{"className":1334},[168],[100,1336,1338],{"className":1337,"style":207},[172],[100,1339],{},[100,1341,1232],{"className":1342},[1343],"mpunct",[100,1345],{"className":1346,"style":777},[354],[100,1348,1350,1353],{"className":1349},[150],[100,1351,18],{"className":1352},[150,154],[100,1354,1356],{"className":1355},[159],[100,1357,1359,1379],{"className":1358},[163,164],[100,1360,1362,1376],{"className":1361},[168],[100,1363,1365],{"className":1364,"style":1317},[172],[100,1366,1367,1370],{"style":809},[100,1368],{"className":1369,"style":181},[180],[100,1371,1373],{"className":1372},[185,186,187,188],[100,1374,1229],{"className":1375},[150,188],[100,1377,200],{"className":1378},[199],[100,1380,1382],{"className":1381},[168],[100,1383,1385],{"className":1384,"style":207},[172],[100,1386],{},[100,1388,1232],{"className":1389},[1343],[100,1391],{"className":1392,"style":777},[354],[100,1394,1396],{"className":1395},[150],"...",[100,1398,1232],{"className":1399},[1343],[100,1401],{"className":1402,"style":777},[354],[100,1404,1406,1409],{"className":1405},[150],[100,1407,1226],{"className":1408},[150,154],[100,1410,1412],{"className":1411},[159],[100,1413,1415,1446],{"className":1414},[163,164],[100,1416,1418,1443],{"className":1417},[168],[100,1419,1422],{"className":1420,"style":1421},[172],"height:0.3283em;",[100,1423,1424,1427],{"style":809},[100,1425],{"className":1426,"style":181},[180],[100,1428,1430],{"className":1429},[185,186,187,188],[100,1431,1433,1437,1440],{"className":1432},[150,188],[100,1434,1258],{"className":1435,"style":1436},[150,154,188],"margin-right:0.1389em;",[100,1438,580],{"className":1439},[681,188],[100,1441,584],{"className":1442},[150,188],[100,1444,200],{"className":1445},[199],[100,1447,1449],{"className":1448},[168],[100,1450,1452],{"className":1451,"style":694},[172],[100,1453],{},[100,1455,239],{"className":1456},[350]," 的概率是",[100,1459,1461],{"className":1460},[555],[100,1462,1464,1583],{"className":1463},[103],[100,1465,1467],{"className":1466},[107],[109,1468,1469],{"xmlns":111,"display":564},[113,1470,1471,1580],{},[116,1472,1473,1476,1478,1480,1482,1484,1486,1488,1495,1497,1503,1505,1522,1524,1526,1538,1540,1546,1548,1554,1556,1562,1564,1570,1572,1578],{},[122,1474,1475],{},"P",[227,1477,230],{"stretchy":229},[122,1479,258],{},[122,1481,624],{"mathvariant":606},[122,1483,127],{},[227,1485,239],{"stretchy":229},[227,1487,242],{},[119,1489,1490,1493],{},[122,1491,1492],{},"ρ",[582,1494,1229],{},[227,1496,230],{"stretchy":229},[119,1498,1499,1501],{},[122,1500,1226],{},[582,1502,1229],{},[227,1504,239],{"stretchy":229},[1506,1507,1508,1511,1520],"munderover",{},[227,1509,1510],{},"∏",[116,1512,1513,1516,1518],{},[122,1514,1515],{},"t",[227,1517,242],{},[582,1519,1229],{},[122,1521,1258],{},[122,1523,1475],{},[227,1525,230],{"stretchy":229},[119,1527,1528,1530],{},[122,1529,1226],{},[116,1531,1532,1534,1536],{},[122,1533,1515],{},[227,1535,580],{},[582,1537,584],{},[122,1539,624],{"mathvariant":606},[119,1541,1542,1544],{},[122,1543,1226],{},[122,1545,1515],{},[227,1547,1232],{"separator":137},[119,1549,1550,1552],{},[122,1551,18],{},[122,1553,1515],{},[227,1555,239],{"stretchy":229},[119,1557,1558,1560],{},[122,1559,124],{},[122,1561,127],{},[227,1563,230],{"stretchy":229},[119,1565,1566,1568],{},[122,1567,18],{},[122,1569,1515],{},[122,1571,624],{"mathvariant":606},[119,1573,1574,1576],{},[122,1575,1226],{},[122,1577,1515],{},[227,1579,239],{"stretchy":229},[129,1581,1582],{"encoding":131},"P(\\tau|\\theta) = \\rho_0 (s_0) \\prod_{t=0}^{T} P(s_{t+1}|s_t, a_t) \\pi_{\\theta}(a_t |s_t)",[100,1584,1586,1619],{"className":1585,"ariaHidden":137},[136],[100,1587,1589,1592,1595,1598,1601,1604,1607,1610,1613,1616],{"className":1588},[141],[100,1590],{"className":1591,"style":295},[145],[100,1593,1475],{"className":1594,"style":1436},[150,154],[100,1596,230],{"className":1597},[303],[100,1599,258],{"className":1600,"style":404},[150,154],[100,1602,624],{"className":1603},[150],[100,1605,127],{"className":1606,"style":195},[150,154],[100,1608,239],{"className":1609},[350],[100,1611],{"className":1612,"style":355},[354],[100,1614,242],{"className":1615},[359],[100,1617],{"className":1618,"style":355},[354],[100,1620,1622,1626,1666,1669,1709,1712,1715,1790,1793,1796,1799,1848,1851,1892,1895,1898,1938,1941,1984,1987,2027,2030,2070],{"className":1621},[141],[100,1623],{"className":1624,"style":1625},[145],"height:3.0954em;vertical-align:-1.2671em;",[100,1627,1629,1632],{"className":1628},[150],[100,1630,1492],{"className":1631},[150,154],[100,1633,1635],{"className":1634},[159],[100,1636,1638,1658],{"className":1637},[163,164],[100,1639,1641,1655],{"className":1640},[168],[100,1642,1644],{"className":1643,"style":1317},[172],[100,1645,1646,1649],{"style":809},[100,1647],{"className":1648,"style":181},[180],[100,1650,1652],{"className":1651},[185,186,187,188],[100,1653,1229],{"className":1654},[150,188],[100,1656,200],{"className":1657},[199],[100,1659,1661],{"className":1660},[168],[100,1662,1664],{"className":1663,"style":207},[172],[100,1665],{},[100,1667,230],{"className":1668},[303],[100,1670,1672,1675],{"className":1671},[150],[100,1673,1226],{"className":1674},[150,154],[100,1676,1678],{"className":1677},[159],[100,1679,1681,1701],{"className":1680},[163,164],[100,1682,1684,1698],{"className":1683},[168],[100,1685,1687],{"className":1686,"style":1317},[172],[100,1688,1689,1692],{"style":809},[100,1690],{"className":1691,"style":181},[180],[100,1693,1695],{"className":1694},[185,186,187,188],[100,1696,1229],{"className":1697},[150,188],[100,1699,200],{"className":1700},[199],[100,1702,1704],{"className":1703},[168],[100,1705,1707],{"className":1706,"style":207},[172],[100,1708],{},[100,1710,239],{"className":1711},[350],[100,1713],{"className":1714,"style":777},[354],[100,1716,1718],{"className":1717},[376,377],[100,1719,1721,1781],{"className":1720},[163,164],[100,1722,1724,1778],{"className":1723},[168],[100,1725,1728,1750,1763],{"className":1726,"style":1727},[172],"height:1.8283em;",[100,1729,1731,1735],{"style":1730},"top:-1.8829em;margin-left:0em;",[100,1732],{"className":1733,"style":1734},[180],"height:3.05em;",[100,1736,1738],{"className":1737},[185,186,187,188],[100,1739,1741,1744,1747],{"className":1740},[150,188],[100,1742,1515],{"className":1743},[150,154,188],[100,1745,242],{"className":1746},[359,188],[100,1748,1229],{"className":1749},[150,188],[100,1751,1753,1756],{"style":1752},"top:-3.05em;",[100,1754],{"className":1755,"style":1734},[180],[100,1757,1758],{},[100,1759,1510],{"className":1760},[376,1761,1762],"op-symbol","large-op",[100,1764,1766,1769],{"style":1765},"top:-4.3em;margin-left:0em;",[100,1767],{"className":1768,"style":1734},[180],[100,1770,1772],{"className":1771},[185,186,187,188],[100,1773,1775],{"className":1774},[150,188],[100,1776,1258],{"className":1777,"style":1436},[150,154,188],[100,1779,200],{"className":1780},[199],[100,1782,1784],{"className":1783},[168],[100,1785,1788],{"className":1786,"style":1787},[172],"height:1.2671em;",[100,1789],{},[100,1791],{"className":1792,"style":777},[354],[100,1794,1475],{"className":1795,"style":1436},[150,154],[100,1797,230],{"className":1798},[303],[100,1800,1802,1805],{"className":1801},[150],[100,1803,1226],{"className":1804},[150,154],[100,1806,1808],{"className":1807},[159],[100,1809,1811,1840],{"className":1810},[163,164],[100,1812,1814,1837],{"className":1813},[168],[100,1815,1817],{"className":1816,"style":1317},[172],[100,1818,1819,1822],{"style":809},[100,1820],{"className":1821,"style":181},[180],[100,1823,1825],{"className":1824},[185,186,187,188],[100,1826,1828,1831,1834],{"className":1827},[150,188],[100,1829,1515],{"className":1830},[150,154,188],[100,1832,580],{"className":1833},[681,188],[100,1835,584],{"className":1836},[150,188],[100,1838,200],{"className":1839},[199],[100,1841,1843],{"className":1842},[168],[100,1844,1846],{"className":1845,"style":694},[172],[100,1847],{},[100,1849,624],{"className":1850},[150],[100,1852,1854,1857],{"className":1853},[150],[100,1855,1226],{"className":1856},[150,154],[100,1858,1860],{"className":1859},[159],[100,1861,1863,1884],{"className":1862},[163,164],[100,1864,1866,1881],{"className":1865},[168],[100,1867,1870],{"className":1868,"style":1869},[172],"height:0.2806em;",[100,1871,1872,1875],{"style":809},[100,1873],{"className":1874,"style":181},[180],[100,1876,1878],{"className":1877},[185,186,187,188],[100,1879,1515],{"className":1880},[150,154,188],[100,1882,200],{"className":1883},[199],[100,1885,1887],{"className":1886},[168],[100,1888,1890],{"className":1889,"style":207},[172],[100,1891],{},[100,1893,1232],{"className":1894},[1343],[100,1896],{"className":1897,"style":777},[354],[100,1899,1901,1904],{"className":1900},[150],[100,1902,18],{"className":1903},[150,154],[100,1905,1907],{"className":1906},[159],[100,1908,1910,1930],{"className":1909},[163,164],[100,1911,1913,1927],{"className":1912},[168],[100,1914,1916],{"className":1915,"style":1869},[172],[100,1917,1918,1921],{"style":809},[100,1919],{"className":1920,"style":181},[180],[100,1922,1924],{"className":1923},[185,186,187,188],[100,1925,1515],{"className":1926},[150,154,188],[100,1928,200],{"className":1929},[199],[100,1931,1933],{"className":1932},[168],[100,1934,1936],{"className":1935,"style":207},[172],[100,1937],{},[100,1939,239],{"className":1940},[350],[100,1942,1944,1947],{"className":1943},[150],[100,1945,124],{"className":1946,"style":155},[150,154],[100,1948,1950],{"className":1949},[159],[100,1951,1953,1976],{"className":1952},[163,164],[100,1954,1956,1973],{"className":1955},[168],[100,1957,1959],{"className":1958,"style":173},[172],[100,1960,1961,1964],{"style":176},[100,1962],{"className":1963,"style":181},[180],[100,1965,1967],{"className":1966},[185,186,187,188],[100,1968,1970],{"className":1969},[150,188],[100,1971,127],{"className":1972,"style":195},[150,154,188],[100,1974,200],{"className":1975},[199],[100,1977,1979],{"className":1978},[168],[100,1980,1982],{"className":1981,"style":207},[172],[100,1983],{},[100,1985,230],{"className":1986},[303],[100,1988,1990,1993],{"className":1989},[150],[100,1991,18],{"className":1992},[150,154],[100,1994,1996],{"className":1995},[159],[100,1997,1999,2019],{"className":1998},[163,164],[100,2000,2002,2016],{"className":2001},[168],[100,2003,2005],{"className":2004,"style":1869},[172],[100,2006,2007,2010],{"style":809},[100,2008],{"className":2009,"style":181},[180],[100,2011,2013],{"className":2012},[185,186,187,188],[100,2014,1515],{"className":2015},[150,154,188],[100,2017,200],{"className":2018},[199],[100,2020,2022],{"className":2021},[168],[100,2023,2025],{"className":2024,"style":207},[172],[100,2026],{},[100,2028,624],{"className":2029},[150],[100,2031,2033,2036],{"className":2032},[150],[100,2034,1226],{"className":2035},[150,154],[100,2037,2039],{"className":2038},[159],[100,2040,2042,2062],{"className":2041},[163,164],[100,2043,2045,2059],{"className":2044},[168],[100,2046,2048],{"className":2047,"style":1869},[172],[100,2049,2050,2053],{"style":809},[100,2051],{"className":2052,"style":181},[180],[100,2054,2056],{"className":2055},[185,186,187,188],[100,2057,1515],{"className":2058},[150,154,188],[100,2060,200],{"className":2061},[199],[100,2063,2065],{"className":2064},[168],[100,2066,2068],{"className":2067,"style":207},[172],[100,2069],{},[100,2071,239],{"className":2072},[350],[14,2074,2075,2078,2079,2127,2128,2156,2157,2195],{},[51,2076,2077],{},"2. 对数求导技巧（log-derivative trick）","。它基于微积分里一条简单的规则：",[100,2080,2082,2103],{"className":2081},[103],[100,2083,2085],{"className":2084},[107],[109,2086,2087],{"xmlns":111},[113,2088,2089,2100],{},[116,2090,2091,2094,2097],{},[122,2092,2093],{},"log",[227,2095,2096],{},"⁡",[122,2098,2099],{},"x",[129,2101,2102],{"encoding":131},"\\log x",[100,2104,2106],{"className":2105,"ariaHidden":137},[136],[100,2107,2109,2113,2121,2124],{"className":2108},[141],[100,2110],{"className":2111,"style":2112},[145],"height:0.8889em;vertical-align:-0.1944em;",[100,2114,2116,2117],{"className":2115},[376],"lo",[100,2118,2120],{"style":2119},"margin-right:0.0139em;","g",[100,2122],{"className":2123,"style":777},[354],[100,2125,2099],{"className":2126},[150,154]," 对 ",[100,2129,2131,2144],{"className":2130},[103],[100,2132,2134],{"className":2133},[107],[109,2135,2136],{"xmlns":111},[113,2137,2138,2142],{},[116,2139,2140],{},[122,2141,2099],{},[129,2143,2099],{"encoding":131},[100,2145,2147],{"className":2146,"ariaHidden":137},[136],[100,2148,2150,2153],{"className":2149},[141],[100,2151],{"className":2152,"style":1277},[145],[100,2154,2099],{"className":2155},[150,154]," 的导数是 ",[100,2158,2160,2179],{"className":2159},[103],[100,2161,2163],{"className":2162},[107],[109,2164,2165],{"xmlns":111},[113,2166,2167,2176],{},[116,2168,2169,2171,2174],{},[582,2170,584],{},[122,2172,2173],{"mathvariant":606},"\u002F",[122,2175,2099],{},[129,2177,2178],{"encoding":131},"1\u002Fx",[100,2180,2182],{"className":2181,"ariaHidden":137},[136],[100,2183,2185,2188,2192],{"className":2184},[141],[100,2186],{"className":2187,"style":295},[145],[100,2189,2191],{"className":2190},[150],"1\u002F",[100,2193,2099],{"className":2194},[150,154],"。稍作变形、再与链式法则结合，得到：",[100,2197,2199],{"className":2198},[555],[100,2200,2202,2268],{"className":2201},[103],[100,2203,2205],{"className":2204},[107],[109,2206,2207],{"xmlns":111,"display":564},[113,2208,2209,2265],{},[116,2210,2211,2217,2219,2221,2223,2225,2227,2229,2231,2233,2235,2237,2239,2241,2243,2249,2251,2253,2255,2257,2259,2261,2263],{},[119,2212,2213,2215],{},[122,2214,607],{"mathvariant":606},[122,2216,127],{},[122,2218,1475],{},[227,2220,230],{"stretchy":229},[122,2222,258],{},[122,2224,624],{"mathvariant":606},[122,2226,127],{},[227,2228,239],{"stretchy":229},[227,2230,242],{},[122,2232,1475],{},[227,2234,230],{"stretchy":229},[122,2236,258],{},[122,2238,624],{"mathvariant":606},[122,2240,127],{},[227,2242,239],{"stretchy":229},[119,2244,2245,2247],{},[122,2246,607],{"mathvariant":606},[122,2248,127],{},[122,2250,2093],{},[227,2252,2096],{},[122,2254,1475],{},[227,2256,230],{"stretchy":229},[122,2258,258],{},[122,2260,624],{"mathvariant":606},[122,2262,127],{},[227,2264,239],{"stretchy":229},[129,2266,2267],{"encoding":131},"\\nabla_{\\theta} P(\\tau | \\theta) = P(\\tau | \\theta) \\nabla_{\\theta} \\log P(\\tau | \\theta)",[100,2269,2271,2347],{"className":2270,"ariaHidden":137},[136],[100,2272,2274,2277,2320,2323,2326,2329,2332,2335,2338,2341,2344],{"className":2273},[141],[100,2275],{"className":2276,"style":295},[145],[100,2278,2280,2283],{"className":2279},[150],[100,2281,607],{"className":2282},[150],[100,2284,2286],{"className":2285},[159],[100,2287,2289,2312],{"className":2288},[163,164],[100,2290,2292,2309],{"className":2291},[168],[100,2293,2295],{"className":2294,"style":173},[172],[100,2296,2297,2300],{"style":809},[100,2298],{"className":2299,"style":181},[180],[100,2301,2303],{"className":2302},[185,186,187,188],[100,2304,2306],{"className":2305},[150,188],[100,2307,127],{"className":2308,"style":195},[150,154,188],[100,2310,200],{"className":2311},[199],[100,2313,2315],{"className":2314},[168],[100,2316,2318],{"className":2317,"style":207},[172],[100,2319],{},[100,2321,1475],{"className":2322,"style":1436},[150,154],[100,2324,230],{"className":2325},[303],[100,2327,258],{"className":2328,"style":404},[150,154],[100,2330,624],{"className":2331},[150],[100,2333,127],{"className":2334,"style":195},[150,154],[100,2336,239],{"className":2337},[350],[100,2339],{"className":2340,"style":355},[354],[100,2342,242],{"className":2343},[359],[100,2345],{"className":2346,"style":355},[354],[100,2348,2350,2353,2356,2359,2362,2365,2368,2371,2414,2417,2422,2425,2428,2431,2434,2437,2440],{"className":2349},[141],[100,2351],{"className":2352,"style":295},[145],[100,2354,1475],{"className":2355,"style":1436},[150,154],[100,2357,230],{"className":2358},[303],[100,2360,258],{"className":2361,"style":404},[150,154],[100,2363,624],{"className":2364},[150],[100,2366,127],{"className":2367,"style":195},[150,154],[100,2369,239],{"className":2370},[350],[100,2372,2374,2377],{"className":2373},[150],[100,2375,607],{"className":2376},[150],[100,2378,2380],{"className":2379},[159],[100,2381,2383,2406],{"className":2382},[163,164],[100,2384,2386,2403],{"className":2385},[168],[100,2387,2389],{"className":2388,"style":173},[172],[100,2390,2391,2394],{"style":809},[100,2392],{"className":2393,"style":181},[180],[100,2395,2397],{"className":2396},[185,186,187,188],[100,2398,2400],{"className":2399},[150,188],[100,2401,127],{"className":2402,"style":195},[150,154,188],[100,2404,200],{"className":2405},[199],[100,2407,2409],{"className":2408},[168],[100,2410,2412],{"className":2411,"style":207},[172],[100,2413],{},[100,2415],{"className":2416,"style":777},[354],[100,2418,2116,2420],{"className":2419},[376],[100,2421,2120],{"style":2119},[100,2423],{"className":2424,"style":777},[354],[100,2426,1475],{"className":2427,"style":1436},[150,154],[100,2429,230],{"className":2430},[303],[100,2432,258],{"className":2433,"style":404},[150,154],[100,2435,624],{"className":2436},[150],[100,2438,127],{"className":2439,"style":195},[150,154],[100,2441,239],{"className":2442},[350],[14,2444,2445,2448],{},[51,2446,2447],{},"3. 轨迹的对数概率","。轨迹的 log 概率就是",[100,2450,2452],{"className":2451},[555],[100,2453,2455,2595],{"className":2454},[103],[100,2456,2458],{"className":2457},[107],[109,2459,2460],{"xmlns":111,"display":564},[113,2461,2462,2592],{},[116,2463,2464,2466,2468,2470,2472,2474,2476,2478,2480,2482,2484,2486,2492,2494,2500,2502,2504,2519,2522,2524,2526,2528,2530,2542,2544,2550,2552,2558,2560,2562,2564,2566,2572,2574,2580,2582,2588,2590],{},[122,2465,2093],{},[227,2467,2096],{},[122,2469,1475],{},[227,2471,230],{"stretchy":229},[122,2473,258],{},[122,2475,624],{"mathvariant":606},[122,2477,127],{},[227,2479,239],{"stretchy":229},[227,2481,242],{},[122,2483,2093],{},[227,2485,2096],{},[119,2487,2488,2490],{},[122,2489,1492],{},[582,2491,1229],{},[227,2493,230],{"stretchy":229},[119,2495,2496,2498],{},[122,2497,1226],{},[582,2499,1229],{},[227,2501,239],{"stretchy":229},[227,2503,580],{},[1506,2505,2506,2509,2517],{},[227,2507,2508],{},"∑",[116,2510,2511,2513,2515],{},[122,2512,1515],{},[227,2514,242],{},[582,2516,1229],{},[122,2518,1258],{},[227,2520,230],{"fence":229,"stretchy":137,"minsize":2521,"maxsize":2521},"2.4em",[122,2523,2093],{},[227,2525,2096],{},[122,2527,1475],{},[227,2529,230],{"stretchy":229},[119,2531,2532,2534],{},[122,2533,1226],{},[116,2535,2536,2538,2540],{},[122,2537,1515],{},[227,2539,580],{},[582,2541,584],{},[122,2543,624],{"mathvariant":606},[119,2545,2546,2548],{},[122,2547,1226],{},[122,2549,1515],{},[227,2551,1232],{"separator":137},[119,2553,2554,2556],{},[122,2555,18],{},[122,2557,1515],{},[227,2559,239],{"stretchy":229},[227,2561,580],{},[122,2563,2093],{},[227,2565,2096],{},[119,2567,2568,2570],{},[122,2569,124],{},[122,2571,127],{},[227,2573,230],{"stretchy":229},[119,2575,2576,2578],{},[122,2577,18],{},[122,2579,1515],{},[122,2581,624],{"mathvariant":606},[119,2583,2584,2586],{},[122,2585,1226],{},[122,2587,1515],{},[227,2589,239],{"stretchy":229},[227,2591,239],{"fence":229,"stretchy":137,"minsize":2521,"maxsize":2521},[129,2593,2594],{"encoding":131},"\\log P(\\tau|\\theta) = \\log \\rho_0 (s_0) + \\sum_{t=0}^{T} \\bigg( \\log P(s_{t+1}|s_t, a_t)  + \\log \\pi_{\\theta}(a_t |s_t)\\bigg)",[100,2596,2598,2639,2748,2998],{"className":2597,"ariaHidden":137},[136],[100,2599,2601,2604,2609,2612,2615,2618,2621,2624,2627,2630,2633,2636],{"className":2600},[141],[100,2602],{"className":2603,"style":295},[145],[100,2605,2116,2607],{"className":2606},[376],[100,2608,2120],{"style":2119},[100,2610],{"className":2611,"style":777},[354],[100,2613,1475],{"className":2614,"style":1436},[150,154],[100,2616,230],{"className":2617},[303],[100,2619,258],{"className":2620,"style":404},[150,154],[100,2622,624],{"className":2623},[150],[100,2625,127],{"className":2626,"style":195},[150,154],[100,2628,239],{"className":2629},[350],[100,2631],{"className":2632,"style":355},[354],[100,2634,242],{"className":2635},[359],[100,2637],{"className":2638,"style":355},[354],[100,2640,2642,2645,2650,2653,2693,2696,2736,2739,2742,2745],{"className":2641},[141],[100,2643],{"className":2644,"style":295},[145],[100,2646,2116,2648],{"className":2647},[376],[100,2649,2120],{"style":2119},[100,2651],{"className":2652,"style":777},[354],[100,2654,2656,2659],{"className":2655},[150],[100,2657,1492],{"className":2658},[150,154],[100,2660,2662],{"className":2661},[159],[100,2663,2665,2685],{"className":2664},[163,164],[100,2666,2668,2682],{"className":2667},[168],[100,2669,2671],{"className":2670,"style":1317},[172],[100,2672,2673,2676],{"style":809},[100,2674],{"className":2675,"style":181},[180],[100,2677,2679],{"className":2678},[185,186,187,188],[100,2680,1229],{"className":2681},[150,188],[100,2683,200],{"className":2684},[199],[100,2686,2688],{"className":2687},[168],[100,2689,2691],{"className":2690,"style":207},[172],[100,2692],{},[100,2694,230],{"className":2695},[303],[100,2697,2699,2702],{"className":2698},[150],[100,2700,1226],{"className":2701},[150,154],[100,2703,2705],{"className":2704},[159],[100,2706,2708,2728],{"className":2707},[163,164],[100,2709,2711,2725],{"className":2710},[168],[100,2712,2714],{"className":2713,"style":1317},[172],[100,2715,2716,2719],{"style":809},[100,2717],{"className":2718,"style":181},[180],[100,2720,2722],{"className":2721},[185,186,187,188],[100,2723,1229],{"className":2724},[150,188],[100,2726,200],{"className":2727},[199],[100,2729,2731],{"className":2730},[168],[100,2732,2734],{"className":2733,"style":207},[172],[100,2735],{},[100,2737,239],{"className":2738},[350],[100,2740],{"className":2741,"style":756},[354],[100,2743,580],{"className":2744},[681],[100,2746],{"className":2747,"style":756},[354],[100,2749,2751,2754,2821,2824,2831,2834,2839,2842,2845,2848,2897,2900,2940,2943,2946,2986,2989,2992,2995],{"className":2750},[141],[100,2752],{"className":2753,"style":1625},[145],[100,2755,2757],{"className":2756},[376,377],[100,2758,2760,2813],{"className":2759},[163,164],[100,2761,2763,2810],{"className":2762},[168],[100,2764,2766,2786,2796],{"className":2765,"style":1727},[172],[100,2767,2768,2771],{"style":1730},[100,2769],{"className":2770,"style":1734},[180],[100,2772,2774],{"className":2773},[185,186,187,188],[100,2775,2777,2780,2783],{"className":2776},[150,188],[100,2778,1515],{"className":2779},[150,154,188],[100,2781,242],{"className":2782},[359,188],[100,2784,1229],{"className":2785},[150,188],[100,2787,2788,2791],{"style":1752},[100,2789],{"className":2790,"style":1734},[180],[100,2792,2793],{},[100,2794,2508],{"className":2795},[376,1761,1762],[100,2797,2798,2801],{"style":1765},[100,2799],{"className":2800,"style":1734},[180],[100,2802,2804],{"className":2803},[185,186,187,188],[100,2805,2807],{"className":2806},[150,188],[100,2808,1258],{"className":2809,"style":1436},[150,154,188],[100,2811,200],{"className":2812},[199],[100,2814,2816],{"className":2815},[168],[100,2817,2819],{"className":2818,"style":1787},[172],[100,2820],{},[100,2822],{"className":2823,"style":777},[354],[100,2825,2827],{"className":2826},[150],[100,2828,230],{"className":2829},[2830,187],"delimsizing",[100,2832],{"className":2833,"style":777},[354],[100,2835,2116,2837],{"className":2836},[376],[100,2838,2120],{"style":2119},[100,2840],{"className":2841,"style":777},[354],[100,2843,1475],{"className":2844,"style":1436},[150,154],[100,2846,230],{"className":2847},[303],[100,2849,2851,2854],{"className":2850},[150],[100,2852,1226],{"className":2853},[150,154],[100,2855,2857],{"className":2856},[159],[100,2858,2860,2889],{"className":2859},[163,164],[100,2861,2863,2886],{"className":2862},[168],[100,2864,2866],{"className":2865,"style":1317},[172],[100,2867,2868,2871],{"style":809},[100,2869],{"className":2870,"style":181},[180],[100,2872,2874],{"className":2873},[185,186,187,188],[100,2875,2877,2880,2883],{"className":2876},[150,188],[100,2878,1515],{"className":2879},[150,154,188],[100,2881,580],{"className":2882},[681,188],[100,2884,584],{"className":2885},[150,188],[100,2887,200],{"className":2888},[199],[100,2890,2892],{"className":2891},[168],[100,2893,2895],{"className":2894,"style":694},[172],[100,2896],{},[100,2898,624],{"className":2899},[150],[100,2901,2903,2906],{"className":2902},[150],[100,2904,1226],{"className":2905},[150,154],[100,2907,2909],{"className":2908},[159],[100,2910,2912,2932],{"className":2911},[163,164],[100,2913,2915,2929],{"className":2914},[168],[100,2916,2918],{"className":2917,"style":1869},[172],[100,2919,2920,2923],{"style":809},[100,2921],{"className":2922,"style":181},[180],[100,2924,2926],{"className":2925},[185,186,187,188],[100,2927,1515],{"className":2928},[150,154,188],[100,2930,200],{"className":2931},[199],[100,2933,2935],{"className":2934},[168],[100,2936,2938],{"className":2937,"style":207},[172],[100,2939],{},[100,2941,1232],{"className":2942},[1343],[100,2944],{"className":2945,"style":777},[354],[100,2947,2949,2952],{"className":2948},[150],[100,2950,18],{"className":2951},[150,154],[100,2953,2955],{"className":2954},[159],[100,2956,2958,2978],{"className":2957},[163,164],[100,2959,2961,2975],{"className":2960},[168],[100,2962,2964],{"className":2963,"style":1869},[172],[100,2965,2966,2969],{"style":809},[100,2967],{"className":2968,"style":181},[180],[100,2970,2972],{"className":2971},[185,186,187,188],[100,2973,1515],{"className":2974},[150,154,188],[100,2976,200],{"className":2977},[199],[100,2979,2981],{"className":2980},[168],[100,2982,2984],{"className":2983,"style":207},[172],[100,2985],{},[100,2987,239],{"className":2988},[350],[100,2990],{"className":2991,"style":756},[354],[100,2993,580],{"className":2994},[681],[100,2996],{"className":2997,"style":756},[354],[100,2999,3001,3005,3010,3013,3056,3059,3099,3102,3142,3145],{"className":3000},[141],[100,3002],{"className":3003,"style":3004},[145],"height:2.4em;vertical-align:-0.95em;",[100,3006,2116,3008],{"className":3007},[376],[100,3009,2120],{"style":2119},[100,3011],{"className":3012,"style":777},[354],[100,3014,3016,3019],{"className":3015},[150],[100,3017,124],{"className":3018,"style":155},[150,154],[100,3020,3022],{"className":3021},[159],[100,3023,3025,3048],{"className":3024},[163,164],[100,3026,3028,3045],{"className":3027},[168],[100,3029,3031],{"className":3030,"style":173},[172],[100,3032,3033,3036],{"style":176},[100,3034],{"className":3035,"style":181},[180],[100,3037,3039],{"className":3038},[185,186,187,188],[100,3040,3042],{"className":3041},[150,188],[100,3043,127],{"className":3044,"style":195},[150,154,188],[100,3046,200],{"className":3047},[199],[100,3049,3051],{"className":3050},[168],[100,3052,3054],{"className":3053,"style":207},[172],[100,3055],{},[100,3057,230],{"className":3058},[303],[100,3060,3062,3065],{"className":3061},[150],[100,3063,18],{"className":3064},[150,154],[100,3066,3068],{"className":3067},[159],[100,3069,3071,3091],{"className":3070},[163,164],[100,3072,3074,3088],{"className":3073},[168],[100,3075,3077],{"className":3076,"style":1869},[172],[100,3078,3079,3082],{"style":809},[100,3080],{"className":3081,"style":181},[180],[100,3083,3085],{"className":3084},[185,186,187,188],[100,3086,1515],{"className":3087},[150,154,188],[100,3089,200],{"className":3090},[199],[100,3092,3094],{"className":3093},[168],[100,3095,3097],{"className":3096,"style":207},[172],[100,3098],{},[100,3100,624],{"className":3101},[150],[100,3103,3105,3108],{"className":3104},[150],[100,3106,1226],{"className":3107},[150,154],[100,3109,3111],{"className":3110},[159],[100,3112,3114,3134],{"className":3113},[163,164],[100,3115,3117,3131],{"className":3116},[168],[100,3118,3120],{"className":3119,"style":1869},[172],[100,3121,3122,3125],{"style":809},[100,3123],{"className":3124,"style":181},[180],[100,3126,3128],{"className":3127},[185,186,187,188],[100,3129,1515],{"className":3130},[150,154,188],[100,3132,200],{"className":3133},[199],[100,3135,3137],{"className":3136},[168],[100,3138,3140],{"className":3139,"style":207},[172],[100,3141],{},[100,3143,239],{"className":3144},[350],[100,3146,3148],{"className":3147},[150],[100,3149,239],{"className":3150},[2830,187],[14,3152,3153,3156,3157,3187,3188,41,3314,3519,3520,3563],{},[51,3154,3155],{},"4. 环境函数的梯度","。环境与 ",[100,3158,3160,3174],{"className":3159},[103],[100,3161,3163],{"className":3162},[107],[109,3164,3165],{"xmlns":111},[113,3166,3167,3171],{},[116,3168,3169],{},[122,3170,127],{},[129,3172,3173],{"encoding":131},"\\theta",[100,3175,3177],{"className":3176,"ariaHidden":137},[136],[100,3178,3180,3184],{"className":3179},[141],[100,3181],{"className":3182,"style":3183},[145],"height:0.6944em;",[100,3185,127],{"className":3186,"style":195},[150,154]," 无关，所以 ",[100,3189,3191,3219],{"className":3190},[103],[100,3192,3194],{"className":3193},[107],[109,3195,3196],{"xmlns":111},[113,3197,3198,3216],{},[116,3199,3200,3206,3208,3214],{},[119,3201,3202,3204],{},[122,3203,1492],{},[582,3205,1229],{},[227,3207,230],{"stretchy":229},[119,3209,3210,3212],{},[122,3211,1226],{},[582,3213,1229],{},[227,3215,239],{"stretchy":229},[129,3217,3218],{"encoding":131},"\\rho_0(s_0)",[100,3220,3222],{"className":3221,"ariaHidden":137},[136],[100,3223,3225,3228,3268,3271,3311],{"className":3224},[141],[100,3226],{"className":3227,"style":295},[145],[100,3229,3231,3234],{"className":3230},[150],[100,3232,1492],{"className":3233},[150,154],[100,3235,3237],{"className":3236},[159],[100,3238,3240,3260],{"className":3239},[163,164],[100,3241,3243,3257],{"className":3242},[168],[100,3244,3246],{"className":3245,"style":1317},[172],[100,3247,3248,3251],{"style":809},[100,3249],{"className":3250,"style":181},[180],[100,3252,3254],{"className":3253},[185,186,187,188],[100,3255,1229],{"className":3256},[150,188],[100,3258,200],{"className":3259},[199],[100,3261,3263],{"className":3262},[168],[100,3264,3266],{"className":3265,"style":207},[172],[100,3267],{},[100,3269,230],{"className":3270},[303],[100,3272,3274,3277],{"className":3273},[150],[100,3275,1226],{"className":3276},[150,154],[100,3278,3280],{"className":3279},[159],[100,3281,3283,3303],{"className":3282},[163,164],[100,3284,3286,3300],{"className":3285},[168],[100,3287,3289],{"className":3288,"style":1317},[172],[100,3290,3291,3294],{"style":809},[100,3292],{"className":3293,"style":181},[180],[100,3295,3297],{"className":3296},[185,186,187,188],[100,3298,1229],{"className":3299},[150,188],[100,3301,200],{"className":3302},[199],[100,3304,3306],{"className":3305},[168],[100,3307,3309],{"className":3308,"style":207},[172],[100,3310],{},[100,3312,239],{"className":3313},[350],[100,3315,3317,3363],{"className":3316},[103],[100,3318,3320],{"className":3319},[107],[109,3321,3322],{"xmlns":111},[113,3323,3324,3360],{},[116,3325,3326,3328,3330,3342,3344,3350,3352,3358],{},[122,3327,1475],{},[227,3329,230],{"stretchy":229},[119,3331,3332,3334],{},[122,3333,1226],{},[116,3335,3336,3338,3340],{},[122,3337,1515],{},[227,3339,580],{},[582,3341,584],{},[122,3343,624],{"mathvariant":606},[119,3345,3346,3348],{},[122,3347,1226],{},[122,3349,1515],{},[227,3351,1232],{"separator":137},[119,3353,3354,3356],{},[122,3355,18],{},[122,3357,1515],{},[227,3359,239],{"stretchy":229},[129,3361,3362],{"encoding":131},"P(s_{t+1}|s_t, a_t)",[100,3364,3366],{"className":3365,"ariaHidden":137},[136],[100,3367,3369,3372,3375,3378,3427,3430,3470,3473,3476,3516],{"className":3368},[141],[100,3370],{"className":3371,"style":295},[145],[100,3373,1475],{"className":3374,"style":1436},[150,154],[100,3376,230],{"className":3377},[303],[100,3379,3381,3384],{"className":3380},[150],[100,3382,1226],{"className":3383},[150,154],[100,3385,3387],{"className":3386},[159],[100,3388,3390,3419],{"className":3389},[163,164],[100,3391,3393,3416],{"className":3392},[168],[100,3394,3396],{"className":3395,"style":1317},[172],[100,3397,3398,3401],{"style":809},[100,3399],{"className":3400,"style":181},[180],[100,3402,3404],{"className":3403},[185,186,187,188],[100,3405,3407,3410,3413],{"className":3406},[150,188],[100,3408,1515],{"className":3409},[150,154,188],[100,3411,580],{"className":3412},[681,188],[100,3414,584],{"className":3415},[150,188],[100,3417,200],{"className":3418},[199],[100,3420,3422],{"className":3421},[168],[100,3423,3425],{"className":3424,"style":694},[172],[100,3426],{},[100,3428,624],{"className":3429},[150],[100,3431,3433,3436],{"className":3432},[150],[100,3434,1226],{"className":3435},[150,154],[100,3437,3439],{"className":3438},[159],[100,3440,3442,3462],{"className":3441},[163,164],[100,3443,3445,3459],{"className":3444},[168],[100,3446,3448],{"className":3447,"style":1869},[172],[100,3449,3450,3453],{"style":809},[100,3451],{"className":3452,"style":181},[180],[100,3454,3456],{"className":3455},[185,186,187,188],[100,3457,1515],{"className":3458},[150,154,188],[100,3460,200],{"className":3461},[199],[100,3463,3465],{"className":3464},[168],[100,3466,3468],{"className":3467,"style":207},[172],[100,3469],{},[100,3471,1232],{"className":3472},[1343],[100,3474],{"className":3475,"style":777},[354],[100,3477,3479,3482],{"className":3478},[150],[100,3480,18],{"className":3481},[150,154],[100,3483,3485],{"className":3484},[159],[100,3486,3488,3508],{"className":3487},[163,164],[100,3489,3491,3505],{"className":3490},[168],[100,3492,3494],{"className":3493,"style":1869},[172],[100,3495,3496,3499],{"style":809},[100,3497],{"className":3498,"style":181},[180],[100,3500,3502],{"className":3501},[185,186,187,188],[100,3503,1515],{"className":3504},[150,154,188],[100,3506,200],{"className":3507},[199],[100,3509,3511],{"className":3510},[168],[100,3512,3514],{"className":3513,"style":207},[172],[100,3515],{},[100,3517,239],{"className":3518},[350]," 和 ",[100,3521,3523,3542],{"className":3522},[103],[100,3524,3526],{"className":3525},[107],[109,3527,3528],{"xmlns":111},[113,3529,3530,3540],{},[116,3531,3532,3534,3536,3538],{},[122,3533,273],{},[227,3535,230],{"stretchy":229},[122,3537,258],{},[227,3539,239],{"stretchy":229},[129,3541,521],{"encoding":131},[100,3543,3545],{"className":3544,"ariaHidden":137},[136],[100,3546,3548,3551,3554,3557,3560],{"className":3547},[141],[100,3549],{"className":3550,"style":295},[145],[100,3552,273],{"className":3553,"style":487},[150,154],[100,3555,230],{"className":3556},[303],[100,3558,258],{"className":3559,"style":404},[150,154],[100,3561,239],{"className":3562},[350]," 的梯度都是零。",[14,3565,3566,3569],{},[51,3567,3568],{},"5. 轨迹的 grad-log-prob","。于是轨迹对数概率的梯度为",[100,3571,3573],{"className":3572},[555],[100,3574,3576,3837],{"className":3575},[103],[100,3577,3579],{"className":3578},[107],[109,3580,3581],{"xmlns":111,"display":564},[113,3582,3583,3834],{},[3584,3585,3589,3768],"mtable",{"rowspacing":3586,"columnalign":3587,"columnspacing":3588},"0.25em","right left","0em",[3590,3591,3592,3622],"mtr",{},[3593,3594,3595],"mtd",{},[3596,3597,3598],"mstyle",{"scriptlevel":1229,"displaystyle":137},[116,3599,3600,3606,3608,3610,3612,3614,3616,3618,3620],{},[119,3601,3602,3604],{},[122,3603,607],{"mathvariant":606},[122,3605,127],{},[122,3607,2093],{},[227,3609,2096],{},[122,3611,1475],{},[227,3613,230],{"stretchy":229},[122,3615,258],{},[122,3617,624],{"mathvariant":606},[122,3619,127],{},[227,3621,239],{"stretchy":229},[3593,3623,3624],{},[3596,3625,3626],{"scriptlevel":1229,"displaystyle":137},[116,3627,3628,3630,3632,3664,3666,3680,3682,3730,3732,3738,3740,3742,3748,3750,3756,3758,3764,3766],{},[116,3629],{},[227,3631,242],{},[3633,3634,3636],"menclose",{"notation":3635},"updiagonalstrike",[116,3637,3638,3644,3646,3648,3654,3656,3662],{},[119,3639,3640,3642],{},[122,3641,607],{"mathvariant":606},[122,3643,127],{},[122,3645,2093],{},[227,3647,2096],{},[119,3649,3650,3652],{},[122,3651,1492],{},[582,3653,1229],{},[227,3655,230],{"stretchy":229},[119,3657,3658,3660],{},[122,3659,1226],{},[582,3661,1229],{},[227,3663,239],{"stretchy":229},[227,3665,580],{},[1506,3667,3668,3670,3678],{},[227,3669,2508],{},[116,3671,3672,3674,3676],{},[122,3673,1515],{},[227,3675,242],{},[582,3677,1229],{},[122,3679,1258],{},[227,3681,230],{"fence":229,"stretchy":137,"minsize":2521,"maxsize":2521},[3633,3683,3684],{"notation":3635},[116,3685,3686,3692,3694,3696,3698,3700,3712,3714,3720,3722,3728],{},[119,3687,3688,3690],{},[122,3689,607],{"mathvariant":606},[122,3691,127],{},[122,3693,2093],{},[227,3695,2096],{},[122,3697,1475],{},[227,3699,230],{"stretchy":229},[119,3701,3702,3704],{},[122,3703,1226],{},[116,3705,3706,3708,3710],{},[122,3707,1515],{},[227,3709,580],{},[582,3711,584],{},[122,3713,624],{"mathvariant":606},[119,3715,3716,3718],{},[122,3717,1226],{},[122,3719,1515],{},[227,3721,1232],{"separator":137},[119,3723,3724,3726],{},[122,3725,18],{},[122,3727,1515],{},[227,3729,239],{"stretchy":229},[227,3731,580],{},[119,3733,3734,3736],{},[122,3735,607],{"mathvariant":606},[122,3737,127],{},[122,3739,2093],{},[227,3741,2096],{},[119,3743,3744,3746],{},[122,3745,124],{},[122,3747,127],{},[227,3749,230],{"stretchy":229},[119,3751,3752,3754],{},[122,3753,18],{},[122,3755,1515],{},[122,3757,624],{"mathvariant":606},[119,3759,3760,3762],{},[122,3761,1226],{},[122,3763,1515],{},[227,3765,239],{"stretchy":229},[227,3767,239],{"fence":229,"stretchy":137,"minsize":2521,"maxsize":2521},[3590,3769,3770,3776],{},[3593,3771,3772],{},[3596,3773,3774],{"scriptlevel":1229,"displaystyle":137},[116,3775],{},[3593,3777,3778],{},[3596,3779,3780],{"scriptlevel":1229,"displaystyle":137},[116,3781,3782,3784,3786,3800,3806,3808,3810,3816,3818,3824,3826,3832],{},[116,3783],{},[227,3785,242],{},[1506,3787,3788,3790,3798],{},[227,3789,2508],{},[116,3791,3792,3794,3796],{},[122,3793,1515],{},[227,3795,242],{},[582,3797,1229],{},[122,3799,1258],{},[119,3801,3802,3804],{},[122,3803,607],{"mathvariant":606},[122,3805,127],{},[122,3807,2093],{},[227,3809,2096],{},[119,3811,3812,3814],{},[122,3813,124],{},[122,3815,127],{},[227,3817,230],{"stretchy":229},[119,3819,3820,3822],{},[122,3821,18],{},[122,3823,1515],{},[122,3825,624],{"mathvariant":606},[119,3827,3828,3830],{},[122,3829,1226],{},[122,3831,1515],{},[227,3833,239],{"stretchy":229},[129,3835,3836],{"encoding":131},"\\begin{aligned}\n\\nabla_{\\theta} \\log P(\\tau | \\theta) &= \\cancel{\\nabla_{\\theta} \\log \\rho_0 (s_0)} + \\sum_{t=0}^{T} \\bigg( \\cancel{\\nabla_{\\theta} \\log P(s_{t+1}|s_t, a_t)}  + \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t)\\bigg) \\\\\n&= \\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t)\n\\end{aligned}",[100,3838,3840],{"className":3839,"ariaHidden":137},[136],[100,3841,3843,3847],{"className":3842},[141],[100,3844],{"className":3845,"style":3846},[145],"height:6.4909em;vertical-align:-2.9954em;",[100,3848,3850],{"className":3849},[150],[100,3851,3853,3970],{"className":3852},[3584],[100,3854,3857],{"className":3855},[3856],"col-align-r",[100,3858,3860,3961],{"className":3859},[163,164],[100,3861,3863,3958],{"className":3862},[168],[100,3864,3867,3949],{"className":3865,"style":3866},[172],"height:3.4954em;",[100,3868,3870,3874],{"style":3869},"top:-5.4954em;",[100,3871],{"className":3872,"style":3873},[180],"height:3.8283em;",[100,3875,3877,3920,3923,3928,3931,3934,3937,3940,3943,3946],{"className":3876},[150],[100,3878,3880,3883],{"className":3879},[150],[100,3881,607],{"className":3882},[150],[100,3884,3886],{"className":3885},[159],[100,3887,3889,3912],{"className":3888},[163,164],[100,3890,3892,3909],{"className":3891},[168],[100,3893,3895],{"className":3894,"style":173},[172],[100,3896,3897,3900],{"style":809},[100,3898],{"className":3899,"style":181},[180],[100,3901,3903],{"className":3902},[185,186,187,188],[100,3904,3906],{"className":3905},[150,188],[100,3907,127],{"className":3908,"style":195},[150,154,188],[100,3910,200],{"className":3911},[199],[100,3913,3915],{"className":3914},[168],[100,3916,3918],{"className":3917,"style":207},[172],[100,3919],{},[100,3921],{"className":3922,"style":777},[354],[100,3924,2116,3926],{"className":3925},[376],[100,3927,2120],{"style":2119},[100,3929],{"className":3930,"style":777},[354],[100,3932,1475],{"className":3933,"style":1436},[150,154],[100,3935,230],{"className":3936},[303],[100,3938,258],{"className":3939,"style":404},[150,154],[100,3941,624],{"className":3942},[150],[100,3944,127],{"className":3945,"style":195},[150,154],[100,3947,239],{"className":3948},[350],[100,3950,3952,3955],{"style":3951},"top:-2.1em;",[100,3953],{"className":3954,"style":3873},[180],[100,3956],{"className":3957},[150],[100,3959,200],{"className":3960},[199],[100,3962,3964],{"className":3963},[168],[100,3965,3968],{"className":3966,"style":3967},[172],"height:2.9954em;",[100,3969],{},[100,3971,3974],{"className":3972},[3973],"col-align-l",[100,3975,3977,5008],{"className":3976},[163,164],[100,3978,3980,5005],{"className":3979},[168],[100,3981,3983,4729],{"className":3982,"style":3866},[172],[100,3984,3985,3988],{"style":3869},[100,3986],{"className":3987,"style":3873},[180],[100,3989,3991,3994,3997,4000,4003,4199,4202,4205,4208,4275,4278,4284,4528,4531,4534,4537,4580,4583,4588,4591,4634,4637,4677,4680,4720,4723],{"className":3990},[150],[100,3992],{"className":3993},[150],[100,3995],{"className":3996,"style":355},[354],[100,3998,242],{"className":3999},[359],[100,4001],{"className":4002,"style":355},[354],[100,4004,4007],{"className":4005},[150,4006],"cancel-lap",[100,4008,4010,4190],{"className":4009},[163,164],[100,4011,4013,4187],{"className":4012},[168],[100,4014,4017,4166],{"className":4015,"style":4016},[172],"height:0.75em;",[100,4018,4019,4022],{"style":459},[100,4020],{"className":4021,"style":394},[180],[100,4023,4026,4069,4072,4077,4080,4120,4123,4163],{"className":4024},[150,4025],"cancel-pad",[100,4027,4029,4032],{"className":4028},[150],[100,4030,607],{"className":4031},[150],[100,4033,4035],{"className":4034},[159],[100,4036,4038,4061],{"className":4037},[163,164],[100,4039,4041,4058],{"className":4040},[168],[100,4042,4044],{"className":4043,"style":173},[172],[100,4045,4046,4049],{"style":809},[100,4047],{"className":4048,"style":181},[180],[100,4050,4052],{"className":4051},[185,186,187,188],[100,4053,4055],{"className":4054},[150,188],[100,4056,127],{"className":4057,"style":195},[150,154,188],[100,4059,200],{"className":4060},[199],[100,4062,4064],{"className":4063},[168],[100,4065,4067],{"className":4066,"style":207},[172],[100,4068],{},[100,4070],{"className":4071,"style":777},[354],[100,4073,2116,4075],{"className":4074},[376],[100,4076,2120],{"style":2119},[100,4078],{"className":4079,"style":777},[354],[100,4081,4083,4086],{"className":4082},[150],[100,4084,1492],{"className":4085},[150,154],[100,4087,4089],{"className":4088},[159],[100,4090,4092,4112],{"className":4091},[163,164],[100,4093,4095,4109],{"className":4094},[168],[100,4096,4098],{"className":4097,"style":1317},[172],[100,4099,4100,4103],{"style":809},[100,4101],{"className":4102,"style":181},[180],[100,4104,4106],{"className":4105},[185,186,187,188],[100,4107,1229],{"className":4108},[150,188],[100,4110,200],{"className":4111},[199],[100,4113,4115],{"className":4114},[168],[100,4116,4118],{"className":4117,"style":207},[172],[100,4119],{},[100,4121,230],{"className":4122},[303],[100,4124,4126,4129],{"className":4125},[150],[100,4127,1226],{"className":4128},[150,154],[100,4130,4132],{"className":4131},[159],[100,4133,4135,4155],{"className":4134},[163,164],[100,4136,4138,4152],{"className":4137},[168],[100,4139,4141],{"className":4140,"style":1317},[172],[100,4142,4143,4146],{"style":809},[100,4144],{"className":4145,"style":181},[180],[100,4147,4149],{"className":4148},[185,186,187,188],[100,4150,1229],{"className":4151},[150,188],[100,4153,200],{"className":4154},[199],[100,4156,4158],{"className":4157},[168],[100,4159,4161],{"className":4160,"style":207},[172],[100,4162],{},[100,4164,239],{"className":4165},[350],[100,4167,4171,4174],{"className":4168,"style":4170},[4169],"svg-align","top:-2.75em;",[100,4172],{"className":4173,"style":394},[180],[100,4175,4177],{"style":4176},"height:1em;",[4178,4179,4183],"svg",{"xmlns":4180,"width":4181,"height":4182},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","100%","1em",[4184,4185],"line",{"x1":1229,"y1":4181,"x2":4181,"y2":1229,"strokeWidth":4186},"0.046em",[100,4188,200],{"className":4189},[199],[100,4191,4193],{"className":4192},[168],[100,4194,4197],{"className":4195,"style":4196},[172],"height:0.25em;",[100,4198],{},[100,4200],{"className":4201,"style":756},[354],[100,4203,580],{"className":4204},[681],[100,4206],{"className":4207,"style":756},[354],[100,4209,4211],{"className":4210},[376,377],[100,4212,4214,4267],{"className":4213},[163,164],[100,4215,4217,4264],{"className":4216},[168],[100,4218,4220,4240,4250],{"className":4219,"style":1727},[172],[100,4221,4222,4225],{"style":1730},[100,4223],{"className":4224,"style":1734},[180],[100,4226,4228],{"className":4227},[185,186,187,188],[100,4229,4231,4234,4237],{"className":4230},[150,188],[100,4232,1515],{"className":4233},[150,154,188],[100,4235,242],{"className":4236},[359,188],[100,4238,1229],{"className":4239},[150,188],[100,4241,4242,4245],{"style":1752},[100,4243],{"className":4244,"style":1734},[180],[100,4246,4247],{},[100,4248,2508],{"className":4249},[376,1761,1762],[100,4251,4252,4255],{"style":1765},[100,4253],{"className":4254,"style":1734},[180],[100,4256,4258],{"className":4257},[185,186,187,188],[100,4259,4261],{"className":4260},[150,188],[100,4262,1258],{"className":4263,"style":1436},[150,154,188],[100,4265,200],{"className":4266},[199],[100,4268,4270],{"className":4269},[168],[100,4271,4273],{"className":4272,"style":1787},[172],[100,4274],{},[100,4276],{"className":4277,"style":777},[354],[100,4279,4281],{"className":4280},[150],[100,4282,230],{"className":4283},[2830,187],[100,4285,4287],{"className":4286},[150,4006],[100,4288,4290,4520],{"className":4289},[163,164],[100,4291,4293,4517],{"className":4292},[168],[100,4294,4296,4505],{"className":4295,"style":4016},[172],[100,4297,4298,4301],{"style":459},[100,4299],{"className":4300,"style":394},[180],[100,4302,4304,4347,4350,4355,4358,4361,4364,4413,4416,4456,4459,4462,4502],{"className":4303},[150,4025],[100,4305,4307,4310],{"className":4306},[150],[100,4308,607],{"className":4309},[150],[100,4311,4313],{"className":4312},[159],[100,4314,4316,4339],{"className":4315},[163,164],[100,4317,4319,4336],{"className":4318},[168],[100,4320,4322],{"className":4321,"style":173},[172],[100,4323,4324,4327],{"style":809},[100,4325],{"className":4326,"style":181},[180],[100,4328,4330],{"className":4329},[185,186,187,188],[100,4331,4333],{"className":4332},[150,188],[100,4334,127],{"className":4335,"style":195},[150,154,188],[100,4337,200],{"className":4338},[199],[100,4340,4342],{"className":4341},[168],[100,4343,4345],{"className":4344,"style":207},[172],[100,4346],{},[100,4348],{"className":4349,"style":777},[354],[100,4351,2116,4353],{"className":4352},[376],[100,4354,2120],{"style":2119},[100,4356],{"className":4357,"style":777},[354],[100,4359,1475],{"className":4360,"style":1436},[150,154],[100,4362,230],{"className":4363},[303],[100,4365,4367,4370],{"className":4366},[150],[100,4368,1226],{"className":4369},[150,154],[100,4371,4373],{"className":4372},[159],[100,4374,4376,4405],{"className":4375},[163,164],[100,4377,4379,4402],{"className":4378},[168],[100,4380,4382],{"className":4381,"style":1317},[172],[100,4383,4384,4387],{"style":809},[100,4385],{"className":4386,"style":181},[180],[100,4388,4390],{"className":4389},[185,186,187,188],[100,4391,4393,4396,4399],{"className":4392},[150,188],[100,4394,1515],{"className":4395},[150,154,188],[100,4397,580],{"className":4398},[681,188],[100,4400,584],{"className":4401},[150,188],[100,4403,200],{"className":4404},[199],[100,4406,4408],{"className":4407},[168],[100,4409,4411],{"className":4410,"style":694},[172],[100,4412],{},[100,4414,624],{"className":4415},[150],[100,4417,4419,4422],{"className":4418},[150],[100,4420,1226],{"className":4421},[150,154],[100,4423,4425],{"className":4424},[159],[100,4426,4428,4448],{"className":4427},[163,164],[100,4429,4431,4445],{"className":4430},[168],[100,4432,4434],{"className":4433,"style":1869},[172],[100,4435,4436,4439],{"style":809},[100,4437],{"className":4438,"style":181},[180],[100,4440,4442],{"className":4441},[185,186,187,188],[100,4443,1515],{"className":4444},[150,154,188],[100,4446,200],{"className":4447},[199],[100,4449,4451],{"className":4450},[168],[100,4452,4454],{"className":4453,"style":207},[172],[100,4455],{},[100,4457,1232],{"className":4458},[1343],[100,4460],{"className":4461,"style":777},[354],[100,4463,4465,4468],{"className":4464},[150],[100,4466,18],{"className":4467},[150,154],[100,4469,4471],{"className":4470},[159],[100,4472,4474,4494],{"className":4473},[163,164],[100,4475,4477,4491],{"className":4476},[168],[100,4478,4480],{"className":4479,"style":1869},[172],[100,4481,4482,4485],{"style":809},[100,4483],{"className":4484,"style":181},[180],[100,4486,4488],{"className":4487},[185,186,187,188],[100,4489,1515],{"className":4490},[150,154,188],[100,4492,200],{"className":4493},[199],[100,4495,4497],{"className":4496},[168],[100,4498,4500],{"className":4499,"style":207},[172],[100,4501],{},[100,4503,239],{"className":4504},[350],[100,4506,4508,4511],{"className":4507,"style":4170},[4169],[100,4509],{"className":4510,"style":394},[180],[100,4512,4513],{"style":4176},[4178,4514,4515],{"xmlns":4180,"width":4181,"height":4182},[4184,4516],{"x1":1229,"y1":4181,"x2":4181,"y2":1229,"strokeWidth":4186},[100,4518,200],{"className":4519},[199],[100,4521,4523],{"className":4522},[168],[100,4524,4526],{"className":4525,"style":4196},[172],[100,4527],{},[100,4529],{"className":4530,"style":756},[354],[100,4532,580],{"className":4533},[681],[100,4535],{"className":4536,"style":756},[354],[100,4538,4540,4543],{"className":4539},[150],[100,4541,607],{"className":4542},[150],[100,4544,4546],{"className":4545},[159],[100,4547,4549,4572],{"className":4548},[163,164],[100,4550,4552,4569],{"className":4551},[168],[100,4553,4555],{"className":4554,"style":173},[172],[100,4556,4557,4560],{"style":809},[100,4558],{"className":4559,"style":181},[180],[100,4561,4563],{"className":4562},[185,186,187,188],[100,4564,4566],{"className":4565},[150,188],[100,4567,127],{"className":4568,"style":195},[150,154,188],[100,4570,200],{"className":4571},[199],[100,4573,4575],{"className":4574},[168],[100,4576,4578],{"className":4577,"style":207},[172],[100,4579],{},[100,4581],{"className":4582,"style":777},[354],[100,4584,2116,4586],{"className":4585},[376],[100,4587,2120],{"style":2119},[100,4589],{"className":4590,"style":777},[354],[100,4592,4594,4597],{"className":4593},[150],[100,4595,124],{"className":4596,"style":155},[150,154],[100,4598,4600],{"className":4599},[159],[100,4601,4603,4626],{"className":4602},[163,164],[100,4604,4606,4623],{"className":4605},[168],[100,4607,4609],{"className":4608,"style":173},[172],[100,4610,4611,4614],{"style":176},[100,4612],{"className":4613,"style":181},[180],[100,4615,4617],{"className":4616},[185,186,187,188],[100,4618,4620],{"className":4619},[150,188],[100,4621,127],{"className":4622,"style":195},[150,154,188],[100,4624,200],{"className":4625},[199],[100,4627,4629],{"className":4628},[168],[100,4630,4632],{"className":4631,"style":207},[172],[100,4633],{},[100,4635,230],{"className":4636},[303],[100,4638,4640,4643],{"className":4639},[150],[100,4641,18],{"className":4642},[150,154],[100,4644,4646],{"className":4645},[159],[100,4647,4649,4669],{"className":4648},[163,164],[100,4650,4652,4666],{"className":4651},[168],[100,4653,4655],{"className":4654,"style":1869},[172],[100,4656,4657,4660],{"style":809},[100,4658],{"className":4659,"style":181},[180],[100,4661,4663],{"className":4662},[185,186,187,188],[100,4664,1515],{"className":4665},[150,154,188],[100,4667,200],{"className":4668},[199],[100,4670,4672],{"className":4671},[168],[100,4673,4675],{"className":4674,"style":207},[172],[100,4676],{},[100,4678,624],{"className":4679},[150],[100,4681,4683,4686],{"className":4682},[150],[100,4684,1226],{"className":4685},[150,154],[100,4687,4689],{"className":4688},[159],[100,4690,4692,4712],{"className":4691},[163,164],[100,4693,4695,4709],{"className":4694},[168],[100,4696,4698],{"className":4697,"style":1869},[172],[100,4699,4700,4703],{"style":809},[100,4701],{"className":4702,"style":181},[180],[100,4704,4706],{"className":4705},[185,186,187,188],[100,4707,1515],{"className":4708},[150,154,188],[100,4710,200],{"className":4711},[199],[100,4713,4715],{"className":4714},[168],[100,4716,4718],{"className":4717,"style":207},[172],[100,4719],{},[100,4721,239],{"className":4722},[350],[100,4724,4726],{"className":4725},[150],[100,4727,239],{"className":4728},[2830,187],[100,4730,4731,4734],{"style":3951},[100,4732],{"className":4733,"style":3873},[180],[100,4735,4737,4740,4743,4746,4749,4816,4819,4862,4865,4870,4873,4916,4919,4959,4962,5002],{"className":4736},[150],[100,4738],{"className":4739},[150],[100,4741],{"className":4742,"style":355},[354],[100,4744,242],{"className":4745},[359],[100,4747],{"className":4748,"style":355},[354],[100,4750,4752],{"className":4751},[376,377],[100,4753,4755,4808],{"className":4754},[163,164],[100,4756,4758,4805],{"className":4757},[168],[100,4759,4761,4781,4791],{"className":4760,"style":1727},[172],[100,4762,4763,4766],{"style":1730},[100,4764],{"className":4765,"style":1734},[180],[100,4767,4769],{"className":4768},[185,186,187,188],[100,4770,4772,4775,4778],{"className":4771},[150,188],[100,4773,1515],{"className":4774},[150,154,188],[100,4776,242],{"className":4777},[359,188],[100,4779,1229],{"className":4780},[150,188],[100,4782,4783,4786],{"style":1752},[100,4784],{"className":4785,"style":1734},[180],[100,4787,4788],{},[100,4789,2508],{"className":4790},[376,1761,1762],[100,4792,4793,4796],{"style":1765},[100,4794],{"className":4795,"style":1734},[180],[100,4797,4799],{"className":4798},[185,186,187,188],[100,4800,4802],{"className":4801},[150,188],[100,4803,1258],{"className":4804,"style":1436},[150,154,188],[100,4806,200],{"className":4807},[199],[100,4809,4811],{"className":4810},[168],[100,4812,4814],{"className":4813,"style":1787},[172],[100,4815],{},[100,4817],{"className":4818,"style":777},[354],[100,4820,4822,4825],{"className":4821},[150],[100,4823,607],{"className":4824},[150],[100,4826,4828],{"className":4827},[159],[100,4829,4831,4854],{"className":4830},[163,164],[100,4832,4834,4851],{"className":4833},[168],[100,4835,4837],{"className":4836,"style":173},[172],[100,4838,4839,4842],{"style":809},[100,4840],{"className":4841,"style":181},[180],[100,4843,4845],{"className":4844},[185,186,187,188],[100,4846,4848],{"className":4847},[150,188],[100,4849,127],{"className":4850,"style":195},[150,154,188],[100,4852,200],{"className":4853},[199],[100,4855,4857],{"className":4856},[168],[100,4858,4860],{"className":4859,"style":207},[172],[100,4861],{},[100,4863],{"className":4864,"style":777},[354],[100,4866,2116,4868],{"className":4867},[376],[100,4869,2120],{"style":2119},[100,4871],{"className":4872,"style":777},[354],[100,4874,4876,4879],{"className":4875},[150],[100,4877,124],{"className":4878,"style":155},[150,154],[100,4880,4882],{"className":4881},[159],[100,4883,4885,4908],{"className":4884},[163,164],[100,4886,4888,4905],{"className":4887},[168],[100,4889,4891],{"className":4890,"style":173},[172],[100,4892,4893,4896],{"style":176},[100,4894],{"className":4895,"style":181},[180],[100,4897,4899],{"className":4898},[185,186,187,188],[100,4900,4902],{"className":4901},[150,188],[100,4903,127],{"className":4904,"style":195},[150,154,188],[100,4906,200],{"className":4907},[199],[100,4909,4911],{"className":4910},[168],[100,4912,4914],{"className":4913,"style":207},[172],[100,4915],{},[100,4917,230],{"className":4918},[303],[100,4920,4922,4925],{"className":4921},[150],[100,4923,18],{"className":4924},[150,154],[100,4926,4928],{"className":4927},[159],[100,4929,4931,4951],{"className":4930},[163,164],[100,4932,4934,4948],{"className":4933},[168],[100,4935,4937],{"className":4936,"style":1869},[172],[100,4938,4939,4942],{"style":809},[100,4940],{"className":4941,"style":181},[180],[100,4943,4945],{"className":4944},[185,186,187,188],[100,4946,1515],{"className":4947},[150,154,188],[100,4949,200],{"className":4950},[199],[100,4952,4954],{"className":4953},[168],[100,4955,4957],{"className":4956,"style":207},[172],[100,4958],{},[100,4960,624],{"className":4961},[150],[100,4963,4965,4968],{"className":4964},[150],[100,4966,1226],{"className":4967},[150,154],[100,4969,4971],{"className":4970},[159],[100,4972,4974,4994],{"className":4973},[163,164],[100,4975,4977,4991],{"className":4976},[168],[100,4978,4980],{"className":4979,"style":1869},[172],[100,4981,4982,4985],{"style":809},[100,4983],{"className":4984,"style":181},[180],[100,4986,4988],{"className":4987},[185,186,187,188],[100,4989,1515],{"className":4990},[150,154,188],[100,4992,200],{"className":4993},[199],[100,4995,4997],{"className":4996},[168],[100,4998,5000],{"className":4999,"style":207},[172],[100,5001],{},[100,5003,239],{"className":5004},[350],[100,5006,200],{"className":5007},[199],[100,5009,5011],{"className":5010},[168],[100,5012,5014],{"className":5013,"style":3967},[172],[100,5015],{},[14,5017,5018],{},"把这些拼起来，推出：",[10,5020,5022],{"title":5021},"基本策略梯度的推导",[100,5023,5025],{"className":5024},[555],[100,5026,5028,5576],{"className":5027},[103],[100,5029,5031],{"className":5030},[107],[109,5032,5033],{"xmlns":111,"display":564},[113,5034,5035,5573],{},[3584,5036,5039,5125,5194,5261,5344,5435],{"rowspacing":3586,"columnalign":5037,"columnspacing":5038},"right left right left","0em 1em 0em",[3590,5040,5041,5065,5113,5119],{},[3593,5042,5043],{},[3596,5044,5045],{"scriptlevel":1229,"displaystyle":137},[116,5046,5047,5053,5055,5057,5063],{},[119,5048,5049,5051],{},[122,5050,607],{"mathvariant":606},[122,5052,127],{},[122,5054,225],{},[227,5056,230],{"stretchy":229},[119,5058,5059,5061],{},[122,5060,124],{},[122,5062,127],{},[227,5064,239],{"stretchy":229},[3593,5066,5067],{},[3596,5068,5069],{"scriptlevel":1229,"displaystyle":137},[116,5070,5071,5073,5075,5081,5101,5103,5105,5107,5109,5111],{},[116,5072],{},[227,5074,242],{},[119,5076,5077,5079],{},[122,5078,607],{"mathvariant":606},[122,5080,127],{},[122,5082,5083],{},[246,5084,5085,5089],{},[227,5086,5087],{},[122,5088,253],{"mathvariant":252},[116,5090,5091,5093,5095],{},[122,5092,258],{},[227,5094,261],{},[119,5096,5097,5099],{},[122,5098,124],{},[122,5100,127],{},[227,5102,270],{"stretchy":229},[122,5104,273],{},[227,5106,230],{"stretchy":229},[122,5108,258],{},[227,5110,239],{"stretchy":229},[227,5112,282],{"stretchy":229},[3593,5114,5115],{},[3596,5116,5117],{"scriptlevel":1229,"displaystyle":137},[116,5118],{},[3593,5120,5121],{},[3596,5122,5123],{"scriptlevel":1229,"displaystyle":137},[116,5124],{},[3590,5126,5127,5133,5176,5182],{},[3593,5128,5129],{},[3596,5130,5131],{"scriptlevel":1229,"displaystyle":137},[116,5132],{},[3593,5134,5135],{},[3596,5136,5137],{"scriptlevel":1229,"displaystyle":137},[116,5138,5139,5141,5143,5149,5156,5158,5160,5162,5164,5166,5168,5170,5172,5174],{},[116,5140],{},[227,5142,242],{},[119,5144,5145,5147],{},[122,5146,607],{"mathvariant":606},[122,5148,127],{},[119,5150,5151,5154],{},[227,5152,5153],{},"∫",[122,5155,258],{},[122,5157,1475],{},[227,5159,230],{"stretchy":229},[122,5161,258],{},[122,5163,624],{"mathvariant":606},[122,5165,127],{},[227,5167,239],{"stretchy":229},[122,5169,273],{},[227,5171,230],{"stretchy":229},[122,5173,258],{},[227,5175,239],{"stretchy":229},[3593,5177,5178],{},[3596,5179,5180],{"scriptlevel":1229,"displaystyle":137},[116,5181],{},[3593,5183,5184],{},[3596,5185,5186],{"scriptlevel":1229,"displaystyle":137},[116,5187,5188,5190],{},[116,5189],{},[5191,5192,5193],"mtext",{},"展开期望",[3590,5195,5196,5202,5244,5250],{},[3593,5197,5198],{},[3596,5199,5200],{"scriptlevel":1229,"displaystyle":137},[116,5201],{},[3593,5203,5204],{},[3596,5205,5206],{"scriptlevel":1229,"displaystyle":137},[116,5207,5208,5210,5212,5218,5224,5226,5228,5230,5232,5234,5236,5238,5240,5242],{},[116,5209],{},[227,5211,242],{},[119,5213,5214,5216],{},[227,5215,5153],{},[122,5217,258],{},[119,5219,5220,5222],{},[122,5221,607],{"mathvariant":606},[122,5223,127],{},[122,5225,1475],{},[227,5227,230],{"stretchy":229},[122,5229,258],{},[122,5231,624],{"mathvariant":606},[122,5233,127],{},[227,5235,239],{"stretchy":229},[122,5237,273],{},[227,5239,230],{"stretchy":229},[122,5241,258],{},[227,5243,239],{"stretchy":229},[3593,5245,5246],{},[3596,5247,5248],{"scriptlevel":1229,"displaystyle":137},[116,5249],{},[3593,5251,5252],{},[3596,5253,5254],{"scriptlevel":1229,"displaystyle":137},[116,5255,5256,5258],{},[116,5257],{},[5191,5259,5260],{},"把梯度移入积分号内",[3590,5262,5263,5269,5327,5333],{},[3593,5264,5265],{},[3596,5266,5267],{"scriptlevel":1229,"displaystyle":137},[116,5268],{},[3593,5270,5271],{},[3596,5272,5273],{"scriptlevel":1229,"displaystyle":137},[116,5274,5275,5277,5279,5285,5287,5289,5291,5293,5295,5297,5303,5305,5307,5309,5311,5313,5315,5317,5319,5321,5323,5325],{},[116,5276],{},[227,5278,242],{},[119,5280,5281,5283],{},[227,5282,5153],{},[122,5284,258],{},[122,5286,1475],{},[227,5288,230],{"stretchy":229},[122,5290,258],{},[122,5292,624],{"mathvariant":606},[122,5294,127],{},[227,5296,239],{"stretchy":229},[119,5298,5299,5301],{},[122,5300,607],{"mathvariant":606},[122,5302,127],{},[122,5304,2093],{},[227,5306,2096],{},[122,5308,1475],{},[227,5310,230],{"stretchy":229},[122,5312,258],{},[122,5314,624],{"mathvariant":606},[122,5316,127],{},[227,5318,239],{"stretchy":229},[122,5320,273],{},[227,5322,230],{"stretchy":229},[122,5324,258],{},[227,5326,239],{"stretchy":229},[3593,5328,5329],{},[3596,5330,5331],{"scriptlevel":1229,"displaystyle":137},[116,5332],{},[3593,5334,5335],{},[3596,5336,5337],{"scriptlevel":1229,"displaystyle":137},[116,5338,5339,5341],{},[116,5340],{},[5191,5342,5343],{},"对数求导技巧",[3590,5345,5346,5352,5418,5424],{},[3593,5347,5348],{},[3596,5349,5350],{"scriptlevel":1229,"displaystyle":137},[116,5351],{},[3593,5353,5354],{},[3596,5355,5356],{"scriptlevel":1229,"displaystyle":137},[116,5357,5358,5360,5362,5382],{},[116,5359],{},[227,5361,242],{},[122,5363,5364],{},[246,5365,5366,5370],{},[227,5367,5368],{},[122,5369,253],{"mathvariant":252},[116,5371,5372,5374,5376],{},[122,5373,258],{},[227,5375,261],{},[119,5377,5378,5380],{},[122,5379,124],{},[122,5381,127],{},[116,5383,5384,5386,5392,5394,5396,5398,5400,5402,5404,5406,5408,5410,5412,5414,5416],{},[227,5385,270],{"fence":137},[119,5387,5388,5390],{},[122,5389,607],{"mathvariant":606},[122,5391,127],{},[122,5393,2093],{},[227,5395,2096],{},[122,5397,1475],{},[227,5399,230],{"stretchy":229},[122,5401,258],{},[122,5403,624],{"mathvariant":606},[122,5405,127],{},[227,5407,239],{"stretchy":229},[122,5409,273],{},[227,5411,230],{"stretchy":229},[122,5413,258],{},[227,5415,239],{"stretchy":229},[227,5417,282],{"fence":137},[3593,5419,5420],{},[3596,5421,5422],{"scriptlevel":1229,"displaystyle":137},[116,5423],{},[3593,5425,5426],{},[3596,5427,5428],{"scriptlevel":1229,"displaystyle":137},[116,5429,5430,5432],{},[116,5431],{},[5191,5433,5434],{},"回到期望形式",[3590,5436,5437,5464,5556,5562],{},[3593,5438,5439],{},[3596,5440,5441],{"scriptlevel":1229,"displaystyle":137},[116,5442,5443,5446,5452,5454,5456,5462],{},[227,5444,5445],{},"∴",[119,5447,5448,5450],{},[122,5449,607],{"mathvariant":606},[122,5451,127],{},[122,5453,225],{},[227,5455,230],{"stretchy":229},[119,5457,5458,5460],{},[122,5459,124],{},[122,5461,127],{},[227,5463,239],{"stretchy":229},[3593,5465,5466],{},[3596,5467,5468],{"scriptlevel":1229,"displaystyle":137},[116,5469,5470,5472,5474,5494],{},[116,5471],{},[227,5473,242],{},[122,5475,5476],{},[246,5477,5478,5482],{},[227,5479,5480],{},[122,5481,253],{"mathvariant":252},[116,5483,5484,5486,5488],{},[122,5485,258],{},[227,5487,261],{},[119,5489,5490,5492],{},[122,5491,124],{},[122,5493,127],{},[116,5495,5496,5498,5512,5518,5520,5522,5528,5530,5536,5538,5544,5546,5548,5550,5552,5554],{},[227,5497,270],{"fence":137},[1506,5499,5500,5502,5510],{},[227,5501,2508],{},[116,5503,5504,5506,5508],{},[122,5505,1515],{},[227,5507,242],{},[582,5509,1229],{},[122,5511,1258],{},[119,5513,5514,5516],{},[122,5515,607],{"mathvariant":606},[122,5517,127],{},[122,5519,2093],{},[227,5521,2096],{},[119,5523,5524,5526],{},[122,5525,124],{},[122,5527,127],{},[227,5529,230],{"stretchy":229},[119,5531,5532,5534],{},[122,5533,18],{},[122,5535,1515],{},[122,5537,624],{"mathvariant":606},[119,5539,5540,5542],{},[122,5541,1226],{},[122,5543,1515],{},[227,5545,239],{"stretchy":229},[122,5547,273],{},[227,5549,230],{"stretchy":229},[122,5551,258],{},[227,5553,239],{"stretchy":229},[227,5555,282],{"fence":137},[3593,5557,5558],{},[3596,5559,5560],{"scriptlevel":1229,"displaystyle":137},[116,5561],{},[3593,5563,5564],{},[3596,5565,5566],{"scriptlevel":1229,"displaystyle":137},[116,5567,5568,5570],{},[116,5569],{},[5191,5571,5572],{},"代入轨迹的 grad-log-prob",[129,5574,5575],{"encoding":131},"\\begin{align*}\n\\nabla_{\\theta} J(\\pi_{\\theta}) &= \\nabla_{\\theta} \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}[R(\\tau)] && \\\\\n&= \\nabla_{\\theta} \\int_{\\tau} P(\\tau|\\theta) R(\\tau) && \\text{展开期望} \\\\\n&= \\int_{\\tau} \\nabla_{\\theta} P(\\tau|\\theta) R(\\tau) && \\text{把梯度移入积分号内} \\\\\n&= \\int_{\\tau} P(\\tau|\\theta) \\nabla_{\\theta} \\log P(\\tau|\\theta) R(\\tau) && \\text{对数求导技巧} \\\\\n&= \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\nabla_{\\theta} \\log P(\\tau|\\theta) R(\\tau)\\right] && \\text{回到期望形式} \\\\\n\\therefore \\nabla_{\\theta} J(\\pi_{\\theta}) &= \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t) R(\\tau)\\right] && \\text{代入轨迹的 grad-log-prob}\n\\end{align*}",[100,5577,5579],{"className":5578,"ariaHidden":137},[136],[100,5580,5582,5586],{"className":5581},[141],[100,5583],{"className":5584,"style":5585},[145],"height:14.703em;vertical-align:-7.1015em;",[100,5587,5589],{"className":5588},[150],[100,5590,5592,5868,7133,7138,7209],{"className":5591},[3584],[100,5593,5595],{"className":5594},[3856],[100,5596,5598,5859],{"className":5597},[163,164],[100,5599,5601,5856],{"className":5600},[168],[100,5602,5605,5709,5718,5727,5736,5745],{"className":5603,"style":5604},[172],"height:7.6015em;",[100,5606,5608,5611],{"style":5607},"top:-10.5898em;",[100,5609],{"className":5610,"style":3873},[180],[100,5612,5614,5657,5660,5663,5706],{"className":5613},[150],[100,5615,5617,5620],{"className":5616},[150],[100,5618,607],{"className":5619},[150],[100,5621,5623],{"className":5622},[159],[100,5624,5626,5649],{"className":5625},[163,164],[100,5627,5629,5646],{"className":5628},[168],[100,5630,5632],{"className":5631,"style":173},[172],[100,5633,5634,5637],{"style":809},[100,5635],{"className":5636,"style":181},[180],[100,5638,5640],{"className":5639},[185,186,187,188],[100,5641,5643],{"className":5642},[150,188],[100,5644,127],{"className":5645,"style":195},[150,154,188],[100,5647,200],{"className":5648},[199],[100,5650,5652],{"className":5651},[168],[100,5653,5655],{"className":5654,"style":207},[172],[100,5656],{},[100,5658,225],{"className":5659,"style":299},[150,154],[100,5661,230],{"className":5662},[303],[100,5664,5666,5669],{"className":5665},[150],[100,5667,124],{"className":5668,"style":155},[150,154],[100,5670,5672],{"className":5671},[159],[100,5673,5675,5698],{"className":5674},[163,164],[100,5676,5678,5695],{"className":5677},[168],[100,5679,5681],{"className":5680,"style":173},[172],[100,5682,5683,5686],{"style":176},[100,5684],{"className":5685,"style":181},[180],[100,5687,5689],{"className":5688},[185,186,187,188],[100,5690,5692],{"className":5691},[150,188],[100,5693,127],{"className":5694,"style":195},[150,154,188],[100,5696,200],{"className":5697},[199],[100,5699,5701],{"className":5700},[168],[100,5702,5704],{"className":5703,"style":207},[172],[100,5705],{},[100,5707,239],{"className":5708},[350],[100,5710,5712,5715],{"style":5711},"top:-8.124em;",[100,5713],{"className":5714,"style":3873},[180],[100,5716],{"className":5717},[150],[100,5719,5721,5724],{"style":5720},"top:-5.552em;",[100,5722],{"className":5723,"style":3873},[180],[100,5725],{"className":5726},[150],[100,5728,5730,5733],{"style":5729},"top:-2.9801em;",[100,5731],{"className":5732,"style":3873},[180],[100,5734],{"className":5735},[150],[100,5737,5739,5742],{"style":5738},"top:-0.9281em;",[100,5740],{"className":5741,"style":3873},[180],[100,5743],{"className":5744},[150],[100,5746,5748,5751],{"style":5747},"top:2.0061em;",[100,5749],{"className":5750,"style":3873},[180],[100,5752,5754,5758,5761,5804,5807,5810,5853],{"className":5753},[150],[100,5755,5445],{"className":5756},[359,5757],"amsrm",[100,5759],{"className":5760,"style":355},[354],[100,5762,5764,5767],{"className":5763},[150],[100,5765,607],{"className":5766},[150],[100,5768,5770],{"className":5769},[159],[100,5771,5773,5796],{"className":5772},[163,164],[100,5774,5776,5793],{"className":5775},[168],[100,5777,5779],{"className":5778,"style":173},[172],[100,5780,5781,5784],{"style":809},[100,5782],{"className":5783,"style":181},[180],[100,5785,5787],{"className":5786},[185,186,187,188],[100,5788,5790],{"className":5789},[150,188],[100,5791,127],{"className":5792,"style":195},[150,154,188],[100,5794,200],{"className":5795},[199],[100,5797,5799],{"className":5798},[168],[100,5800,5802],{"className":5801,"style":207},[172],[100,5803],{},[100,5805,225],{"className":5806,"style":299},[150,154],[100,5808,230],{"className":5809},[303],[100,5811,5813,5816],{"className":5812},[150],[100,5814,124],{"className":5815,"style":155},[150,154],[100,5817,5819],{"className":5818},[159],[100,5820,5822,5845],{"className":5821},[163,164],[100,5823,5825,5842],{"className":5824},[168],[100,5826,5828],{"className":5827,"style":173},[172],[100,5829,5830,5833],{"style":176},[100,5831],{"className":5832,"style":181},[180],[100,5834,5836],{"className":5835},[185,186,187,188],[100,5837,5839],{"className":5838},[150,188],[100,5840,127],{"className":5841,"style":195},[150,154,188],[100,5843,200],{"className":5844},[199],[100,5846,5848],{"className":5847},[168],[100,5849,5851],{"className":5850,"style":207},[172],[100,5852],{},[100,5854,239],{"className":5855},[350],[100,5857,200],{"className":5858},[199],[100,5860,5862],{"className":5861},[168],[100,5863,5866],{"className":5864,"style":5865},[172],"height:7.1015em;",[100,5867],{},[100,5869,5871],{"className":5870},[3973],[100,5872,5874,7125],{"className":5873},[163,164],[100,5875,5877,7122],{"className":5876},[168],[100,5878,5880,6054,6200,6339,6507,6719],{"className":5879,"style":5604},[172],[100,5881,5882,5885],{"style":5607},[100,5883],{"className":5884,"style":3873},[180],[100,5886,5888,5891,5894,5897,5900,5943,6039,6042,6045,6048,6051],{"className":5887},[150],[100,5889],{"className":5890},[150],[100,5892],{"className":5893,"style":355},[354],[100,5895,242],{"className":5896},[359],[100,5898],{"className":5899,"style":355},[354],[100,5901,5903,5906],{"className":5902},[150],[100,5904,607],{"className":5905},[150],[100,5907,5909],{"className":5908},[159],[100,5910,5912,5935],{"className":5911},[163,164],[100,5913,5915,5932],{"className":5914},[168],[100,5916,5918],{"className":5917,"style":173},[172],[100,5919,5920,5923],{"style":809},[100,5921],{"className":5922,"style":181},[180],[100,5924,5926],{"className":5925},[185,186,187,188],[100,5927,5929],{"className":5928},[150,188],[100,5930,127],{"className":5931,"style":195},[150,154,188],[100,5933,200],{"className":5934},[199],[100,5936,5938],{"className":5937},[168],[100,5939,5941],{"className":5940,"style":207},[172],[100,5942],{},[100,5944,5946],{"className":5945},[150],[100,5947,5949],{"className":5948},[376,377],[100,5950,5952,6031],{"className":5951},[163,164],[100,5953,5955,6028],{"className":5954},[168],[100,5956,5958,6018],{"className":5957,"style":387},[172],[100,5959,5960,5963],{"style":390},[100,5961],{"className":5962,"style":394},[180],[100,5964,5966],{"className":5965},[185,186,187,188],[100,5967,5969,5972,5975],{"className":5968},[150,188],[100,5970,258],{"className":5971,"style":404},[150,154,188],[100,5973,261],{"className":5974},[359,188],[100,5976,5978,5981],{"className":5977},[150,188],[100,5979,124],{"className":5980,"style":155},[150,154,188],[100,5982,5984],{"className":5983},[159],[100,5985,5987,6010],{"className":5986},[163,164],[100,5988,5990,6007],{"className":5989},[168],[100,5991,5993],{"className":5992,"style":426},[172],[100,5994,5995,5998],{"style":429},[100,5996],{"className":5997,"style":433},[180],[100,5999,6001],{"className":6000},[185,437,438,188],[100,6002,6004],{"className":6003},[150,188],[100,6005,127],{"className":6006,"style":195},[150,154,188],[100,6008,200],{"className":6009},[199],[100,6011,6013],{"className":6012},[168],[100,6014,6016],{"className":6015,"style":454},[172],[100,6017],{},[100,6019,6020,6023],{"style":459},[100,6021],{"className":6022,"style":394},[180],[100,6024,6025],{},[100,6026,253],{"className":6027},[376,468],[100,6029,200],{"className":6030},[199],[100,6032,6034],{"className":6033},[168],[100,6035,6037],{"className":6036,"style":478},[172],[100,6038],{},[100,6040,270],{"className":6041},[303],[100,6043,273],{"className":6044,"style":487},[150,154],[100,6046,230],{"className":6047},[303],[100,6049,258],{"className":6050,"style":404},[150,154],[100,6052,497],{"className":6053},[350],[100,6055,6056,6059],{"style":5711},[100,6057],{"className":6058,"style":3873},[180],[100,6060,6062,6065,6068,6071,6074,6117,6120,6167,6170,6173,6176,6179,6182,6185,6188,6191,6194,6197],{"className":6061},[150],[100,6063],{"className":6064},[150],[100,6066],{"className":6067,"style":355},[354],[100,6069,242],{"className":6070},[359],[100,6072],{"className":6073,"style":355},[354],[100,6075,6077,6080],{"className":6076},[150],[100,6078,607],{"className":6079},[150],[100,6081,6083],{"className":6082},[159],[100,6084,6086,6109],{"className":6085},[163,164],[100,6087,6089,6106],{"className":6088},[168],[100,6090,6092],{"className":6091,"style":173},[172],[100,6093,6094,6097],{"style":809},[100,6095],{"className":6096,"style":181},[180],[100,6098,6100],{"className":6099},[185,186,187,188],[100,6101,6103],{"className":6102},[150,188],[100,6104,127],{"className":6105,"style":195},[150,154,188],[100,6107,200],{"className":6108},[199],[100,6110,6112],{"className":6111},[168],[100,6113,6115],{"className":6114,"style":207},[172],[100,6116],{},[100,6118],{"className":6119,"style":777},[354],[100,6121,6123,6127],{"className":6122},[376],[100,6124,5153],{"className":6125,"style":6126},[376,1761,1762],"margin-right:0.4445em;position:relative;top:-0.0011em;",[100,6128,6130],{"className":6129},[159],[100,6131,6133,6158],{"className":6132},[163,164],[100,6134,6136,6155],{"className":6135},[168],[100,6137,6140],{"className":6138,"style":6139},[172],"height:-0.6106em;",[100,6141,6143,6146],{"style":6142},"top:-1.7881em;margin-left:-0.4445em;margin-right:0.05em;",[100,6144],{"className":6145,"style":181},[180],[100,6147,6149],{"className":6148},[185,186,187,188],[100,6150,6152],{"className":6151},[150,188],[100,6153,258],{"className":6154,"style":404},[150,154,188],[100,6156,200],{"className":6157},[199],[100,6159,6161],{"className":6160},[168],[100,6162,6165],{"className":6163,"style":6164},[172],"height:0.9119em;",[100,6166],{},[100,6168],{"className":6169,"style":777},[354],[100,6171,1475],{"className":6172,"style":1436},[150,154],[100,6174,230],{"className":6175},[303],[100,6177,258],{"className":6178,"style":404},[150,154],[100,6180,624],{"className":6181},[150],[100,6183,127],{"className":6184,"style":195},[150,154],[100,6186,239],{"className":6187},[350],[100,6189,273],{"className":6190,"style":487},[150,154],[100,6192,230],{"className":6193},[303],[100,6195,258],{"className":6196,"style":404},[150,154],[100,6198,239],{"className":6199},[350],[100,6201,6202,6205],{"style":5720},[100,6203],{"className":6204,"style":3873},[180],[100,6206,6208,6211,6214,6217,6220,6263,6266,6309,6312,6315,6318,6321,6324,6327,6330,6333,6336],{"className":6207},[150],[100,6209],{"className":6210},[150],[100,6212],{"className":6213,"style":355},[354],[100,6215,242],{"className":6216},[359],[100,6218],{"className":6219,"style":355},[354],[100,6221,6223,6226],{"className":6222},[376],[100,6224,5153],{"className":6225,"style":6126},[376,1761,1762],[100,6227,6229],{"className":6228},[159],[100,6230,6232,6255],{"className":6231},[163,164],[100,6233,6235,6252],{"className":6234},[168],[100,6236,6238],{"className":6237,"style":6139},[172],[100,6239,6240,6243],{"style":6142},[100,6241],{"className":6242,"style":181},[180],[100,6244,6246],{"className":6245},[185,186,187,188],[100,6247,6249],{"className":6248},[150,188],[100,6250,258],{"className":6251,"style":404},[150,154,188],[100,6253,200],{"className":6254},[199],[100,6256,6258],{"className":6257},[168],[100,6259,6261],{"className":6260,"style":6164},[172],[100,6262],{},[100,6264],{"className":6265,"style":777},[354],[100,6267,6269,6272],{"className":6268},[150],[100,6270,607],{"className":6271},[150],[100,6273,6275],{"className":6274},[159],[100,6276,6278,6301],{"className":6277},[163,164],[100,6279,6281,6298],{"className":6280},[168],[100,6282,6284],{"className":6283,"style":173},[172],[100,6285,6286,6289],{"style":809},[100,6287],{"className":6288,"style":181},[180],[100,6290,6292],{"className":6291},[185,186,187,188],[100,6293,6295],{"className":6294},[150,188],[100,6296,127],{"className":6297,"style":195},[150,154,188],[100,6299,200],{"className":6300},[199],[100,6302,6304],{"className":6303},[168],[100,6305,6307],{"className":6306,"style":207},[172],[100,6308],{},[100,6310,1475],{"className":6311,"style":1436},[150,154],[100,6313,230],{"className":6314},[303],[100,6316,258],{"className":6317,"style":404},[150,154],[100,6319,624],{"className":6320},[150],[100,6322,127],{"className":6323,"style":195},[150,154],[100,6325,239],{"className":6326},[350],[100,6328,273],{"className":6329,"style":487},[150,154],[100,6331,230],{"className":6332},[303],[100,6334,258],{"className":6335,"style":404},[150,154],[100,6337,239],{"className":6338},[350],[100,6340,6341,6344],{"style":5729},[100,6342],{"className":6343,"style":3873},[180],[100,6345,6347,6350,6353,6356,6359,6402,6405,6408,6411,6414,6417,6420,6423,6466,6469,6474,6477,6480,6483,6486,6489,6492,6495,6498,6501,6504],{"className":6346},[150],[100,6348],{"className":6349},[150],[100,6351],{"className":6352,"style":355},[354],[100,6354,242],{"className":6355},[359],[100,6357],{"className":6358,"style":355},[354],[100,6360,6362,6365],{"className":6361},[376],[100,6363,5153],{"className":6364,"style":6126},[376,1761,1762],[100,6366,6368],{"className":6367},[159],[100,6369,6371,6394],{"className":6370},[163,164],[100,6372,6374,6391],{"className":6373},[168],[100,6375,6377],{"className":6376,"style":6139},[172],[100,6378,6379,6382],{"style":6142},[100,6380],{"className":6381,"style":181},[180],[100,6383,6385],{"className":6384},[185,186,187,188],[100,6386,6388],{"className":6387},[150,188],[100,6389,258],{"className":6390,"style":404},[150,154,188],[100,6392,200],{"className":6393},[199],[100,6395,6397],{"className":6396},[168],[100,6398,6400],{"className":6399,"style":6164},[172],[100,6401],{},[100,6403],{"className":6404,"style":777},[354],[100,6406,1475],{"className":6407,"style":1436},[150,154],[100,6409,230],{"className":6410},[303],[100,6412,258],{"className":6413,"style":404},[150,154],[100,6415,624],{"className":6416},[150],[100,6418,127],{"className":6419,"style":195},[150,154],[100,6421,239],{"className":6422},[350],[100,6424,6426,6429],{"className":6425},[150],[100,6427,607],{"className":6428},[150],[100,6430,6432],{"className":6431},[159],[100,6433,6435,6458],{"className":6434},[163,164],[100,6436,6438,6455],{"className":6437},[168],[100,6439,6441],{"className":6440,"style":173},[172],[100,6442,6443,6446],{"style":809},[100,6444],{"className":6445,"style":181},[180],[100,6447,6449],{"className":6448},[185,186,187,188],[100,6450,6452],{"className":6451},[150,188],[100,6453,127],{"className":6454,"style":195},[150,154,188],[100,6456,200],{"className":6457},[199],[100,6459,6461],{"className":6460},[168],[100,6462,6464],{"className":6463,"style":207},[172],[100,6465],{},[100,6467],{"className":6468,"style":777},[354],[100,6470,2116,6472],{"className":6471},[376],[100,6473,2120],{"style":2119},[100,6475],{"className":6476,"style":777},[354],[100,6478,1475],{"className":6479,"style":1436},[150,154],[100,6481,230],{"className":6482},[303],[100,6484,258],{"className":6485,"style":404},[150,154],[100,6487,624],{"className":6488},[150],[100,6490,127],{"className":6491,"style":195},[150,154],[100,6493,239],{"className":6494},[350],[100,6496,273],{"className":6497,"style":487},[150,154],[100,6499,230],{"className":6500},[303],[100,6502,258],{"className":6503,"style":404},[150,154],[100,6505,239],{"className":6506},[350],[100,6508,6509,6512],{"style":5738},[100,6510],{"className":6511,"style":3873},[180],[100,6513,6515,6518,6521,6524,6527,6623,6626],{"className":6514},[150],[100,6516],{"className":6517},[150],[100,6519],{"className":6520,"style":355},[354],[100,6522,242],{"className":6523},[359],[100,6525],{"className":6526,"style":355},[354],[100,6528,6530],{"className":6529},[150],[100,6531,6533],{"className":6532},[376,377],[100,6534,6536,6615],{"className":6535},[163,164],[100,6537,6539,6612],{"className":6538},[168],[100,6540,6542,6602],{"className":6541,"style":387},[172],[100,6543,6544,6547],{"style":390},[100,6545],{"className":6546,"style":394},[180],[100,6548,6550],{"className":6549},[185,186,187,188],[100,6551,6553,6556,6559],{"className":6552},[150,188],[100,6554,258],{"className":6555,"style":404},[150,154,188],[100,6557,261],{"className":6558},[359,188],[100,6560,6562,6565],{"className":6561},[150,188],[100,6563,124],{"className":6564,"style":155},[150,154,188],[100,6566,6568],{"className":6567},[159],[100,6569,6571,6594],{"className":6570},[163,164],[100,6572,6574,6591],{"className":6573},[168],[100,6575,6577],{"className":6576,"style":426},[172],[100,6578,6579,6582],{"style":429},[100,6580],{"className":6581,"style":433},[180],[100,6583,6585],{"className":6584},[185,437,438,188],[100,6586,6588],{"className":6587},[150,188],[100,6589,127],{"className":6590,"style":195},[150,154,188],[100,6592,200],{"className":6593},[199],[100,6595,6597],{"className":6596},[168],[100,6598,6600],{"className":6599,"style":454},[172],[100,6601],{},[100,6603,6604,6607],{"style":459},[100,6605],{"className":6606,"style":394},[180],[100,6608,6609],{},[100,6610,253],{"className":6611},[376,468],[100,6613,200],{"className":6614},[199],[100,6616,6618],{"className":6617},[168],[100,6619,6621],{"className":6620,"style":478},[172],[100,6622],{},[100,6624],{"className":6625,"style":777},[354],[100,6627,6629,6632,6675,6678,6683,6686,6689,6692,6695,6698,6701,6704,6707,6710,6713,6716],{"className":6628},[781],[100,6630,270],{"className":6631,"style":889},[303,888],[100,6633,6635,6638],{"className":6634},[150],[100,6636,607],{"className":6637},[150],[100,6639,6641],{"className":6640},[159],[100,6642,6644,6667],{"className":6643},[163,164],[100,6645,6647,6664],{"className":6646},[168],[100,6648,6650],{"className":6649,"style":173},[172],[100,6651,6652,6655],{"style":809},[100,6653],{"className":6654,"style":181},[180],[100,6656,6658],{"className":6657},[185,186,187,188],[100,6659,6661],{"className":6660},[150,188],[100,6662,127],{"className":6663,"style":195},[150,154,188],[100,6665,200],{"className":6666},[199],[100,6668,6670],{"className":6669},[168],[100,6671,6673],{"className":6672,"style":207},[172],[100,6674],{},[100,6676],{"className":6677,"style":777},[354],[100,6679,2116,6681],{"className":6680},[376],[100,6682,2120],{"style":2119},[100,6684],{"className":6685,"style":777},[354],[100,6687,1475],{"className":6688,"style":1436},[150,154],[100,6690,230],{"className":6691},[303],[100,6693,258],{"className":6694,"style":404},[150,154],[100,6696,624],{"className":6697},[150],[100,6699,127],{"className":6700,"style":195},[150,154],[100,6702,239],{"className":6703},[350],[100,6705,273],{"className":6706,"style":487},[150,154],[100,6708,230],{"className":6709},[303],[100,6711,258],{"className":6712,"style":404},[150,154],[100,6714,239],{"className":6715},[350],[100,6717,282],{"className":6718,"style":889},[350,888],[100,6720,6721,6724],{"style":5747},[100,6722],{"className":6723,"style":3873},[180],[100,6725,6727,6730,6733,6736,6739,6835,6838],{"className":6726},[150],[100,6728],{"className":6729},[150],[100,6731],{"className":6732,"style":355},[354],[100,6734,242],{"className":6735},[359],[100,6737],{"className":6738,"style":355},[354],[100,6740,6742],{"className":6741},[150],[100,6743,6745],{"className":6744},[376,377],[100,6746,6748,6827],{"className":6747},[163,164],[100,6749,6751,6824],{"className":6750},[168],[100,6752,6754,6814],{"className":6753,"style":387},[172],[100,6755,6756,6759],{"style":390},[100,6757],{"className":6758,"style":394},[180],[100,6760,6762],{"className":6761},[185,186,187,188],[100,6763,6765,6768,6771],{"className":6764},[150,188],[100,6766,258],{"className":6767,"style":404},[150,154,188],[100,6769,261],{"className":6770},[359,188],[100,6772,6774,6777],{"className":6773},[150,188],[100,6775,124],{"className":6776,"style":155},[150,154,188],[100,6778,6780],{"className":6779},[159],[100,6781,6783,6806],{"className":6782},[163,164],[100,6784,6786,6803],{"className":6785},[168],[100,6787,6789],{"className":6788,"style":426},[172],[100,6790,6791,6794],{"style":429},[100,6792],{"className":6793,"style":433},[180],[100,6795,6797],{"className":6796},[185,437,438,188],[100,6798,6800],{"className":6799},[150,188],[100,6801,127],{"className":6802,"style":195},[150,154,188],[100,6804,200],{"className":6805},[199],[100,6807,6809],{"className":6808},[168],[100,6810,6812],{"className":6811,"style":454},[172],[100,6813],{},[100,6815,6816,6819],{"style":459},[100,6817],{"className":6818,"style":394},[180],[100,6820,6821],{},[100,6822,253],{"className":6823},[376,468],[100,6825,200],{"className":6826},[199],[100,6828,6830],{"className":6829},[168],[100,6831,6833],{"className":6832,"style":478},[172],[100,6834],{},[100,6836],{"className":6837,"style":777},[354],[100,6839,6841,6848,6915,6918,6961,6964,6969,6972,7015,7018,7058,7061,7101,7104,7107,7110,7113,7116],{"className":6840},[781],[100,6842,6844],{"className":6843,"style":889},[303,888],[100,6845,270],{"className":6846},[2830,6847],"size4",[100,6849,6851],{"className":6850},[376,377],[100,6852,6854,6907],{"className":6853},[163,164],[100,6855,6857,6904],{"className":6856},[168],[100,6858,6860,6880,6890],{"className":6859,"style":1727},[172],[100,6861,6862,6865],{"style":1730},[100,6863],{"className":6864,"style":1734},[180],[100,6866,6868],{"className":6867},[185,186,187,188],[100,6869,6871,6874,6877],{"className":6870},[150,188],[100,6872,1515],{"className":6873},[150,154,188],[100,6875,242],{"className":6876},[359,188],[100,6878,1229],{"className":6879},[150,188],[100,6881,6882,6885],{"style":1752},[100,6883],{"className":6884,"style":1734},[180],[100,6886,6887],{},[100,6888,2508],{"className":6889},[376,1761,1762],[100,6891,6892,6895],{"style":1765},[100,6893],{"className":6894,"style":1734},[180],[100,6896,6898],{"className":6897},[185,186,187,188],[100,6899,6901],{"className":6900},[150,188],[100,6902,1258],{"className":6903,"style":1436},[150,154,188],[100,6905,200],{"className":6906},[199],[100,6908,6910],{"className":6909},[168],[100,6911,6913],{"className":6912,"style":1787},[172],[100,6914],{},[100,6916],{"className":6917,"style":777},[354],[100,6919,6921,6924],{"className":6920},[150],[100,6922,607],{"className":6923},[150],[100,6925,6927],{"className":6926},[159],[100,6928,6930,6953],{"className":6929},[163,164],[100,6931,6933,6950],{"className":6932},[168],[100,6934,6936],{"className":6935,"style":173},[172],[100,6937,6938,6941],{"style":809},[100,6939],{"className":6940,"style":181},[180],[100,6942,6944],{"className":6943},[185,186,187,188],[100,6945,6947],{"className":6946},[150,188],[100,6948,127],{"className":6949,"style":195},[150,154,188],[100,6951,200],{"className":6952},[199],[100,6954,6956],{"className":6955},[168],[100,6957,6959],{"className":6958,"style":207},[172],[100,6960],{},[100,6962],{"className":6963,"style":777},[354],[100,6965,2116,6967],{"className":6966},[376],[100,6968,2120],{"style":2119},[100,6970],{"className":6971,"style":777},[354],[100,6973,6975,6978],{"className":6974},[150],[100,6976,124],{"className":6977,"style":155},[150,154],[100,6979,6981],{"className":6980},[159],[100,6982,6984,7007],{"className":6983},[163,164],[100,6985,6987,7004],{"className":6986},[168],[100,6988,6990],{"className":6989,"style":173},[172],[100,6991,6992,6995],{"style":176},[100,6993],{"className":6994,"style":181},[180],[100,6996,6998],{"className":6997},[185,186,187,188],[100,6999,7001],{"className":7000},[150,188],[100,7002,127],{"className":7003,"style":195},[150,154,188],[100,7005,200],{"className":7006},[199],[100,7008,7010],{"className":7009},[168],[100,7011,7013],{"className":7012,"style":207},[172],[100,7014],{},[100,7016,230],{"className":7017},[303],[100,7019,7021,7024],{"className":7020},[150],[100,7022,18],{"className":7023},[150,154],[100,7025,7027],{"className":7026},[159],[100,7028,7030,7050],{"className":7029},[163,164],[100,7031,7033,7047],{"className":7032},[168],[100,7034,7036],{"className":7035,"style":1869},[172],[100,7037,7038,7041],{"style":809},[100,7039],{"className":7040,"style":181},[180],[100,7042,7044],{"className":7043},[185,186,187,188],[100,7045,1515],{"className":7046},[150,154,188],[100,7048,200],{"className":7049},[199],[100,7051,7053],{"className":7052},[168],[100,7054,7056],{"className":7055,"style":207},[172],[100,7057],{},[100,7059,624],{"className":7060},[150],[100,7062,7064,7067],{"className":7063},[150],[100,7065,1226],{"className":7066},[150,154],[100,7068,7070],{"className":7069},[159],[100,7071,7073,7093],{"className":7072},[163,164],[100,7074,7076,7090],{"className":7075},[168],[100,7077,7079],{"className":7078,"style":1869},[172],[100,7080,7081,7084],{"style":809},[100,7082],{"className":7083,"style":181},[180],[100,7085,7087],{"className":7086},[185,186,187,188],[100,7088,1515],{"className":7089},[150,154,188],[100,7091,200],{"className":7092},[199],[100,7094,7096],{"className":7095},[168],[100,7097,7099],{"className":7098,"style":207},[172],[100,7100],{},[100,7102,239],{"className":7103},[350],[100,7105,273],{"className":7106,"style":487},[150,154],[100,7108,230],{"className":7109},[303],[100,7111,258],{"className":7112,"style":404},[150,154],[100,7114,239],{"className":7115},[350],[100,7117,7119],{"className":7118,"style":889},[350,888],[100,7120,282],{"className":7121},[2830,6847],[100,7123,200],{"className":7124},[199],[100,7126,7128],{"className":7127},[168],[100,7129,7131],{"className":7130,"style":5865},[172],[100,7132],{},[100,7134],{"className":7135,"style":7137},[7136],"arraycolsep","width:1em;",[100,7139,7141],{"className":7140},[3856],[100,7142,7144,7201],{"className":7143},[163,164],[100,7145,7147,7198],{"className":7146},[168],[100,7148,7150,7158,7166,7174,7182,7190],{"className":7149,"style":5604},[172],[100,7151,7152,7155],{"style":5607},[100,7153],{"className":7154,"style":3873},[180],[100,7156],{"className":7157},[150],[100,7159,7160,7163],{"style":5711},[100,7161],{"className":7162,"style":3873},[180],[100,7164],{"className":7165},[150],[100,7167,7168,7171],{"style":5720},[100,7169],{"className":7170,"style":3873},[180],[100,7172],{"className":7173},[150],[100,7175,7176,7179],{"style":5729},[100,7177],{"className":7178,"style":3873},[180],[100,7180],{"className":7181},[150],[100,7183,7184,7187],{"style":5738},[100,7185],{"className":7186,"style":3873},[180],[100,7188],{"className":7189},[150],[100,7191,7192,7195],{"style":5747},[100,7193],{"className":7194,"style":3873},[180],[100,7196],{"className":7197},[150],[100,7199,200],{"className":7200},[199],[100,7202,7204],{"className":7203},[168],[100,7205,7207],{"className":7206,"style":5865},[172],[100,7208],{},[100,7210,7212],{"className":7211},[3973],[100,7213,7215,7327],{"className":7214},[163,164],[100,7216,7218,7324],{"className":7217},[168],[100,7219,7221,7232,7251,7268,7285,7302],{"className":7220,"style":5604},[172],[100,7222,7223,7226],{"style":5607},[100,7224],{"className":7225,"style":3873},[180],[100,7227,7229],{"className":7228},[150],[100,7230],{"className":7231},[150],[100,7233,7234,7237],{"style":5711},[100,7235],{"className":7236,"style":3873},[180],[100,7238,7240,7243],{"className":7239},[150],[100,7241],{"className":7242},[150],[100,7244,7247],{"className":7245},[150,7246],"text",[100,7248,5193],{"className":7249},[150,7250],"cjk_fallback",[100,7252,7253,7256],{"style":5720},[100,7254],{"className":7255,"style":3873},[180],[100,7257,7259,7262],{"className":7258},[150],[100,7260],{"className":7261},[150],[100,7263,7265],{"className":7264},[150,7246],[100,7266,5260],{"className":7267},[150,7250],[100,7269,7270,7273],{"style":5729},[100,7271],{"className":7272,"style":3873},[180],[100,7274,7276,7279],{"className":7275},[150],[100,7277],{"className":7278},[150],[100,7280,7282],{"className":7281},[150,7246],[100,7283,5343],{"className":7284},[150,7250],[100,7286,7287,7290],{"style":5738},[100,7288],{"className":7289,"style":3873},[180],[100,7291,7293,7296],{"className":7292},[150],[100,7294],{"className":7295},[150],[100,7297,7299],{"className":7298},[150,7246],[100,7300,5434],{"className":7301},[150,7250],[100,7303,7304,7307],{"style":5747},[100,7305],{"className":7306,"style":3873},[180],[100,7308,7310,7313],{"className":7309},[150],[100,7311],{"className":7312},[150],[100,7314,7316,7320],{"className":7315},[150,7246],[100,7317,7319],{"className":7318},[150,7250],"代入轨迹的",[100,7321,7323],{"className":7322},[150]," grad-log-prob",[100,7325,200],{"className":7326},[199],[100,7328,7330],{"className":7329},[168],[100,7331,7333],{"className":7332,"style":5865},[172],[100,7334],{},[14,7336,7337,7338,7533,7534,7606],{},"这是一个期望，意味着可以用样本均值来估计它。收集一组轨迹 ",[100,7339,7341,7395],{"className":7340},[103],[100,7342,7344],{"className":7343},[107],[109,7345,7346],{"xmlns":111},[113,7347,7348,7392],{},[116,7349,7350,7354,7356,7359,7366],{},[122,7351,7353],{"mathvariant":7352},"script","D",[227,7355,242],{},[227,7357,7358],{"stretchy":229},"{",[119,7360,7361,7363],{},[122,7362,258],{},[122,7364,7365],{},"i",[119,7367,7368,7371],{},[227,7369,7370],{"stretchy":229},"}",[116,7372,7373,7375,7377,7379,7381,7383,7385,7387,7389],{},[122,7374,7365],{},[227,7376,242],{},[582,7378,584],{},[227,7380,1232],{"separator":137},[122,7382,1243],{"mathvariant":606},[122,7384,1243],{"mathvariant":606},[122,7386,1243],{"mathvariant":606},[227,7388,1232],{"separator":137},[122,7390,7391],{},"N",[129,7393,7394],{"encoding":131},"\\mathcal{D} = \\{\\tau_i\\}_{i=1,...,N}",[100,7396,7398,7418],{"className":7397,"ariaHidden":137},[136],[100,7399,7401,7405,7409,7412,7415],{"className":7400},[141],[100,7402],{"className":7403,"style":7404},[145],"height:0.6833em;",[100,7406,7353],{"className":7407,"style":195},[150,7408],"mathcal",[100,7410],{"className":7411,"style":355},[354],[100,7413,242],{"className":7414},[359],[100,7416],{"className":7417,"style":355},[354],[100,7419,7421,7425,7428,7470],{"className":7420},[141],[100,7422],{"className":7423,"style":7424},[145],"height:1.0361em;vertical-align:-0.2861em;",[100,7426,7358],{"className":7427},[303],[100,7429,7431,7434],{"className":7430},[150],[100,7432,258],{"className":7433,"style":404},[150,154],[100,7435,7437],{"className":7436},[159],[100,7438,7440,7462],{"className":7439},[163,164],[100,7441,7443,7459],{"className":7442},[168],[100,7444,7447],{"className":7445,"style":7446},[172],"height:0.3117em;",[100,7448,7450,7453],{"style":7449},"top:-2.55em;margin-left:-0.1132em;margin-right:0.05em;",[100,7451],{"className":7452,"style":181},[180],[100,7454,7456],{"className":7455},[185,186,187,188],[100,7457,7365],{"className":7458},[150,154,188],[100,7460,200],{"className":7461},[199],[100,7463,7465],{"className":7464},[168],[100,7466,7468],{"className":7467,"style":207},[172],[100,7469],{},[100,7471,7473,7476],{"className":7472},[350],[100,7474,7370],{"className":7475},[350],[100,7477,7479],{"className":7478},[159],[100,7480,7482,7524],{"className":7481},[163,164],[100,7483,7485,7521],{"className":7484},[168],[100,7486,7488],{"className":7487,"style":1421},[172],[100,7489,7490,7493],{"style":809},[100,7491],{"className":7492,"style":181},[180],[100,7494,7496],{"className":7495},[185,186,187,188],[100,7497,7499,7502,7505,7508,7511,7514,7517],{"className":7498},[150,188],[100,7500,7365],{"className":7501},[150,154,188],[100,7503,242],{"className":7504},[359,188],[100,7506,584],{"className":7507},[150,188],[100,7509,1232],{"className":7510},[1343,188],[100,7512,1396],{"className":7513},[150,188],[100,7515,1232],{"className":7516},[1343,188],[100,7518,7391],{"className":7519,"style":7520},[150,154,188],"margin-right:0.109em;",[100,7522,200],{"className":7523},[199],[100,7525,7527],{"className":7526},[168],[100,7528,7531],{"className":7529,"style":7530},[172],"height:0.2861em;",[100,7532],{},"，每条轨迹都让智能体按策略 ",[100,7535,7537,7554],{"className":7536},[103],[100,7538,7540],{"className":7539},[107],[109,7541,7542],{"xmlns":111},[113,7543,7544,7552],{},[116,7545,7546],{},[119,7547,7548,7550],{},[122,7549,124],{},[122,7551,127],{},[129,7553,132],{"encoding":131},[100,7555,7557],{"className":7556,"ariaHidden":137},[136],[100,7558,7560,7563],{"className":7559},[141],[100,7561],{"className":7562,"style":146},[145],[100,7564,7566,7569],{"className":7565},[150],[100,7567,124],{"className":7568,"style":155},[150,154],[100,7570,7572],{"className":7571},[159],[100,7573,7575,7598],{"className":7574},[163,164],[100,7576,7578,7595],{"className":7577},[168],[100,7579,7581],{"className":7580,"style":173},[172],[100,7582,7583,7586],{"style":176},[100,7584],{"className":7585,"style":181},[180],[100,7587,7589],{"className":7588},[185,186,187,188],[100,7590,7592],{"className":7591},[150,188],[100,7593,127],{"className":7594,"style":195},[150,154,188],[100,7596,200],{"className":7597},[199],[100,7599,7601],{"className":7600},[168],[100,7602,7604],{"className":7603,"style":207},[172],[100,7605],{}," 在环境中行动得到，策略梯度就可以这样估计：",[100,7608,7610],{"className":7609},[555],[100,7611,7613,7717],{"className":7612},[103],[100,7614,7616],{"className":7615},[107],[109,7617,7618],{"xmlns":111,"display":564},[113,7619,7620,7714],{},[116,7621,7622,7630,7632,7645,7658,7672,7678,7680,7682,7688,7690,7696,7698,7704,7706,7708,7710,7712],{},[7623,7624,7625,7627],"mover",{"accent":137},[122,7626,2120],{},[227,7628,7629],{},"^",[227,7631,242],{},[7633,7634,7635,7637],"mfrac",{},[582,7636,584],{},[116,7638,7639,7641,7643],{},[122,7640,624],{"mathvariant":606},[122,7642,7353],{"mathvariant":7352},[122,7644,624],{"mathvariant":606},[246,7646,7647,7649],{},[227,7648,2508],{},[116,7650,7651,7653,7656],{},[122,7652,258],{},[227,7654,7655],{},"∈",[122,7657,7353],{"mathvariant":7352},[1506,7659,7660,7662,7670],{},[227,7661,2508],{},[116,7663,7664,7666,7668],{},[122,7665,1515],{},[227,7667,242],{},[582,7669,1229],{},[122,7671,1258],{},[119,7673,7674,7676],{},[122,7675,607],{"mathvariant":606},[122,7677,127],{},[122,7679,2093],{},[227,7681,2096],{},[119,7683,7684,7686],{},[122,7685,124],{},[122,7687,127],{},[227,7689,230],{"stretchy":229},[119,7691,7692,7694],{},[122,7693,18],{},[122,7695,1515],{},[122,7697,624],{"mathvariant":606},[119,7699,7700,7702],{},[122,7701,1226],{},[122,7703,1515],{},[227,7705,239],{"stretchy":229},[122,7707,273],{},[227,7709,230],{"stretchy":229},[122,7711,258],{},[227,7713,239],{"stretchy":229},[129,7715,7716],{"encoding":131},"\\hat{g} = \\frac{1}{|\\mathcal{D}|} \\sum_{\\tau \\in \\mathcal{D}} \\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t) R(\\tau)",[100,7718,7720,7781],{"className":7719,"ariaHidden":137},[136],[100,7721,7723,7726,7772,7775,7778],{"className":7722},[141],[100,7724],{"className":7725,"style":2112},[145],[100,7727,7730],{"className":7728},[150,7729],"accent",[100,7731,7733,7763],{"className":7732},[163,164],[100,7734,7736,7760],{"className":7735},[168],[100,7737,7739,7747],{"className":7738,"style":3183},[172],[100,7740,7741,7744],{"style":459},[100,7742],{"className":7743,"style":394},[180],[100,7745,2120],{"className":7746,"style":155},[150,154],[100,7748,7749,7752],{"style":459},[100,7750],{"className":7751,"style":394},[180],[100,7753,7757],{"className":7754,"style":7756},[7755],"accent-body","left:-0.2222em;",[100,7758,7629],{"className":7759},[150],[100,7761,200],{"className":7762},[199],[100,7764,7766],{"className":7765},[168],[100,7767,7770],{"className":7768,"style":7769},[172],"height:0.1944em;",[100,7771],{},[100,7773],{"className":7774,"style":355},[354],[100,7776,242],{"className":7777},[359],[100,7779],{"className":7780,"style":355},[354],[100,7782,7784,7788,7863,7866,7922,7925,7992,7995,8038,8041,8046,8049,8092,8095,8135,8138,8178,8181,8184,8187,8190],{"className":7783},[141],[100,7785],{"className":7786,"style":7787},[145],"height:3.15em;vertical-align:-1.3217em;",[100,7789,7791,7794,7860],{"className":7790},[150],[100,7792],{"className":7793},[303,788],[100,7795,7797],{"className":7796},[7633],[100,7798,7800,7851],{"className":7799},[163,164],[100,7801,7803,7848],{"className":7802},[168],[100,7804,7807,7825,7836],{"className":7805,"style":7806},[172],"height:1.3214em;",[100,7808,7810,7813],{"style":7809},"top:-2.314em;",[100,7811],{"className":7812,"style":394},[180],[100,7814,7816,7819,7822],{"className":7815},[150],[100,7817,624],{"className":7818},[150],[100,7820,7353],{"className":7821,"style":195},[150,7408],[100,7823,624],{"className":7824},[150],[100,7826,7828,7831],{"style":7827},"top:-3.23em;",[100,7829],{"className":7830,"style":394},[180],[100,7832],{"className":7833,"style":7835},[7834],"frac-line","border-bottom-width:0.04em;",[100,7837,7839,7842],{"style":7838},"top:-3.677em;",[100,7840],{"className":7841,"style":394},[180],[100,7843,7845],{"className":7844},[150],[100,7846,584],{"className":7847},[150],[100,7849,200],{"className":7850},[199],[100,7852,7854],{"className":7853},[168],[100,7855,7858],{"className":7856,"style":7857},[172],"height:0.936em;",[100,7859],{},[100,7861],{"className":7862},[350,788],[100,7864],{"className":7865,"style":777},[354],[100,7867,7869],{"className":7868},[376,377],[100,7870,7872,7913],{"className":7871},[163,164],[100,7873,7875,7910],{"className":7874},[168],[100,7876,7879,7900],{"className":7877,"style":7878},[172],"height:1.05em;",[100,7880,7882,7885],{"style":7881},"top:-1.8557em;margin-left:0em;",[100,7883],{"className":7884,"style":1734},[180],[100,7886,7888],{"className":7887},[185,186,187,188],[100,7889,7891,7894,7897],{"className":7890},[150,188],[100,7892,258],{"className":7893,"style":404},[150,154,188],[100,7895,7655],{"className":7896},[359,188],[100,7898,7353],{"className":7899,"style":195},[150,7408,188],[100,7901,7902,7905],{"style":1752},[100,7903],{"className":7904,"style":1734},[180],[100,7906,7907],{},[100,7908,2508],{"className":7909},[376,1761,1762],[100,7911,200],{"className":7912},[199],[100,7914,7916],{"className":7915},[168],[100,7917,7920],{"className":7918,"style":7919},[172],"height:1.3217em;",[100,7921],{},[100,7923],{"className":7924,"style":777},[354],[100,7926,7928],{"className":7927},[376,377],[100,7929,7931,7984],{"className":7930},[163,164],[100,7932,7934,7981],{"className":7933},[168],[100,7935,7937,7957,7967],{"className":7936,"style":1727},[172],[100,7938,7939,7942],{"style":1730},[100,7940],{"className":7941,"style":1734},[180],[100,7943,7945],{"className":7944},[185,186,187,188],[100,7946,7948,7951,7954],{"className":7947},[150,188],[100,7949,1515],{"className":7950},[150,154,188],[100,7952,242],{"className":7953},[359,188],[100,7955,1229],{"className":7956},[150,188],[100,7958,7959,7962],{"style":1752},[100,7960],{"className":7961,"style":1734},[180],[100,7963,7964],{},[100,7965,2508],{"className":7966},[376,1761,1762],[100,7968,7969,7972],{"style":1765},[100,7970],{"className":7971,"style":1734},[180],[100,7973,7975],{"className":7974},[185,186,187,188],[100,7976,7978],{"className":7977},[150,188],[100,7979,1258],{"className":7980,"style":1436},[150,154,188],[100,7982,200],{"className":7983},[199],[100,7985,7987],{"className":7986},[168],[100,7988,7990],{"className":7989,"style":1787},[172],[100,7991],{},[100,7993],{"className":7994,"style":777},[354],[100,7996,7998,8001],{"className":7997},[150],[100,7999,607],{"className":8000},[150],[100,8002,8004],{"className":8003},[159],[100,8005,8007,8030],{"className":8006},[163,164],[100,8008,8010,8027],{"className":8009},[168],[100,8011,8013],{"className":8012,"style":173},[172],[100,8014,8015,8018],{"style":809},[100,8016],{"className":8017,"style":181},[180],[100,8019,8021],{"className":8020},[185,186,187,188],[100,8022,8024],{"className":8023},[150,188],[100,8025,127],{"className":8026,"style":195},[150,154,188],[100,8028,200],{"className":8029},[199],[100,8031,8033],{"className":8032},[168],[100,8034,8036],{"className":8035,"style":207},[172],[100,8037],{},[100,8039],{"className":8040,"style":777},[354],[100,8042,2116,8044],{"className":8043},[376],[100,8045,2120],{"style":2119},[100,8047],{"className":8048,"style":777},[354],[100,8050,8052,8055],{"className":8051},[150],[100,8053,124],{"className":8054,"style":155},[150,154],[100,8056,8058],{"className":8057},[159],[100,8059,8061,8084],{"className":8060},[163,164],[100,8062,8064,8081],{"className":8063},[168],[100,8065,8067],{"className":8066,"style":173},[172],[100,8068,8069,8072],{"style":176},[100,8070],{"className":8071,"style":181},[180],[100,8073,8075],{"className":8074},[185,186,187,188],[100,8076,8078],{"className":8077},[150,188],[100,8079,127],{"className":8080,"style":195},[150,154,188],[100,8082,200],{"className":8083},[199],[100,8085,8087],{"className":8086},[168],[100,8088,8090],{"className":8089,"style":207},[172],[100,8091],{},[100,8093,230],{"className":8094},[303],[100,8096,8098,8101],{"className":8097},[150],[100,8099,18],{"className":8100},[150,154],[100,8102,8104],{"className":8103},[159],[100,8105,8107,8127],{"className":8106},[163,164],[100,8108,8110,8124],{"className":8109},[168],[100,8111,8113],{"className":8112,"style":1869},[172],[100,8114,8115,8118],{"style":809},[100,8116],{"className":8117,"style":181},[180],[100,8119,8121],{"className":8120},[185,186,187,188],[100,8122,1515],{"className":8123},[150,154,188],[100,8125,200],{"className":8126},[199],[100,8128,8130],{"className":8129},[168],[100,8131,8133],{"className":8132,"style":207},[172],[100,8134],{},[100,8136,624],{"className":8137},[150],[100,8139,8141,8144],{"className":8140},[150],[100,8142,1226],{"className":8143},[150,154],[100,8145,8147],{"className":8146},[159],[100,8148,8150,8170],{"className":8149},[163,164],[100,8151,8153,8167],{"className":8152},[168],[100,8154,8156],{"className":8155,"style":1869},[172],[100,8157,8158,8161],{"style":809},[100,8159],{"className":8160,"style":181},[180],[100,8162,8164],{"className":8163},[185,186,187,188],[100,8165,1515],{"className":8166},[150,154,188],[100,8168,200],{"className":8169},[199],[100,8171,8173],{"className":8172},[168],[100,8174,8176],{"className":8175,"style":207},[172],[100,8177],{},[100,8179,239],{"className":8180},[350],[100,8182,273],{"className":8183,"style":487},[150,154],[100,8185,230],{"className":8186},[303],[100,8188,258],{"className":8189,"style":404},[150,154],[100,8191,239],{"className":8192},[350],[14,8194,8195,8196,8235,8236,8265,8266,8294],{},"其中 ",[100,8197,8199,8217],{"className":8198},[103],[100,8200,8202],{"className":8201},[107],[109,8203,8204],{"xmlns":111},[113,8205,8206,8214],{},[116,8207,8208,8210,8212],{},[122,8209,624],{"mathvariant":606},[122,8211,7353],{"mathvariant":7352},[122,8213,624],{"mathvariant":606},[129,8215,8216],{"encoding":131},"|\\mathcal{D}|",[100,8218,8220],{"className":8219,"ariaHidden":137},[136],[100,8221,8223,8226,8229,8232],{"className":8222},[141],[100,8224],{"className":8225,"style":295},[145],[100,8227,624],{"className":8228},[150],[100,8230,7353],{"className":8231,"style":195},[150,7408],[100,8233,624],{"className":8234},[150]," 是 ",[100,8237,8239,8253],{"className":8238},[103],[100,8240,8242],{"className":8241},[107],[109,8243,8244],{"xmlns":111},[113,8245,8246,8250],{},[116,8247,8248],{},[122,8249,7353],{"mathvariant":7352},[129,8251,8252],{"encoding":131},"\\mathcal{D}",[100,8254,8256],{"className":8255,"ariaHidden":137},[136],[100,8257,8259,8262],{"className":8258},[141],[100,8260],{"className":8261,"style":7404},[145],[100,8263,7353],{"className":8264,"style":195},[150,7408]," 里的轨迹条数（这里是 ",[100,8267,8269,8282],{"className":8268},[103],[100,8270,8272],{"className":8271},[107],[109,8273,8274],{"xmlns":111},[113,8275,8276,8280],{},[116,8277,8278],{},[122,8279,7391],{},[129,8281,7391],{"encoding":131},[100,8283,8285],{"className":8284,"ariaHidden":137},[136],[100,8286,8288,8291],{"className":8287},[141],[100,8289],{"className":8290,"style":7404},[145],[100,8292,7391],{"className":8293,"style":7520},[150,154],"）。",[14,8296,8297,8298,8460],{},"最后这个式子就是我们想要的「可计算表达式」的最简单的版本。只要策略的表示方式让我们能算出 ",[100,8299,8301,8339],{"className":8300},[103],[100,8302,8304],{"className":8303},[107],[109,8305,8306],{"xmlns":111},[113,8307,8308,8336],{},[116,8309,8310,8316,8318,8320,8326,8328,8330,8332,8334],{},[119,8311,8312,8314],{},[122,8313,607],{"mathvariant":606},[122,8315,127],{},[122,8317,2093],{},[227,8319,2096],{},[119,8321,8322,8324],{},[122,8323,124],{},[122,8325,127],{},[227,8327,230],{"stretchy":229},[122,8329,18],{},[122,8331,624],{"mathvariant":606},[122,8333,1226],{},[227,8335,239],{"stretchy":229},[129,8337,8338],{"encoding":131},"\\nabla_{\\theta} \\log \\pi_{\\theta}(a|s)",[100,8340,8342],{"className":8341,"ariaHidden":137},[136],[100,8343,8345,8348,8391,8394,8399,8402,8445,8448,8451,8454,8457],{"className":8344},[141],[100,8346],{"className":8347,"style":295},[145],[100,8349,8351,8354],{"className":8350},[150],[100,8352,607],{"className":8353},[150],[100,8355,8357],{"className":8356},[159],[100,8358,8360,8383],{"className":8359},[163,164],[100,8361,8363,8380],{"className":8362},[168],[100,8364,8366],{"className":8365,"style":173},[172],[100,8367,8368,8371],{"style":809},[100,8369],{"className":8370,"style":181},[180],[100,8372,8374],{"className":8373},[185,186,187,188],[100,8375,8377],{"className":8376},[150,188],[100,8378,127],{"className":8379,"style":195},[150,154,188],[100,8381,200],{"className":8382},[199],[100,8384,8386],{"className":8385},[168],[100,8387,8389],{"className":8388,"style":207},[172],[100,8390],{},[100,8392],{"className":8393,"style":777},[354],[100,8395,2116,8397],{"className":8396},[376],[100,8398,2120],{"style":2119},[100,8400],{"className":8401,"style":777},[354],[100,8403,8405,8408],{"className":8404},[150],[100,8406,124],{"className":8407,"style":155},[150,154],[100,8409,8411],{"className":8410},[159],[100,8412,8414,8437],{"className":8413},[163,164],[100,8415,8417,8434],{"className":8416},[168],[100,8418,8420],{"className":8419,"style":173},[172],[100,8421,8422,8425],{"style":176},[100,8423],{"className":8424,"style":181},[180],[100,8426,8428],{"className":8427},[185,186,187,188],[100,8429,8431],{"className":8430},[150,188],[100,8432,127],{"className":8433,"style":195},[150,154,188],[100,8435,200],{"className":8436},[199],[100,8438,8440],{"className":8439},[168],[100,8441,8443],{"className":8442,"style":207},[172],[100,8444],{},[100,8446,230],{"className":8447},[303],[100,8449,18],{"className":8450},[150,154],[100,8452,624],{"className":8453},[150],[100,8455,1226],{"className":8456},[150,154],[100,8458,239],{"className":8459},[350],"，并且能把策略放进环境里跑、收集轨迹数据集，我们就能算出策略梯度、走一步更新。",[93,8462,8463],{"id":8463},"实现最简单的策略梯度",[14,8465,8466,8467,8471,8472,8477],{},"这个最简单版本策略梯度算法的 PyTorch 短实现放在 ",[8468,8469,8470],"code",{},"spinup\u002Fexamples\u002Fpytorch\u002Fpg_math\u002F1_simple_pg.py","（也可以",[18,8473,8476],{"href":8474,"rel":8475},"https:\u002F\u002Fgithub.com\u002Fopenai\u002Fspinningup\u002Fblob\u002Fmaster\u002Fspinup\u002Fexamples\u002Fpytorch\u002Fpg_math\u002F1_simple_pg.py",[22],"在 GitHub 上查看","）。它只有 128 行，我们强烈建议通读一遍。这里不逐行讲整个文件，只挑几块重要的部分解释。",[10,8479,8481],{"title":8480},"你应该知道",[14,8482,8483,8484,8489],{},"本节最初写的是 Tensorflow 例子，旧版 Tensorflow 的内容见",[18,8485,8488],{"href":8486,"rel":8487},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Fextra_tf_pg_implementation.html#implementing-the-simplest-policy-gradient",[22],"这里","。",[14,8491,8492],{},[51,8493,8494],{},"1. 搭建策略网络。",[8496,8497,8502],"pre",{"className":8498,"code":8499,"language":8500,"meta":8501,"style":8501},"language-python3 shiki shiki-themes github-light github-dark","# 搭建策略网络的核心\nlogits_net = mlp(sizes=[obs_dim]+hidden_sizes+[n_acts])\n\n# 构造计算动作分布的函数\ndef get_policy(obs):\n    logits = logits_net(obs)\n    return Categorical(logits=logits)\n\n# 构造选动作的函数（输出整数动作，从策略中采样）\ndef get_action(obs):\n    return get_policy(obs).sample().item()\n","python3","",[8468,8503,8504,8511,8543,8550,8556,8569,8580,8597,8602,8608,8618],{"__ignoreMap":8501},[100,8505,8507],{"class":4184,"line":8506},1,[100,8508,8510],{"class":8509},"sJ8bj","# 搭建策略网络的核心\n",[100,8512,8514,8518,8521,8524,8528,8530,8533,8535,8538,8540],{"class":4184,"line":8513},2,[100,8515,8517],{"class":8516},"sVt8B","logits_net ",[100,8519,242],{"class":8520},"szBVR",[100,8522,8523],{"class":8516}," mlp(",[100,8525,8527],{"class":8526},"s4XuR","sizes",[100,8529,242],{"class":8520},[100,8531,8532],{"class":8516},"[obs_dim]",[100,8534,580],{"class":8520},[100,8536,8537],{"class":8516},"hidden_sizes",[100,8539,580],{"class":8520},[100,8541,8542],{"class":8516},"[n_acts])\n",[100,8544,8546],{"class":4184,"line":8545},3,[100,8547,8549],{"emptyLinePlaceholder":8548},true,"\n",[100,8551,8553],{"class":4184,"line":8552},4,[100,8554,8555],{"class":8509},"# 构造计算动作分布的函数\n",[100,8557,8559,8562,8566],{"class":4184,"line":8558},5,[100,8560,8561],{"class":8520},"def",[100,8563,8565],{"class":8564},"sScJk"," get_policy",[100,8567,8568],{"class":8516},"(obs):\n",[100,8570,8572,8575,8577],{"class":4184,"line":8571},6,[100,8573,8574],{"class":8516},"    logits ",[100,8576,242],{"class":8520},[100,8578,8579],{"class":8516}," logits_net(obs)\n",[100,8581,8583,8586,8589,8592,8594],{"class":4184,"line":8582},7,[100,8584,8585],{"class":8520},"    return",[100,8587,8588],{"class":8516}," Categorical(",[100,8590,8591],{"class":8526},"logits",[100,8593,242],{"class":8520},[100,8595,8596],{"class":8516},"logits)\n",[100,8598,8600],{"class":4184,"line":8599},8,[100,8601,8549],{"emptyLinePlaceholder":8548},[100,8603,8605],{"class":4184,"line":8604},9,[100,8606,8607],{"class":8509},"# 构造选动作的函数（输出整数动作，从策略中采样）\n",[100,8609,8611,8613,8616],{"class":4184,"line":8610},10,[100,8612,8561],{"class":8520},[100,8614,8615],{"class":8564}," get_action",[100,8617,8568],{"class":8516},[100,8619,8621,8623],{"class":4184,"line":8620},11,[100,8622,8585],{"class":8520},[100,8624,8625],{"class":8516}," get_policy(obs).sample().item()\n",[14,8627,8628,8629,8294,8633,8636,8637,8640,8641,8643,8644,8647,8648,8651,8652,8657],{},"这一块搭建了前馈神经网络分类策略所需的模块和函数（想复习可以看",[18,8630,8632],{"href":8631},"\u002Fposts\u002Fspinning-up-rl-intro#%E9%9A%8F%E6%9C%BA%E5%9E%8B%E7%AD%96%E7%95%A5","第一篇的「随机型策略」一节",[8468,8634,8635],{},"logits_net"," 模块的输出可以用来构造动作的对数概率和概率，",[8468,8638,8639],{},"get_action"," 函数根据 logits 算出的概率采样动作。（注意：这个 ",[8468,8642,8639],{}," 假定每次只传入一个 ",[8468,8645,8646],{},"obs","，因此也只输出一个整数动作。这就是它用 ",[8468,8649,8650],{},".item()"," 的原因，这个方法用来",[18,8653,8656],{"href":8654,"rel":8655},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Ftensors.html#torch.Tensor.item",[22],"取出只含单个元素的 Tensor 的内容","。）",[14,8659,8660,8661,8664,8665,8668,8669,8674],{},"这个例子里有大量工作是由源文件第 36 行的 ",[8468,8662,8663],{},"Categorical"," 对象完成的。它是 PyTorch 的 ",[8468,8666,8667],{},"Distribution"," 对象，封装了与概率分布相关的一些数学函数：既有从分布采样的方法（第 40 行用到了），也有计算给定样本对数概率的方法（后面会用到）。PyTorch 的 distributions 对 RL 非常有用，建议翻一翻",[18,8670,8673],{"href":8671,"rel":8672},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Fdistributions.html",[22],"它们的文档","，感受一下怎么用。",[10,8676,8677,8784],{"title":8480},[14,8678,8679,8680,8753,8754,8783],{},"友好提醒！说一个分类分布有「logits」，意思是各结果的概率由 logits 的 softmax 函数给出。也就是说，logits 为 ",[100,8681,8683,8702],{"className":8682},[103],[100,8684,8686],{"className":8685},[107],[109,8687,8688],{"xmlns":111},[113,8689,8690,8699],{},[116,8691,8692],{},[119,8693,8694,8696],{},[122,8695,2099],{},[122,8697,8698],{},"j",[129,8700,8701],{"encoding":131},"x_j",[100,8703,8705],{"className":8704,"ariaHidden":137},[136],[100,8706,8708,8712],{"className":8707},[141],[100,8709],{"className":8710,"style":8711},[145],"height:0.7167em;vertical-align:-0.2861em;",[100,8713,8715,8718],{"className":8714},[150],[100,8716,2099],{"className":8717},[150,154],[100,8719,8721],{"className":8720},[159],[100,8722,8724,8745],{"className":8723},[163,164],[100,8725,8727,8742],{"className":8726},[168],[100,8728,8730],{"className":8729,"style":7446},[172],[100,8731,8732,8735],{"style":809},[100,8733],{"className":8734,"style":181},[180],[100,8736,8738],{"className":8737},[185,186,187,188],[100,8739,8698],{"className":8740,"style":8741},[150,154,188],"margin-right:0.0572em;",[100,8743,200],{"className":8744},[199],[100,8746,8748],{"className":8747},[168],[100,8749,8751],{"className":8750,"style":7530},[172],[100,8752],{}," 的分类分布下，动作 ",[100,8755,8757,8770],{"className":8756},[103],[100,8758,8760],{"className":8759},[107],[109,8761,8762],{"xmlns":111},[113,8763,8764,8768],{},[116,8765,8766],{},[122,8767,8698],{},[129,8769,8698],{"encoding":131},[100,8771,8773],{"className":8772,"ariaHidden":137},[136],[100,8774,8776,8780],{"className":8775},[141],[100,8777],{"className":8778,"style":8779},[145],"height:0.854em;vertical-align:-0.1944em;",[100,8781,8698],{"className":8782,"style":8741},[150,154]," 的概率是：",[100,8785,8787],{"className":8786},[555],[100,8788,8790,8851],{"className":8789},[103],[100,8791,8793],{"className":8792},[107],[109,8794,8795],{"xmlns":111,"display":564},[113,8796,8797,8848],{},[116,8798,8799,8805,8807],{},[119,8800,8801,8803],{},[122,8802,14],{},[122,8804,8698],{},[227,8806,242],{},[7633,8808,8809,8826],{},[116,8810,8811,8814,8816,8818,8824],{},[122,8812,8813],{},"exp",[227,8815,2096],{},[227,8817,230],{"stretchy":229},[119,8819,8820,8822],{},[122,8821,2099],{},[122,8823,8698],{},[227,8825,239],{"stretchy":229},[116,8827,8828,8834,8836,8838,8840,8846],{},[246,8829,8830,8832],{},[227,8831,2508],{},[122,8833,7365],{},[122,8835,8813],{},[227,8837,2096],{},[227,8839,230],{"stretchy":229},[119,8841,8842,8844],{},[122,8843,2099],{},[122,8845,7365],{},[227,8847,239],{"stretchy":229},[129,8849,8850],{"encoding":131},"p_j = \\frac{\\exp(x_j)}{\\sum_{i} \\exp(x_i)}",[100,8852,8854,8909],{"className":8853,"ariaHidden":137},[136],[100,8855,8857,8860,8900,8903,8906],{"className":8856},[141],[100,8858],{"className":8859,"style":8711},[145],[100,8861,8863,8866],{"className":8862},[150],[100,8864,14],{"className":8865},[150,154],[100,8867,8869],{"className":8868},[159],[100,8870,8872,8892],{"className":8871},[163,164],[100,8873,8875,8889],{"className":8874},[168],[100,8876,8878],{"className":8877,"style":7446},[172],[100,8879,8880,8883],{"style":809},[100,8881],{"className":8882,"style":181},[180],[100,8884,8886],{"className":8885},[185,186,187,188],[100,8887,8698],{"className":8888,"style":8741},[150,154,188],[100,8890,200],{"className":8891},[199],[100,8893,8895],{"className":8894},[168],[100,8896,8898],{"className":8897,"style":7530},[172],[100,8899],{},[100,8901],{"className":8902,"style":355},[354],[100,8904,242],{"className":8905},[359],[100,8907],{"className":8908,"style":355},[354],[100,8910,8912,8916],{"className":8911},[141],[100,8913],{"className":8914,"style":8915},[145],"height:2.4127em;vertical-align:-0.9857em;",[100,8917,8919,8922,9120],{"className":8918},[150],[100,8920],{"className":8921},[303,788],[100,8923,8925],{"className":8924},[7633],[100,8926,8928,9111],{"className":8927},[163,164],[100,8929,8931,9108],{"className":8930},[168],[100,8932,8935,9043,9051],{"className":8933,"style":8934},[172],"height:1.427em;",[100,8936,8937,8940],{"style":7809},[100,8938],{"className":8939,"style":394},[180],[100,8941,8943,8991,8994,8997,9000,9040],{"className":8942},[150],[100,8944,8946,8951],{"className":8945},[376],[100,8947,2508],{"className":8948,"style":8950},[376,1761,8949],"small-op","position:relative;top:0em;",[100,8952,8954],{"className":8953},[159],[100,8955,8957,8982],{"className":8956},[163,164],[100,8958,8960,8979],{"className":8959},[168],[100,8961,8964],{"className":8962,"style":8963},[172],"height:0.162em;",[100,8965,8967,8970],{"style":8966},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[100,8968],{"className":8969,"style":181},[180],[100,8971,8973],{"className":8972},[185,186,187,188],[100,8974,8976],{"className":8975},[150,188],[100,8977,7365],{"className":8978},[150,154,188],[100,8980,200],{"className":8981},[199],[100,8983,8985],{"className":8984},[168],[100,8986,8989],{"className":8987,"style":8988},[172],"height:0.2997em;",[100,8990],{},[100,8992],{"className":8993,"style":777},[354],[100,8995,8813],{"className":8996},[376],[100,8998,230],{"className":8999},[303],[100,9001,9003,9006],{"className":9002},[150],[100,9004,2099],{"className":9005},[150,154],[100,9007,9009],{"className":9008},[159],[100,9010,9012,9032],{"className":9011},[163,164],[100,9013,9015,9029],{"className":9014},[168],[100,9016,9018],{"className":9017,"style":7446},[172],[100,9019,9020,9023],{"style":809},[100,9021],{"className":9022,"style":181},[180],[100,9024,9026],{"className":9025},[185,186,187,188],[100,9027,7365],{"className":9028},[150,154,188],[100,9030,200],{"className":9031},[199],[100,9033,9035],{"className":9034},[168],[100,9036,9038],{"className":9037,"style":207},[172],[100,9039],{},[100,9041,239],{"className":9042},[350],[100,9044,9045,9048],{"style":7827},[100,9046],{"className":9047,"style":394},[180],[100,9049],{"className":9050,"style":7835},[7834],[100,9052,9053,9056],{"style":7838},[100,9054],{"className":9055,"style":394},[180],[100,9057,9059,9062,9065,9105],{"className":9058},[150],[100,9060,8813],{"className":9061},[376],[100,9063,230],{"className":9064},[303],[100,9066,9068,9071],{"className":9067},[150],[100,9069,2099],{"className":9070},[150,154],[100,9072,9074],{"className":9073},[159],[100,9075,9077,9097],{"className":9076},[163,164],[100,9078,9080,9094],{"className":9079},[168],[100,9081,9083],{"className":9082,"style":7446},[172],[100,9084,9085,9088],{"style":809},[100,9086],{"className":9087,"style":181},[180],[100,9089,9091],{"className":9090},[185,186,187,188],[100,9092,8698],{"className":9093,"style":8741},[150,154,188],[100,9095,200],{"className":9096},[199],[100,9098,9100],{"className":9099},[168],[100,9101,9103],{"className":9102,"style":7530},[172],[100,9104],{},[100,9106,239],{"className":9107},[350],[100,9109,200],{"className":9110},[199],[100,9112,9114],{"className":9113},[168],[100,9115,9118],{"className":9116,"style":9117},[172],"height:0.9857em;",[100,9119],{},[100,9121],{"className":9122},[350,788],[14,9124,9125],{},[51,9126,9127],{},"2. 构造损失函数。",[8496,9129,9131],{"className":8498,"code":9130,"language":8500,"meta":8501,"style":8501},"# 构造损失函数：喂进合适的数据时，它的梯度就是策略梯度\ndef compute_loss(obs, act, weights):\n    logp = get_policy(obs).log_prob(act)\n    return -(logp * weights).mean()\n",[8468,9132,9133,9138,9148,9158],{"__ignoreMap":8501},[100,9134,9135],{"class":4184,"line":8506},[100,9136,9137],{"class":8509},"# 构造损失函数：喂进合适的数据时，它的梯度就是策略梯度\n",[100,9139,9140,9142,9145],{"class":4184,"line":8513},[100,9141,8561],{"class":8520},[100,9143,9144],{"class":8564}," compute_loss",[100,9146,9147],{"class":8516},"(obs, act, weights):\n",[100,9149,9150,9153,9155],{"class":4184,"line":8545},[100,9151,9152],{"class":8516},"    logp ",[100,9154,242],{"class":8520},[100,9156,9157],{"class":8516}," get_policy(obs).log_prob(act)\n",[100,9159,9160,9162,9165,9168,9171],{"class":4184,"line":8552},[100,9161,8585],{"class":8520},[100,9163,9164],{"class":8520}," -",[100,9166,9167],{"class":8516},"(logp ",[100,9169,9170],{"class":8520},"*",[100,9172,9173],{"class":8516}," weights).mean()\n",[14,9175,9176],{},"这一块为策略梯度算法构造一个「损失」函数。喂进正确的数据时，这个损失的梯度就等于策略梯度。所谓正确的数据，是一组按当前策略行动时收集的（状态、动作、权重）三元组，其中状态-动作对的权重是它所属回合的回报。（后面的小节会说明，权重还可以换成别的值，同样正确。）",[10,9178,9179,9186,9192,9287,9333],{"title":8480},[14,9180,9181,9182,9185],{},"虽然我们把它叫作损失函数，但它",[51,9183,9184],{},"不是","监督学习里通常意义上的那种损失函数。它与标准损失函数有两点主要区别。",[14,9187,9188,9191],{},[51,9189,9190],{},"1. 数据分布依赖参数","。损失函数通常定义在一个与待优化参数无关的固定数据分布上。这里不是这样：数据必须从最新的策略里采样。",[14,9193,9194,9197,9198,9286],{},[51,9195,9196],{},"2. 它不度量性能","。损失函数通常度量我们关心的性能指标。这里我们关心的是期望回报 ",[100,9199,9201,9225],{"className":9200},[103],[100,9202,9204],{"className":9203},[107],[109,9205,9206],{"xmlns":111},[113,9207,9208,9222],{},[116,9209,9210,9212,9214,9220],{},[122,9211,225],{},[227,9213,230],{"stretchy":229},[119,9215,9216,9218],{},[122,9217,124],{},[122,9219,127],{},[227,9221,239],{"stretchy":229},[129,9223,9224],{"encoding":131},"J(\\pi_{\\theta})",[100,9226,9228],{"className":9227,"ariaHidden":137},[136],[100,9229,9231,9234,9237,9240,9283],{"className":9230},[141],[100,9232],{"className":9233,"style":295},[145],[100,9235,225],{"className":9236,"style":299},[150,154],[100,9238,230],{"className":9239},[303],[100,9241,9243,9246],{"className":9242},[150],[100,9244,124],{"className":9245,"style":155},[150,154],[100,9247,9249],{"className":9248},[159],[100,9250,9252,9275],{"className":9251},[163,164],[100,9253,9255,9272],{"className":9254},[168],[100,9256,9258],{"className":9257,"style":173},[172],[100,9259,9260,9263],{"style":176},[100,9261],{"className":9262,"style":181},[180],[100,9264,9266],{"className":9265},[185,186,187,188],[100,9267,9269],{"className":9268},[150,188],[100,9270,127],{"className":9271,"style":195},[150,154,188],[100,9273,200],{"className":9274},[199],[100,9276,9278],{"className":9277},[168],[100,9279,9281],{"className":9280,"style":207},[172],[100,9282],{},[100,9284,239],{"className":9285},[350],"，但这个「损失」函数连在期望意义上都不近似它。这个「损失」函数之所以有用，只是因为：在当前参数处、用当前参数生成的数据来评估时，它的梯度是性能的负梯度。",[14,9288,9289,9290,9294,9295,9332],{},"但走出梯度下降的第一步之后，它就与性能没有任何联系了。这意味着：对给定的一批数据，最小化这个「损失」函数",[9291,9292,9293],"em",{},"完全","不保证提升期望回报。你可以把这个损失压到 ",[100,9296,9298,9316],{"className":9297},[103],[100,9299,9301],{"className":9300},[107],[109,9302,9303],{"xmlns":111},[113,9304,9305,9313],{},[116,9306,9307,9310],{},[227,9308,9309],{},"−",[122,9311,9312],{"mathvariant":606},"∞",[129,9314,9315],{"encoding":131},"-\\infty",[100,9317,9319],{"className":9318,"ariaHidden":137},[136],[100,9320,9322,9326,9329],{"className":9321},[141],[100,9323],{"className":9324,"style":9325},[145],"height:0.6667em;vertical-align:-0.0833em;",[100,9327,9309],{"className":9328},[150],[100,9330,9312],{"className":9331},[150],"，而策略性能照样会崩；实际上通常就会崩。有的深度 RL 研究者会把这种结果描述成策略对这批数据「过拟合」了。这个说法有描述作用，但别按字面理解，因为它与泛化误差无关。",[14,9334,9335],{},"我们特意提这一点，是因为机器学习从业者习惯把损失函数当作训练中的有用信号：「损失降了，一切安好。」在策略梯度里，这个直觉是错的，你只该关心平均回报。损失函数没有任何意义。",[10,9337,9338,9352,9376],{"title":8480},[14,9339,9340,9341,9344,9345,9347,9348,9351],{},"这里构造 ",[8468,9342,9343],{},"logp"," 张量的方式（调用 PyTorch ",[8468,9346,8663],{}," 对象的 ",[8468,9349,9350],{},"log_prob"," 方法）换成其他分布对象时，可能需要一些修改。",[14,9353,9354,9355,9360,9361,9364,9365,9368,9369,9371,9372,9375],{},"比如，如果你用的是 ",[18,9356,9359],{"href":9357,"rel":9358},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Fdistributions.html#normal",[22],"Normal 分布","（对应对角高斯策略），调用 ",[8468,9362,9363],{},"policy.log_prob(act)"," 返回的张量会给每个向量型动作的每个分量各一个对数概率：输入形状 ",[8468,9366,9367],{},"(batch, act_dim)"," 的张量，出来的还是 ",[8468,9370,9367],{},"，而构造 RL 损失需要的是 ",[8468,9373,9374],{},"(batch,)","。这时要把动作分量的对数概率加总，得到动作的对数概率，也就是计算：",[8496,9377,9379],{"className":8498,"code":9378,"language":8500,"meta":8501,"style":8501},"logp = get_policy(obs).log_prob(act).sum(axis=-1)\n",[8468,9380,9381],{"__ignoreMap":8501},[100,9382,9383,9386,9388,9391,9394,9397,9400],{"class":4184,"line":8506},[100,9384,9385],{"class":8516},"logp ",[100,9387,242],{"class":8520},[100,9389,9390],{"class":8516}," get_policy(obs).log_prob(act).sum(",[100,9392,9393],{"class":8526},"axis",[100,9395,9396],{"class":8520},"=-",[100,9398,584],{"class":9399},"sj4cs",[100,9401,9402],{"class":8516},")\n",[14,9404,9405],{},[51,9406,9407],{},"3. 跑一轮训练。",[8496,9409,9411],{"className":8498,"code":9410,"language":8500,"meta":8501,"style":8501},"# 用于训练策略\ndef train_one_epoch():\n    # 建几个空列表，用于记录\n    batch_obs = []          # 观测\n    batch_acts = []         # 动作\n    batch_weights = []      # 策略梯度里的 R(tau) 权重\n    batch_rets = []         # 记录回合回报\n    batch_lens = []         # 记录回合长度\n\n    # 重置回合内变量\n    obs = env.reset()       # 第一个观测来自起始分布\n    done = False            # 环境给出的「回合结束」信号\n    ep_rews = []            # 本回合累计的奖励列表\n\n    # 每个 epoch 渲染第一个回合\n    finished_rendering_this_epoch = False\n\n    # 用当前策略在环境中行动，收集经验\n    while True:\n\n        # 渲染\n        if (not finished_rendering_this_epoch) and render:\n            env.render()\n\n        # 保存观测\n        batch_obs.append(obs.copy())\n\n        # 在环境中行动\n        act = get_action(torch.as_tensor(obs, dtype=torch.float32))\n        obs, rew, done, _ = env.step(act)\n\n        # 保存动作与奖励\n        batch_acts.append(act)\n        ep_rews.append(rew)\n\n        if done:\n            # 回合结束，记录该回合的信息\n            ep_ret, ep_len = sum(ep_rews), len(ep_rews)\n            batch_rets.append(ep_ret)\n            batch_lens.append(ep_len)\n\n            # 每个 logprob(a|s) 的权重是 R(tau)\n            batch_weights += [ep_ret] * ep_len\n\n            # 重置回合内变量\n            obs, done, ep_rews = env.reset(), False, []\n\n            # 这个 epoch 不再渲染\n            finished_rendering_this_epoch = True\n\n            # 经验够了就结束收集循环\n            if len(batch_obs) > batch_size:\n                break\n\n    # 做一次策略梯度更新\n    optimizer.zero_grad()\n    batch_loss = compute_loss(obs=torch.as_tensor(batch_obs, dtype=torch.float32),\n                              act=torch.as_tensor(batch_acts, dtype=torch.int32),\n                              weights=torch.as_tensor(batch_weights, dtype=torch.float32)\n                              )\n    batch_loss.backward()\n    optimizer.step()\n    return batch_loss, batch_rets, batch_lens\n",[8468,9412,9413,9418,9428,9433,9446,9459,9472,9484,9496,9500,9505,9518,9532,9546,9551,9557,9568,9573,9579,9591,9596,9602,9623,9629,9634,9640,9646,9651,9657,9676,9687,9692,9698,9704,9710,9715,9723,9729,9749,9755,9761,9766,9772,9789,9794,9800,9817,9822,9828,9839,9844,9850,9868,9874,9879,9885,9891,9916,9934,9952,9958,9964,9970],{"__ignoreMap":8501},[100,9414,9415],{"class":4184,"line":8506},[100,9416,9417],{"class":8509},"# 用于训练策略\n",[100,9419,9420,9422,9425],{"class":4184,"line":8513},[100,9421,8561],{"class":8520},[100,9423,9424],{"class":8564}," train_one_epoch",[100,9426,9427],{"class":8516},"():\n",[100,9429,9430],{"class":4184,"line":8545},[100,9431,9432],{"class":8509},"    # 建几个空列表，用于记录\n",[100,9434,9435,9438,9440,9443],{"class":4184,"line":8552},[100,9436,9437],{"class":8516},"    batch_obs ",[100,9439,242],{"class":8520},[100,9441,9442],{"class":8516}," []          ",[100,9444,9445],{"class":8509},"# 观测\n",[100,9447,9448,9451,9453,9456],{"class":4184,"line":8558},[100,9449,9450],{"class":8516},"    batch_acts ",[100,9452,242],{"class":8520},[100,9454,9455],{"class":8516}," []         ",[100,9457,9458],{"class":8509},"# 动作\n",[100,9460,9461,9464,9466,9469],{"class":4184,"line":8571},[100,9462,9463],{"class":8516},"    batch_weights ",[100,9465,242],{"class":8520},[100,9467,9468],{"class":8516}," []      ",[100,9470,9471],{"class":8509},"# 策略梯度里的 R(tau) 权重\n",[100,9473,9474,9477,9479,9481],{"class":4184,"line":8582},[100,9475,9476],{"class":8516},"    batch_rets ",[100,9478,242],{"class":8520},[100,9480,9455],{"class":8516},[100,9482,9483],{"class":8509},"# 记录回合回报\n",[100,9485,9486,9489,9491,9493],{"class":4184,"line":8599},[100,9487,9488],{"class":8516},"    batch_lens ",[100,9490,242],{"class":8520},[100,9492,9455],{"class":8516},[100,9494,9495],{"class":8509},"# 记录回合长度\n",[100,9497,9498],{"class":4184,"line":8604},[100,9499,8549],{"emptyLinePlaceholder":8548},[100,9501,9502],{"class":4184,"line":8610},[100,9503,9504],{"class":8509},"    # 重置回合内变量\n",[100,9506,9507,9510,9512,9515],{"class":4184,"line":8620},[100,9508,9509],{"class":8516},"    obs ",[100,9511,242],{"class":8520},[100,9513,9514],{"class":8516}," env.reset()       ",[100,9516,9517],{"class":8509},"# 第一个观测来自起始分布\n",[100,9519,9521,9524,9526,9529],{"class":4184,"line":9520},12,[100,9522,9523],{"class":8516},"    done ",[100,9525,242],{"class":8520},[100,9527,9528],{"class":9399}," False",[100,9530,9531],{"class":8509},"            # 环境给出的「回合结束」信号\n",[100,9533,9535,9538,9540,9543],{"class":4184,"line":9534},13,[100,9536,9537],{"class":8516},"    ep_rews ",[100,9539,242],{"class":8520},[100,9541,9542],{"class":8516}," []            ",[100,9544,9545],{"class":8509},"# 本回合累计的奖励列表\n",[100,9547,9549],{"class":4184,"line":9548},14,[100,9550,8549],{"emptyLinePlaceholder":8548},[100,9552,9554],{"class":4184,"line":9553},15,[100,9555,9556],{"class":8509},"    # 每个 epoch 渲染第一个回合\n",[100,9558,9560,9563,9565],{"class":4184,"line":9559},16,[100,9561,9562],{"class":8516},"    finished_rendering_this_epoch ",[100,9564,242],{"class":8520},[100,9566,9567],{"class":9399}," False\n",[100,9569,9571],{"class":4184,"line":9570},17,[100,9572,8549],{"emptyLinePlaceholder":8548},[100,9574,9576],{"class":4184,"line":9575},18,[100,9577,9578],{"class":8509},"    # 用当前策略在环境中行动，收集经验\n",[100,9580,9582,9585,9588],{"class":4184,"line":9581},19,[100,9583,9584],{"class":8520},"    while",[100,9586,9587],{"class":9399}," True",[100,9589,9590],{"class":8516},":\n",[100,9592,9594],{"class":4184,"line":9593},20,[100,9595,8549],{"emptyLinePlaceholder":8548},[100,9597,9599],{"class":4184,"line":9598},21,[100,9600,9601],{"class":8509},"        # 渲染\n",[100,9603,9605,9608,9611,9614,9617,9620],{"class":4184,"line":9604},22,[100,9606,9607],{"class":8520},"        if",[100,9609,9610],{"class":8516}," (",[100,9612,9613],{"class":8520},"not",[100,9615,9616],{"class":8516}," finished_rendering_this_epoch) ",[100,9618,9619],{"class":8520},"and",[100,9621,9622],{"class":8516}," render:\n",[100,9624,9626],{"class":4184,"line":9625},23,[100,9627,9628],{"class":8516},"            env.render()\n",[100,9630,9632],{"class":4184,"line":9631},24,[100,9633,8549],{"emptyLinePlaceholder":8548},[100,9635,9637],{"class":4184,"line":9636},25,[100,9638,9639],{"class":8509},"        # 保存观测\n",[100,9641,9643],{"class":4184,"line":9642},26,[100,9644,9645],{"class":8516},"        batch_obs.append(obs.copy())\n",[100,9647,9649],{"class":4184,"line":9648},27,[100,9650,8549],{"emptyLinePlaceholder":8548},[100,9652,9654],{"class":4184,"line":9653},28,[100,9655,9656],{"class":8509},"        # 在环境中行动\n",[100,9658,9660,9663,9665,9668,9671,9673],{"class":4184,"line":9659},29,[100,9661,9662],{"class":8516},"        act ",[100,9664,242],{"class":8520},[100,9666,9667],{"class":8516}," get_action(torch.as_tensor(obs, ",[100,9669,9670],{"class":8526},"dtype",[100,9672,242],{"class":8520},[100,9674,9675],{"class":8516},"torch.float32))\n",[100,9677,9679,9682,9684],{"class":4184,"line":9678},30,[100,9680,9681],{"class":8516},"        obs, rew, done, _ ",[100,9683,242],{"class":8520},[100,9685,9686],{"class":8516}," env.step(act)\n",[100,9688,9690],{"class":4184,"line":9689},31,[100,9691,8549],{"emptyLinePlaceholder":8548},[100,9693,9695],{"class":4184,"line":9694},32,[100,9696,9697],{"class":8509},"        # 保存动作与奖励\n",[100,9699,9701],{"class":4184,"line":9700},33,[100,9702,9703],{"class":8516},"        batch_acts.append(act)\n",[100,9705,9707],{"class":4184,"line":9706},34,[100,9708,9709],{"class":8516},"        ep_rews.append(rew)\n",[100,9711,9713],{"class":4184,"line":9712},35,[100,9714,8549],{"emptyLinePlaceholder":8548},[100,9716,9718,9720],{"class":4184,"line":9717},36,[100,9719,9607],{"class":8520},[100,9721,9722],{"class":8516}," done:\n",[100,9724,9726],{"class":4184,"line":9725},37,[100,9727,9728],{"class":8509},"            # 回合结束，记录该回合的信息\n",[100,9730,9732,9735,9737,9740,9743,9746],{"class":4184,"line":9731},38,[100,9733,9734],{"class":8516},"            ep_ret, ep_len ",[100,9736,242],{"class":8520},[100,9738,9739],{"class":9399}," sum",[100,9741,9742],{"class":8516},"(ep_rews), ",[100,9744,9745],{"class":9399},"len",[100,9747,9748],{"class":8516},"(ep_rews)\n",[100,9750,9752],{"class":4184,"line":9751},39,[100,9753,9754],{"class":8516},"            batch_rets.append(ep_ret)\n",[100,9756,9758],{"class":4184,"line":9757},40,[100,9759,9760],{"class":8516},"            batch_lens.append(ep_len)\n",[100,9762,9764],{"class":4184,"line":9763},41,[100,9765,8549],{"emptyLinePlaceholder":8548},[100,9767,9769],{"class":4184,"line":9768},42,[100,9770,9771],{"class":8509},"            # 每个 logprob(a|s) 的权重是 R(tau)\n",[100,9773,9775,9778,9781,9784,9786],{"class":4184,"line":9774},43,[100,9776,9777],{"class":8516},"            batch_weights ",[100,9779,9780],{"class":8520},"+=",[100,9782,9783],{"class":8516}," [ep_ret] ",[100,9785,9170],{"class":8520},[100,9787,9788],{"class":8516}," ep_len\n",[100,9790,9792],{"class":4184,"line":9791},44,[100,9793,8549],{"emptyLinePlaceholder":8548},[100,9795,9797],{"class":4184,"line":9796},45,[100,9798,9799],{"class":8509},"            # 重置回合内变量\n",[100,9801,9803,9806,9808,9811,9814],{"class":4184,"line":9802},46,[100,9804,9805],{"class":8516},"            obs, done, ep_rews ",[100,9807,242],{"class":8520},[100,9809,9810],{"class":8516}," env.reset(), ",[100,9812,9813],{"class":9399},"False",[100,9815,9816],{"class":8516},", []\n",[100,9818,9820],{"class":4184,"line":9819},47,[100,9821,8549],{"emptyLinePlaceholder":8548},[100,9823,9825],{"class":4184,"line":9824},48,[100,9826,9827],{"class":8509},"            # 这个 epoch 不再渲染\n",[100,9829,9831,9834,9836],{"class":4184,"line":9830},49,[100,9832,9833],{"class":8516},"            finished_rendering_this_epoch ",[100,9835,242],{"class":8520},[100,9837,9838],{"class":9399}," True\n",[100,9840,9842],{"class":4184,"line":9841},50,[100,9843,8549],{"emptyLinePlaceholder":8548},[100,9845,9847],{"class":4184,"line":9846},51,[100,9848,9849],{"class":8509},"            # 经验够了就结束收集循环\n",[100,9851,9853,9856,9859,9862,9865],{"class":4184,"line":9852},52,[100,9854,9855],{"class":8520},"            if",[100,9857,9858],{"class":9399}," len",[100,9860,9861],{"class":8516},"(batch_obs) ",[100,9863,9864],{"class":8520},">",[100,9866,9867],{"class":8516}," batch_size:\n",[100,9869,9871],{"class":4184,"line":9870},53,[100,9872,9873],{"class":8520},"                break\n",[100,9875,9877],{"class":4184,"line":9876},54,[100,9878,8549],{"emptyLinePlaceholder":8548},[100,9880,9882],{"class":4184,"line":9881},55,[100,9883,9884],{"class":8509},"    # 做一次策略梯度更新\n",[100,9886,9888],{"class":4184,"line":9887},56,[100,9889,9890],{"class":8516},"    optimizer.zero_grad()\n",[100,9892,9894,9897,9899,9902,9904,9906,9909,9911,9913],{"class":4184,"line":9893},57,[100,9895,9896],{"class":8516},"    batch_loss ",[100,9898,242],{"class":8520},[100,9900,9901],{"class":8516}," compute_loss(",[100,9903,8646],{"class":8526},[100,9905,242],{"class":8520},[100,9907,9908],{"class":8516},"torch.as_tensor(batch_obs, ",[100,9910,9670],{"class":8526},[100,9912,242],{"class":8520},[100,9914,9915],{"class":8516},"torch.float32),\n",[100,9917,9919,9922,9924,9927,9929,9931],{"class":4184,"line":9918},58,[100,9920,9921],{"class":8526},"                              act",[100,9923,242],{"class":8520},[100,9925,9926],{"class":8516},"torch.as_tensor(batch_acts, ",[100,9928,9670],{"class":8526},[100,9930,242],{"class":8520},[100,9932,9933],{"class":8516},"torch.int32),\n",[100,9935,9937,9940,9942,9945,9947,9949],{"class":4184,"line":9936},59,[100,9938,9939],{"class":8526},"                              weights",[100,9941,242],{"class":8520},[100,9943,9944],{"class":8516},"torch.as_tensor(batch_weights, ",[100,9946,9670],{"class":8526},[100,9948,242],{"class":8520},[100,9950,9951],{"class":8516},"torch.float32)\n",[100,9953,9955],{"class":4184,"line":9954},60,[100,9956,9957],{"class":8516},"                              )\n",[100,9959,9961],{"class":4184,"line":9960},61,[100,9962,9963],{"class":8516},"    batch_loss.backward()\n",[100,9965,9967],{"class":4184,"line":9966},62,[100,9968,9969],{"class":8516},"    optimizer.step()\n",[100,9971,9973,9975],{"class":4184,"line":9972},63,[100,9974,8585],{"class":8520},[100,9976,9977],{"class":8516}," batch_loss, batch_rets, batch_lens\n",[14,9979,9980,9983,9984,8489],{},[8468,9981,9982],{},"train_one_epoch()"," 函数跑一轮（epoch）策略梯度，我们把它定义为：1）经验收集步（源文件第 67 到 102 行），智能体用最新策略在环境中跑若干个回合；2）紧接一次策略梯度更新步（第 104 到 111 行）。算法的主循环就是反复调用 ",[8468,9985,9982],{},[10,9987,9988],{"title":8480},[14,9989,9990],{},"如果你还不熟悉 PyTorch 的优化流程，注意看第 104 到 111 行做一次梯度下降步的模式：先清空梯度缓存；然后计算损失函数；再对损失做一次反向传播，把新梯度累积进梯度缓存；最后让优化器走一步。",[93,9992,9994],{"id":9993},"eglp-引理","EGLP 引理",[14,9996,9997,9998,8489],{},"本小节推导一个在策略梯度理论里到处会用到的中间结果，我们把它叫作",[51,9999,10000],{},"期望 Grad-Log-Prob（EGLP）引理",[14,10002,10003,10005,10006,10081,10082,10110],{},[51,10004,9994],{},"。设 ",[100,10007,10009,10027],{"className":10008},[103],[100,10010,10012],{"className":10011},[107],[109,10013,10014],{"xmlns":111},[113,10015,10016,10024],{},[116,10017,10018],{},[119,10019,10020,10022],{},[122,10021,1475],{},[122,10023,127],{},[129,10025,10026],{"encoding":131},"P_{\\theta}",[100,10028,10030],{"className":10029,"ariaHidden":137},[136],[100,10031,10033,10037],{"className":10032},[141],[100,10034],{"className":10035,"style":10036},[145],"height:0.8333em;vertical-align:-0.15em;",[100,10038,10040,10043],{"className":10039},[150],[100,10041,1475],{"className":10042,"style":1436},[150,154],[100,10044,10046],{"className":10045},[159],[100,10047,10049,10073],{"className":10048},[163,164],[100,10050,10052,10070],{"className":10051},[168],[100,10053,10055],{"className":10054,"style":173},[172],[100,10056,10058,10061],{"style":10057},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[100,10059],{"className":10060,"style":181},[180],[100,10062,10064],{"className":10063},[185,186,187,188],[100,10065,10067],{"className":10066},[150,188],[100,10068,127],{"className":10069,"style":195},[150,154,188],[100,10071,200],{"className":10072},[199],[100,10074,10076],{"className":10075},[168],[100,10077,10079],{"className":10078,"style":207},[172],[100,10080],{}," 是随机变量 ",[100,10083,10085,10098],{"className":10084},[103],[100,10086,10088],{"className":10087},[107],[109,10089,10090],{"xmlns":111},[113,10091,10092,10096],{},[116,10093,10094],{},[122,10095,2099],{},[129,10097,2099],{"encoding":131},[100,10099,10101],{"className":10100,"ariaHidden":137},[136],[100,10102,10104,10107],{"className":10103},[141],[100,10105],{"className":10106,"style":1277},[145],[100,10108,2099],{"className":10109},[150,154]," 上一个带参数的概率分布，那么：",[100,10112,10114],{"className":10113},[555],[100,10115,10117,10181],{"className":10116},[103],[100,10118,10120],{"className":10119},[107],[109,10121,10122],{"xmlns":111,"display":564},[113,10123,10124,10178],{},[116,10125,10126,10146,10174,10176],{},[122,10127,10128],{},[246,10129,10130,10134],{},[227,10131,10132],{},[122,10133,253],{"mathvariant":252},[116,10135,10136,10138,10140],{},[122,10137,2099],{},[227,10139,261],{},[119,10141,10142,10144],{},[122,10143,1475],{},[122,10145,127],{},[116,10147,10148,10150,10156,10158,10160,10166,10168,10170,10172],{},[227,10149,270],{"fence":137},[119,10151,10152,10154],{},[122,10153,607],{"mathvariant":606},[122,10155,127],{},[122,10157,2093],{},[227,10159,2096],{},[119,10161,10162,10164],{},[122,10163,1475],{},[122,10165,127],{},[227,10167,230],{"stretchy":229},[122,10169,2099],{},[227,10171,239],{"stretchy":229},[227,10173,282],{"fence":137},[227,10175,242],{},[582,10177,1229],{},[129,10179,10180],{"encoding":131},"\\underset{x \\sim P_{\\theta}}{\\mathbb{E}}\\left[\\nabla_{\\theta} \\log P_{\\theta}(x)\\right] = 0",[100,10182,10184,10417],{"className":10183,"ariaHidden":137},[136],[100,10185,10187,10191,10290,10293,10408,10411,10414],{"className":10186},[141],[100,10188],{"className":10189,"style":10190},[145],"height:1.6002em;vertical-align:-0.8502em;",[100,10192,10194],{"className":10193},[150],[100,10195,10197],{"className":10196},[376,377],[100,10198,10200,10281],{"className":10199},[163,164],[100,10201,10203,10278],{"className":10202},[168],[100,10204,10206,10268],{"className":10205,"style":387},[172],[100,10207,10209,10212],{"style":10208},"top:-2.3557em;margin-left:0em;",[100,10210],{"className":10211,"style":394},[180],[100,10213,10215],{"className":10214},[185,186,187,188],[100,10216,10218,10221,10224],{"className":10217},[150,188],[100,10219,2099],{"className":10220},[150,154,188],[100,10222,261],{"className":10223},[359,188],[100,10225,10227,10230],{"className":10226},[150,188],[100,10228,1475],{"className":10229,"style":1436},[150,154,188],[100,10231,10233],{"className":10232},[159],[100,10234,10236,10260],{"className":10235},[163,164],[100,10237,10239,10257],{"className":10238},[168],[100,10240,10242],{"className":10241,"style":426},[172],[100,10243,10245,10248],{"style":10244},"top:-2.3488em;margin-left:-0.1389em;margin-right:0.0714em;",[100,10246],{"className":10247,"style":433},[180],[100,10249,10251],{"className":10250},[185,437,438,188],[100,10252,10254],{"className":10253},[150,188],[100,10255,127],{"className":10256,"style":195},[150,154,188],[100,10258,200],{"className":10259},[199],[100,10261,10263],{"className":10262},[168],[100,10264,10266],{"className":10265,"style":454},[172],[100,10267],{},[100,10269,10270,10273],{"style":459},[100,10271],{"className":10272,"style":394},[180],[100,10274,10275],{},[100,10276,253],{"className":10277},[376,468],[100,10279,200],{"className":10280},[199],[100,10282,10284],{"className":10283},[168],[100,10285,10288],{"className":10286,"style":10287},[172],"height:0.8502em;",[100,10289],{},[100,10291],{"className":10292,"style":777},[354],[100,10294,10296,10299,10342,10345,10350,10353,10396,10399,10402,10405],{"className":10295},[781],[100,10297,270],{"className":10298,"style":889},[303,888],[100,10300,10302,10305],{"className":10301},[150],[100,10303,607],{"className":10304},[150],[100,10306,10308],{"className":10307},[159],[100,10309,10311,10334],{"className":10310},[163,164],[100,10312,10314,10331],{"className":10313},[168],[100,10315,10317],{"className":10316,"style":173},[172],[100,10318,10319,10322],{"style":809},[100,10320],{"className":10321,"style":181},[180],[100,10323,10325],{"className":10324},[185,186,187,188],[100,10326,10328],{"className":10327},[150,188],[100,10329,127],{"className":10330,"style":195},[150,154,188],[100,10332,200],{"className":10333},[199],[100,10335,10337],{"className":10336},[168],[100,10338,10340],{"className":10339,"style":207},[172],[100,10341],{},[100,10343],{"className":10344,"style":777},[354],[100,10346,2116,10348],{"className":10347},[376],[100,10349,2120],{"style":2119},[100,10351],{"className":10352,"style":777},[354],[100,10354,10356,10359],{"className":10355},[150],[100,10357,1475],{"className":10358,"style":1436},[150,154],[100,10360,10362],{"className":10361},[159],[100,10363,10365,10388],{"className":10364},[163,164],[100,10366,10368,10385],{"className":10367},[168],[100,10369,10371],{"className":10370,"style":173},[172],[100,10372,10373,10376],{"style":10057},[100,10374],{"className":10375,"style":181},[180],[100,10377,10379],{"className":10378},[185,186,187,188],[100,10380,10382],{"className":10381},[150,188],[100,10383,127],{"className":10384,"style":195},[150,154,188],[100,10386,200],{"className":10387},[199],[100,10389,10391],{"className":10390},[168],[100,10392,10394],{"className":10393,"style":207},[172],[100,10395],{},[100,10397,230],{"className":10398},[303],[100,10400,2099],{"className":10401},[150,154],[100,10403,239],{"className":10404},[350],[100,10406,282],{"className":10407,"style":889},[350,888],[100,10409],{"className":10410,"style":355},[354],[100,10412,242],{"className":10413},[359],[100,10415],{"className":10416,"style":355},[354],[100,10418,10420,10424],{"className":10419},[141],[100,10421],{"className":10422,"style":10423},[145],"height:0.6444em;",[100,10425,1229],{"className":10426},[150],[10,10428,10430,10437,10600,10603,10888,10891],{"title":10429},"证明",[14,10431,10432,10433,10436],{},"回忆所有概率分布都是",[9291,10434,10435],{},"归一化","的：",[100,10438,10440],{"className":10439},[555],[100,10441,10443,10477],{"className":10442},[103],[100,10444,10446],{"className":10445},[107],[109,10447,10448],{"xmlns":111,"display":564},[113,10449,10450,10474],{},[116,10451,10452,10458,10464,10466,10468,10470,10472],{},[119,10453,10454,10456],{},[227,10455,5153],{},[122,10457,2099],{},[119,10459,10460,10462],{},[122,10461,1475],{},[122,10463,127],{},[227,10465,230],{"stretchy":229},[122,10467,2099],{},[227,10469,239],{"stretchy":229},[227,10471,242],{},[582,10473,584],{},[129,10475,10476],{"encoding":131},"\\int_x P_{\\theta}(x) = 1",[100,10478,10480,10591],{"className":10479,"ariaHidden":137},[136],[100,10481,10483,10487,10527,10530,10573,10576,10579,10582,10585,10588],{"className":10482},[141],[100,10484],{"className":10485,"style":10486},[145],"height:2.2719em;vertical-align:-0.9119em;",[100,10488,10490,10493],{"className":10489},[376],[100,10491,5153],{"className":10492,"style":6126},[376,1761,1762],[100,10494,10496],{"className":10495},[159],[100,10497,10499,10519],{"className":10498},[163,164],[100,10500,10502,10516],{"className":10501},[168],[100,10503,10505],{"className":10504,"style":6139},[172],[100,10506,10507,10510],{"style":6142},[100,10508],{"className":10509,"style":181},[180],[100,10511,10513],{"className":10512},[185,186,187,188],[100,10514,2099],{"className":10515},[150,154,188],[100,10517,200],{"className":10518},[199],[100,10520,10522],{"className":10521},[168],[100,10523,10525],{"className":10524,"style":6164},[172],[100,10526],{},[100,10528],{"className":10529,"style":777},[354],[100,10531,10533,10536],{"className":10532},[150],[100,10534,1475],{"className":10535,"style":1436},[150,154],[100,10537,10539],{"className":10538},[159],[100,10540,10542,10565],{"className":10541},[163,164],[100,10543,10545,10562],{"className":10544},[168],[100,10546,10548],{"className":10547,"style":173},[172],[100,10549,10550,10553],{"style":10057},[100,10551],{"className":10552,"style":181},[180],[100,10554,10556],{"className":10555},[185,186,187,188],[100,10557,10559],{"className":10558},[150,188],[100,10560,127],{"className":10561,"style":195},[150,154,188],[100,10563,200],{"className":10564},[199],[100,10566,10568],{"className":10567},[168],[100,10569,10571],{"className":10570,"style":207},[172],[100,10572],{},[100,10574,230],{"className":10575},[303],[100,10577,2099],{"className":10578},[150,154],[100,10580,239],{"className":10581},[350],[100,10583],{"className":10584,"style":355},[354],[100,10586,242],{"className":10587},[359],[100,10589],{"className":10590,"style":355},[354],[100,10592,10594,10597],{"className":10593},[141],[100,10595],{"className":10596,"style":10423},[145],[100,10598,584],{"className":10599},[150],[14,10601,10602],{},"对归一化条件两边取梯度：",[100,10604,10606],{"className":10605},[555],[100,10607,10609,10659],{"className":10608},[103],[100,10610,10612],{"className":10611},[107],[109,10613,10614],{"xmlns":111,"display":564},[113,10615,10616,10656],{},[116,10617,10618,10624,10630,10636,10638,10640,10642,10644,10650,10652,10654],{},[119,10619,10620,10622],{},[122,10621,607],{"mathvariant":606},[122,10623,127],{},[119,10625,10626,10628],{},[227,10627,5153],{},[122,10629,2099],{},[119,10631,10632,10634],{},[122,10633,1475],{},[122,10635,127],{},[227,10637,230],{"stretchy":229},[122,10639,2099],{},[227,10641,239],{"stretchy":229},[227,10643,242],{},[119,10645,10646,10648],{},[122,10647,607],{"mathvariant":606},[122,10649,127],{},[582,10651,584],{},[227,10653,242],{},[582,10655,1229],{},[129,10657,10658],{"encoding":131},"\\nabla_{\\theta} \\int_x P_{\\theta}(x) = \\nabla_{\\theta} 1 = 0",[100,10660,10662,10818,10879],{"className":10661,"ariaHidden":137},[136],[100,10663,10665,10668,10711,10714,10754,10757,10800,10803,10806,10809,10812,10815],{"className":10664},[141],[100,10666],{"className":10667,"style":10486},[145],[100,10669,10671,10674],{"className":10670},[150],[100,10672,607],{"className":10673},[150],[100,10675,10677],{"className":10676},[159],[100,10678,10680,10703],{"className":10679},[163,164],[100,10681,10683,10700],{"className":10682},[168],[100,10684,10686],{"className":10685,"style":173},[172],[100,10687,10688,10691],{"style":809},[100,10689],{"className":10690,"style":181},[180],[100,10692,10694],{"className":10693},[185,186,187,188],[100,10695,10697],{"className":10696},[150,188],[100,10698,127],{"className":10699,"style":195},[150,154,188],[100,10701,200],{"className":10702},[199],[100,10704,10706],{"className":10705},[168],[100,10707,10709],{"className":10708,"style":207},[172],[100,10710],{},[100,10712],{"className":10713,"style":777},[354],[100,10715,10717,10720],{"className":10716},[376],[100,10718,5153],{"className":10719,"style":6126},[376,1761,1762],[100,10721,10723],{"className":10722},[159],[100,10724,10726,10746],{"className":10725},[163,164],[100,10727,10729,10743],{"className":10728},[168],[100,10730,10732],{"className":10731,"style":6139},[172],[100,10733,10734,10737],{"style":6142},[100,10735],{"className":10736,"style":181},[180],[100,10738,10740],{"className":10739},[185,186,187,188],[100,10741,2099],{"className":10742},[150,154,188],[100,10744,200],{"className":10745},[199],[100,10747,10749],{"className":10748},[168],[100,10750,10752],{"className":10751,"style":6164},[172],[100,10753],{},[100,10755],{"className":10756,"style":777},[354],[100,10758,10760,10763],{"className":10759},[150],[100,10761,1475],{"className":10762,"style":1436},[150,154],[100,10764,10766],{"className":10765},[159],[100,10767,10769,10792],{"className":10768},[163,164],[100,10770,10772,10789],{"className":10771},[168],[100,10773,10775],{"className":10774,"style":173},[172],[100,10776,10777,10780],{"style":10057},[100,10778],{"className":10779,"style":181},[180],[100,10781,10783],{"className":10782},[185,186,187,188],[100,10784,10786],{"className":10785},[150,188],[100,10787,127],{"className":10788,"style":195},[150,154,188],[100,10790,200],{"className":10791},[199],[100,10793,10795],{"className":10794},[168],[100,10796,10798],{"className":10797,"style":207},[172],[100,10799],{},[100,10801,230],{"className":10802},[303],[100,10804,2099],{"className":10805},[150,154],[100,10807,239],{"className":10808},[350],[100,10810],{"className":10811,"style":355},[354],[100,10813,242],{"className":10814},[359],[100,10816],{"className":10817,"style":355},[354],[100,10819,10821,10824,10867,10870,10873,10876],{"className":10820},[141],[100,10822],{"className":10823,"style":10036},[145],[100,10825,10827,10830],{"className":10826},[150],[100,10828,607],{"className":10829},[150],[100,10831,10833],{"className":10832},[159],[100,10834,10836,10859],{"className":10835},[163,164],[100,10837,10839,10856],{"className":10838},[168],[100,10840,10842],{"className":10841,"style":173},[172],[100,10843,10844,10847],{"style":809},[100,10845],{"className":10846,"style":181},[180],[100,10848,10850],{"className":10849},[185,186,187,188],[100,10851,10853],{"className":10852},[150,188],[100,10854,127],{"className":10855,"style":195},[150,154,188],[100,10857,200],{"className":10858},[199],[100,10860,10862],{"className":10861},[168],[100,10863,10865],{"className":10864,"style":207},[172],[100,10866],{},[100,10868,584],{"className":10869},[150],[100,10871],{"className":10872,"style":355},[354],[100,10874,242],{"className":10875},[359],[100,10877],{"className":10878,"style":355},[354],[100,10880,10882,10885],{"className":10881},[141],[100,10883],{"className":10884,"style":10423},[145],[100,10886,1229],{"className":10887},[150],[14,10889,10890],{},"用对数求导技巧得到：",[100,10892,10894],{"className":10893},[555],[100,10895,10897,11121],{"className":10896},[103],[100,10898,10900],{"className":10899},[107],[109,10901,10902],{"xmlns":111,"display":564},[113,10903,10904,11118],{},[3584,10905,10906,10948,10990,11048],{"rowspacing":3586,"columnalign":3587,"columnspacing":3588},[3590,10907,10908,10914],{},[3593,10909,10910],{},[3596,10911,10912],{"scriptlevel":1229,"displaystyle":137},[582,10913,1229],{},[3593,10915,10916],{},[3596,10917,10918],{"scriptlevel":1229,"displaystyle":137},[116,10919,10920,10922,10924,10930,10936,10942,10944,10946],{},[116,10921],{},[227,10923,242],{},[119,10925,10926,10928],{},[122,10927,607],{"mathvariant":606},[122,10929,127],{},[119,10931,10932,10934],{},[227,10933,5153],{},[122,10935,2099],{},[119,10937,10938,10940],{},[122,10939,1475],{},[122,10941,127],{},[227,10943,230],{"stretchy":229},[122,10945,2099],{},[227,10947,239],{"stretchy":229},[3590,10949,10950,10956],{},[3593,10951,10952],{},[3596,10953,10954],{"scriptlevel":1229,"displaystyle":137},[116,10955],{},[3593,10957,10958],{},[3596,10959,10960],{"scriptlevel":1229,"displaystyle":137},[116,10961,10962,10964,10966,10972,10978,10984,10986,10988],{},[116,10963],{},[227,10965,242],{},[119,10967,10968,10970],{},[227,10969,5153],{},[122,10971,2099],{},[119,10973,10974,10976],{},[122,10975,607],{"mathvariant":606},[122,10977,127],{},[119,10979,10980,10982],{},[122,10981,1475],{},[122,10983,127],{},[227,10985,230],{"stretchy":229},[122,10987,2099],{},[227,10989,239],{"stretchy":229},[3590,10991,10992,10998],{},[3593,10993,10994],{},[3596,10995,10996],{"scriptlevel":1229,"displaystyle":137},[116,10997],{},[3593,10999,11000],{},[3596,11001,11002],{"scriptlevel":1229,"displaystyle":137},[116,11003,11004,11006,11008,11014,11020,11022,11024,11026,11032,11034,11036,11042,11044,11046],{},[116,11005],{},[227,11007,242],{},[119,11009,11010,11012],{},[227,11011,5153],{},[122,11013,2099],{},[119,11015,11016,11018],{},[122,11017,1475],{},[122,11019,127],{},[227,11021,230],{"stretchy":229},[122,11023,2099],{},[227,11025,239],{"stretchy":229},[119,11027,11028,11030],{},[122,11029,607],{"mathvariant":606},[122,11031,127],{},[122,11033,2093],{},[227,11035,2096],{},[119,11037,11038,11040],{},[122,11039,1475],{},[122,11041,127],{},[227,11043,230],{"stretchy":229},[122,11045,2099],{},[227,11047,239],{"stretchy":229},[3590,11049,11050,11060],{},[3593,11051,11052],{},[3596,11053,11054],{"scriptlevel":1229,"displaystyle":137},[116,11055,11056,11058],{},[227,11057,5445],{},[582,11059,1229],{},[3593,11061,11062],{},[3596,11063,11064],{"scriptlevel":1229,"displaystyle":137},[116,11065,11066,11068,11070,11090],{},[116,11067],{},[227,11069,242],{},[122,11071,11072],{},[246,11073,11074,11078],{},[227,11075,11076],{},[122,11077,253],{"mathvariant":252},[116,11079,11080,11082,11084],{},[122,11081,2099],{},[227,11083,261],{},[119,11085,11086,11088],{},[122,11087,1475],{},[122,11089,127],{},[116,11091,11092,11094,11100,11102,11104,11110,11112,11114,11116],{},[227,11093,270],{"fence":137},[119,11095,11096,11098],{},[122,11097,607],{"mathvariant":606},[122,11099,127],{},[122,11101,2093],{},[227,11103,2096],{},[119,11105,11106,11108],{},[122,11107,1475],{},[122,11109,127],{},[227,11111,230],{"stretchy":229},[122,11113,2099],{},[227,11115,239],{"stretchy":229},[227,11117,282],{"fence":137},[129,11119,11120],{"encoding":131},"\\begin{aligned}\n0 &= \\nabla_{\\theta} \\int_x P_{\\theta}(x) \\\\\n&= \\int_x \\nabla_{\\theta} P_{\\theta}(x) \\\\\n&= \\int_x P_{\\theta}(x) \\nabla_{\\theta} \\log P_{\\theta}(x) \\\\\n\\therefore 0 &= \\underset{x \\sim P_{\\theta}}{\\mathbb{E}}\\left[\\nabla_{\\theta} \\log P_{\\theta}(x)\\right]\n\\end{aligned}",[100,11122,11124],{"className":11123,"ariaHidden":137},[136],[100,11125,11127,11131],{"className":11126},[141],[100,11128],{"className":11129,"style":11130},[145],"height:9.406em;vertical-align:-4.453em;",[100,11132,11134],{"className":11133},[150],[100,11135,11137,11211],{"className":11136},[3584],[100,11138,11140],{"className":11139},[3856],[100,11141,11143,11202],{"className":11142},[163,164],[100,11144,11146,11199],{"className":11145},[168],[100,11147,11150,11163,11172,11181],{"className":11148,"style":11149},[172],"height:4.953em;",[100,11151,11153,11157],{"style":11152},"top:-6.953em;",[100,11154],{"className":11155,"style":11156},[180],"height:3.36em;",[100,11158,11160],{"className":11159},[150],[100,11161,1229],{"className":11162},[150],[100,11164,11166,11169],{"style":11165},"top:-4.3811em;",[100,11167],{"className":11168,"style":11156},[180],[100,11170],{"className":11171},[150],[100,11173,11175,11178],{"style":11174},"top:-1.8091em;",[100,11176],{"className":11177,"style":11156},[180],[100,11179],{"className":11180},[150],[100,11182,11184,11187],{"style":11183},"top:0.2428em;",[100,11185],{"className":11186,"style":11156},[180],[100,11188,11190,11193,11196],{"className":11189},[150],[100,11191,5445],{"className":11192},[359,5757],[100,11194],{"className":11195,"style":355},[354],[100,11197,1229],{"className":11198},[150],[100,11200,200],{"className":11201},[199],[100,11203,11205],{"className":11204},[168],[100,11206,11209],{"className":11207,"style":11208},[172],"height:4.453em;",[100,11210],{},[100,11212,11214],{"className":11213},[3973],[100,11215,11217,12000],{"className":11216},[163,164],[100,11218,11220,11997],{"className":11219},[168],[100,11221,11223,11384,11542,11763],{"className":11222,"style":11149},[172],[100,11224,11225,11228],{"style":11152},[100,11226],{"className":11227,"style":11156},[180],[100,11229,11231,11234,11237,11240,11243,11286,11289,11329,11332,11375,11378,11381],{"className":11230},[150],[100,11232],{"className":11233},[150],[100,11235],{"className":11236,"style":355},[354],[100,11238,242],{"className":11239},[359],[100,11241],{"className":11242,"style":355},[354],[100,11244,11246,11249],{"className":11245},[150],[100,11247,607],{"className":11248},[150],[100,11250,11252],{"className":11251},[159],[100,11253,11255,11278],{"className":11254},[163,164],[100,11256,11258,11275],{"className":11257},[168],[100,11259,11261],{"className":11260,"style":173},[172],[100,11262,11263,11266],{"style":809},[100,11264],{"className":11265,"style":181},[180],[100,11267,11269],{"className":11268},[185,186,187,188],[100,11270,11272],{"className":11271},[150,188],[100,11273,127],{"className":11274,"style":195},[150,154,188],[100,11276,200],{"className":11277},[199],[100,11279,11281],{"className":11280},[168],[100,11282,11284],{"className":11283,"style":207},[172],[100,11285],{},[100,11287],{"className":11288,"style":777},[354],[100,11290,11292,11295],{"className":11291},[376],[100,11293,5153],{"className":11294,"style":6126},[376,1761,1762],[100,11296,11298],{"className":11297},[159],[100,11299,11301,11321],{"className":11300},[163,164],[100,11302,11304,11318],{"className":11303},[168],[100,11305,11307],{"className":11306,"style":6139},[172],[100,11308,11309,11312],{"style":6142},[100,11310],{"className":11311,"style":181},[180],[100,11313,11315],{"className":11314},[185,186,187,188],[100,11316,2099],{"className":11317},[150,154,188],[100,11319,200],{"className":11320},[199],[100,11322,11324],{"className":11323},[168],[100,11325,11327],{"className":11326,"style":6164},[172],[100,11328],{},[100,11330],{"className":11331,"style":777},[354],[100,11333,11335,11338],{"className":11334},[150],[100,11336,1475],{"className":11337,"style":1436},[150,154],[100,11339,11341],{"className":11340},[159],[100,11342,11344,11367],{"className":11343},[163,164],[100,11345,11347,11364],{"className":11346},[168],[100,11348,11350],{"className":11349,"style":173},[172],[100,11351,11352,11355],{"style":10057},[100,11353],{"className":11354,"style":181},[180],[100,11356,11358],{"className":11357},[185,186,187,188],[100,11359,11361],{"className":11360},[150,188],[100,11362,127],{"className":11363,"style":195},[150,154,188],[100,11365,200],{"className":11366},[199],[100,11368,11370],{"className":11369},[168],[100,11371,11373],{"className":11372,"style":207},[172],[100,11374],{},[100,11376,230],{"className":11377},[303],[100,11379,2099],{"className":11380},[150,154],[100,11382,239],{"className":11383},[350],[100,11385,11386,11389],{"style":11165},[100,11387],{"className":11388,"style":11156},[180],[100,11390,11392,11395,11398,11401,11404,11444,11447,11490,11533,11536,11539],{"className":11391},[150],[100,11393],{"className":11394},[150],[100,11396],{"className":11397,"style":355},[354],[100,11399,242],{"className":11400},[359],[100,11402],{"className":11403,"style":355},[354],[100,11405,11407,11410],{"className":11406},[376],[100,11408,5153],{"className":11409,"style":6126},[376,1761,1762],[100,11411,11413],{"className":11412},[159],[100,11414,11416,11436],{"className":11415},[163,164],[100,11417,11419,11433],{"className":11418},[168],[100,11420,11422],{"className":11421,"style":6139},[172],[100,11423,11424,11427],{"style":6142},[100,11425],{"className":11426,"style":181},[180],[100,11428,11430],{"className":11429},[185,186,187,188],[100,11431,2099],{"className":11432},[150,154,188],[100,11434,200],{"className":11435},[199],[100,11437,11439],{"className":11438},[168],[100,11440,11442],{"className":11441,"style":6164},[172],[100,11443],{},[100,11445],{"className":11446,"style":777},[354],[100,11448,11450,11453],{"className":11449},[150],[100,11451,607],{"className":11452},[150],[100,11454,11456],{"className":11455},[159],[100,11457,11459,11482],{"className":11458},[163,164],[100,11460,11462,11479],{"className":11461},[168],[100,11463,11465],{"className":11464,"style":173},[172],[100,11466,11467,11470],{"style":809},[100,11468],{"className":11469,"style":181},[180],[100,11471,11473],{"className":11472},[185,186,187,188],[100,11474,11476],{"className":11475},[150,188],[100,11477,127],{"className":11478,"style":195},[150,154,188],[100,11480,200],{"className":11481},[199],[100,11483,11485],{"className":11484},[168],[100,11486,11488],{"className":11487,"style":207},[172],[100,11489],{},[100,11491,11493,11496],{"className":11492},[150],[100,11494,1475],{"className":11495,"style":1436},[150,154],[100,11497,11499],{"className":11498},[159],[100,11500,11502,11525],{"className":11501},[163,164],[100,11503,11505,11522],{"className":11504},[168],[100,11506,11508],{"className":11507,"style":173},[172],[100,11509,11510,11513],{"style":10057},[100,11511],{"className":11512,"style":181},[180],[100,11514,11516],{"className":11515},[185,186,187,188],[100,11517,11519],{"className":11518},[150,188],[100,11520,127],{"className":11521,"style":195},[150,154,188],[100,11523,200],{"className":11524},[199],[100,11526,11528],{"className":11527},[168],[100,11529,11531],{"className":11530,"style":207},[172],[100,11532],{},[100,11534,230],{"className":11535},[303],[100,11537,2099],{"className":11538},[150,154],[100,11540,239],{"className":11541},[350],[100,11543,11544,11547],{"style":11174},[100,11545],{"className":11546,"style":11156},[180],[100,11548,11550,11553,11556,11559,11562,11602,11605,11648,11651,11654,11657,11700,11703,11708,11711,11754,11757,11760],{"className":11549},[150],[100,11551],{"className":11552},[150],[100,11554],{"className":11555,"style":355},[354],[100,11557,242],{"className":11558},[359],[100,11560],{"className":11561,"style":355},[354],[100,11563,11565,11568],{"className":11564},[376],[100,11566,5153],{"className":11567,"style":6126},[376,1761,1762],[100,11569,11571],{"className":11570},[159],[100,11572,11574,11594],{"className":11573},[163,164],[100,11575,11577,11591],{"className":11576},[168],[100,11578,11580],{"className":11579,"style":6139},[172],[100,11581,11582,11585],{"style":6142},[100,11583],{"className":11584,"style":181},[180],[100,11586,11588],{"className":11587},[185,186,187,188],[100,11589,2099],{"className":11590},[150,154,188],[100,11592,200],{"className":11593},[199],[100,11595,11597],{"className":11596},[168],[100,11598,11600],{"className":11599,"style":6164},[172],[100,11601],{},[100,11603],{"className":11604,"style":777},[354],[100,11606,11608,11611],{"className":11607},[150],[100,11609,1475],{"className":11610,"style":1436},[150,154],[100,11612,11614],{"className":11613},[159],[100,11615,11617,11640],{"className":11616},[163,164],[100,11618,11620,11637],{"className":11619},[168],[100,11621,11623],{"className":11622,"style":173},[172],[100,11624,11625,11628],{"style":10057},[100,11626],{"className":11627,"style":181},[180],[100,11629,11631],{"className":11630},[185,186,187,188],[100,11632,11634],{"className":11633},[150,188],[100,11635,127],{"className":11636,"style":195},[150,154,188],[100,11638,200],{"className":11639},[199],[100,11641,11643],{"className":11642},[168],[100,11644,11646],{"className":11645,"style":207},[172],[100,11647],{},[100,11649,230],{"className":11650},[303],[100,11652,2099],{"className":11653},[150,154],[100,11655,239],{"className":11656},[350],[100,11658,11660,11663],{"className":11659},[150],[100,11661,607],{"className":11662},[150],[100,11664,11666],{"className":11665},[159],[100,11667,11669,11692],{"className":11668},[163,164],[100,11670,11672,11689],{"className":11671},[168],[100,11673,11675],{"className":11674,"style":173},[172],[100,11676,11677,11680],{"style":809},[100,11678],{"className":11679,"style":181},[180],[100,11681,11683],{"className":11682},[185,186,187,188],[100,11684,11686],{"className":11685},[150,188],[100,11687,127],{"className":11688,"style":195},[150,154,188],[100,11690,200],{"className":11691},[199],[100,11693,11695],{"className":11694},[168],[100,11696,11698],{"className":11697,"style":207},[172],[100,11699],{},[100,11701],{"className":11702,"style":777},[354],[100,11704,2116,11706],{"className":11705},[376],[100,11707,2120],{"style":2119},[100,11709],{"className":11710,"style":777},[354],[100,11712,11714,11717],{"className":11713},[150],[100,11715,1475],{"className":11716,"style":1436},[150,154],[100,11718,11720],{"className":11719},[159],[100,11721,11723,11746],{"className":11722},[163,164],[100,11724,11726,11743],{"className":11725},[168],[100,11727,11729],{"className":11728,"style":173},[172],[100,11730,11731,11734],{"style":10057},[100,11732],{"className":11733,"style":181},[180],[100,11735,11737],{"className":11736},[185,186,187,188],[100,11738,11740],{"className":11739},[150,188],[100,11741,127],{"className":11742,"style":195},[150,154,188],[100,11744,200],{"className":11745},[199],[100,11747,11749],{"className":11748},[168],[100,11750,11752],{"className":11751,"style":207},[172],[100,11753],{},[100,11755,230],{"className":11756},[303],[100,11758,2099],{"className":11759},[150,154],[100,11761,239],{"className":11762},[350],[100,11764,11765,11768],{"style":11183},[100,11766],{"className":11767,"style":11156},[180],[100,11769,11771,11774,11777,11780,11783,11879,11882],{"className":11770},[150],[100,11772],{"className":11773},[150],[100,11775],{"className":11776,"style":355},[354],[100,11778,242],{"className":11779},[359],[100,11781],{"className":11782,"style":355},[354],[100,11784,11786],{"className":11785},[150],[100,11787,11789],{"className":11788},[376,377],[100,11790,11792,11871],{"className":11791},[163,164],[100,11793,11795,11868],{"className":11794},[168],[100,11796,11798,11858],{"className":11797,"style":387},[172],[100,11799,11800,11803],{"style":10208},[100,11801],{"className":11802,"style":394},[180],[100,11804,11806],{"className":11805},[185,186,187,188],[100,11807,11809,11812,11815],{"className":11808},[150,188],[100,11810,2099],{"className":11811},[150,154,188],[100,11813,261],{"className":11814},[359,188],[100,11816,11818,11821],{"className":11817},[150,188],[100,11819,1475],{"className":11820,"style":1436},[150,154,188],[100,11822,11824],{"className":11823},[159],[100,11825,11827,11850],{"className":11826},[163,164],[100,11828,11830,11847],{"className":11829},[168],[100,11831,11833],{"className":11832,"style":426},[172],[100,11834,11835,11838],{"style":10244},[100,11836],{"className":11837,"style":433},[180],[100,11839,11841],{"className":11840},[185,437,438,188],[100,11842,11844],{"className":11843},[150,188],[100,11845,127],{"className":11846,"style":195},[150,154,188],[100,11848,200],{"className":11849},[199],[100,11851,11853],{"className":11852},[168],[100,11854,11856],{"className":11855,"style":454},[172],[100,11857],{},[100,11859,11860,11863],{"style":459},[100,11861],{"className":11862,"style":394},[180],[100,11864,11865],{},[100,11866,253],{"className":11867},[376,468],[100,11869,200],{"className":11870},[199],[100,11872,11874],{"className":11873},[168],[100,11875,11877],{"className":11876,"style":10287},[172],[100,11878],{},[100,11880],{"className":11881,"style":777},[354],[100,11883,11885,11888,11931,11934,11939,11942,11985,11988,11991,11994],{"className":11884},[781],[100,11886,270],{"className":11887,"style":889},[303,888],[100,11889,11891,11894],{"className":11890},[150],[100,11892,607],{"className":11893},[150],[100,11895,11897],{"className":11896},[159],[100,11898,11900,11923],{"className":11899},[163,164],[100,11901,11903,11920],{"className":11902},[168],[100,11904,11906],{"className":11905,"style":173},[172],[100,11907,11908,11911],{"style":809},[100,11909],{"className":11910,"style":181},[180],[100,11912,11914],{"className":11913},[185,186,187,188],[100,11915,11917],{"className":11916},[150,188],[100,11918,127],{"className":11919,"style":195},[150,154,188],[100,11921,200],{"className":11922},[199],[100,11924,11926],{"className":11925},[168],[100,11927,11929],{"className":11928,"style":207},[172],[100,11930],{},[100,11932],{"className":11933,"style":777},[354],[100,11935,2116,11937],{"className":11936},[376],[100,11938,2120],{"style":2119},[100,11940],{"className":11941,"style":777},[354],[100,11943,11945,11948],{"className":11944},[150],[100,11946,1475],{"className":11947,"style":1436},[150,154],[100,11949,11951],{"className":11950},[159],[100,11952,11954,11977],{"className":11953},[163,164],[100,11955,11957,11974],{"className":11956},[168],[100,11958,11960],{"className":11959,"style":173},[172],[100,11961,11962,11965],{"style":10057},[100,11963],{"className":11964,"style":181},[180],[100,11966,11968],{"className":11967},[185,186,187,188],[100,11969,11971],{"className":11970},[150,188],[100,11972,127],{"className":11973,"style":195},[150,154,188],[100,11975,200],{"className":11976},[199],[100,11978,11980],{"className":11979},[168],[100,11981,11983],{"className":11982,"style":207},[172],[100,11984],{},[100,11986,230],{"className":11987},[303],[100,11989,2099],{"className":11990},[150,154],[100,11992,239],{"className":11993},[350],[100,11995,282],{"className":11996,"style":889},[350,888],[100,11998,200],{"className":11999},[199],[100,12001,12003],{"className":12002},[168],[100,12004,12006],{"className":12005,"style":11208},[172],[100,12007],{},[10,12009,12011],{"title":12010},"注",[14,12012,12013],{},"本文作者不知道这个引理在文献里是否有标准名字。但它出现得太频繁了，值得给它起个名字，方便引用。",[93,12015,12016],{"id":12016},"别让过去干扰你",[14,12018,12019],{},"看看我们刚才的策略梯度表达式：",[100,12021,12023],{"className":12022},[555],[100,12024,12026,12140],{"className":12025},[103],[100,12027,12029],{"className":12028},[107],[109,12030,12031],{"xmlns":111,"display":564},[113,12032,12033,12137],{},[116,12034,12035,12041,12043,12045,12051,12053,12055,12075],{},[119,12036,12037,12039],{},[122,12038,607],{"mathvariant":606},[122,12040,127],{},[122,12042,225],{},[227,12044,230],{"stretchy":229},[119,12046,12047,12049],{},[122,12048,124],{},[122,12050,127],{},[227,12052,239],{"stretchy":229},[227,12054,242],{},[122,12056,12057],{},[246,12058,12059,12063],{},[227,12060,12061],{},[122,12062,253],{"mathvariant":252},[116,12064,12065,12067,12069],{},[122,12066,258],{},[227,12068,261],{},[119,12070,12071,12073],{},[122,12072,124],{},[122,12074,127],{},[116,12076,12077,12079,12093,12099,12101,12103,12109,12111,12117,12119,12125,12127,12129,12131,12133,12135],{},[227,12078,270],{"fence":137},[1506,12080,12081,12083,12091],{},[227,12082,2508],{},[116,12084,12085,12087,12089],{},[122,12086,1515],{},[227,12088,242],{},[582,12090,1229],{},[122,12092,1258],{},[119,12094,12095,12097],{},[122,12096,607],{"mathvariant":606},[122,12098,127],{},[122,12100,2093],{},[227,12102,2096],{},[119,12104,12105,12107],{},[122,12106,124],{},[122,12108,127],{},[227,12110,230],{"stretchy":229},[119,12112,12113,12115],{},[122,12114,18],{},[122,12116,1515],{},[122,12118,624],{"mathvariant":606},[119,12120,12121,12123],{},[122,12122,1226],{},[122,12124,1515],{},[227,12126,239],{"stretchy":229},[122,12128,273],{},[227,12130,230],{"stretchy":229},[122,12132,258],{},[227,12134,239],{"stretchy":229},[227,12136,282],{"fence":137},[129,12138,12139],{"encoding":131},"\\nabla_{\\theta} J(\\pi_{\\theta}) = \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t) R(\\tau)\\right]",[100,12141,12143,12253],{"className":12142,"ariaHidden":137},[136],[100,12144,12146,12149,12192,12195,12198,12241,12244,12247,12250],{"className":12145},[141],[100,12147],{"className":12148,"style":295},[145],[100,12150,12152,12155],{"className":12151},[150],[100,12153,607],{"className":12154},[150],[100,12156,12158],{"className":12157},[159],[100,12159,12161,12184],{"className":12160},[163,164],[100,12162,12164,12181],{"className":12163},[168],[100,12165,12167],{"className":12166,"style":173},[172],[100,12168,12169,12172],{"style":809},[100,12170],{"className":12171,"style":181},[180],[100,12173,12175],{"className":12174},[185,186,187,188],[100,12176,12178],{"className":12177},[150,188],[100,12179,127],{"className":12180,"style":195},[150,154,188],[100,12182,200],{"className":12183},[199],[100,12185,12187],{"className":12186},[168],[100,12188,12190],{"className":12189,"style":207},[172],[100,12191],{},[100,12193,225],{"className":12194,"style":299},[150,154],[100,12196,230],{"className":12197},[303],[100,12199,12201,12204],{"className":12200},[150],[100,12202,124],{"className":12203,"style":155},[150,154],[100,12205,12207],{"className":12206},[159],[100,12208,12210,12233],{"className":12209},[163,164],[100,12211,12213,12230],{"className":12212},[168],[100,12214,12216],{"className":12215,"style":173},[172],[100,12217,12218,12221],{"style":176},[100,12219],{"className":12220,"style":181},[180],[100,12222,12224],{"className":12223},[185,186,187,188],[100,12225,12227],{"className":12226},[150,188],[100,12228,127],{"className":12229,"style":195},[150,154,188],[100,12231,200],{"className":12232},[199],[100,12234,12236],{"className":12235},[168],[100,12237,12239],{"className":12238,"style":207},[172],[100,12240],{},[100,12242,239],{"className":12243},[350],[100,12245],{"className":12246,"style":355},[354],[100,12248,242],{"className":12249},[359],[100,12251],{"className":12252,"style":355},[354],[100,12254,12256,12259,12355,12358],{"className":12255},[141],[100,12257],{"className":12258,"style":1625},[145],[100,12260,12262],{"className":12261},[150],[100,12263,12265],{"className":12264},[376,377],[100,12266,12268,12347],{"className":12267},[163,164],[100,12269,12271,12344],{"className":12270},[168],[100,12272,12274,12334],{"className":12273,"style":387},[172],[100,12275,12276,12279],{"style":390},[100,12277],{"className":12278,"style":394},[180],[100,12280,12282],{"className":12281},[185,186,187,188],[100,12283,12285,12288,12291],{"className":12284},[150,188],[100,12286,258],{"className":12287,"style":404},[150,154,188],[100,12289,261],{"className":12290},[359,188],[100,12292,12294,12297],{"className":12293},[150,188],[100,12295,124],{"className":12296,"style":155},[150,154,188],[100,12298,12300],{"className":12299},[159],[100,12301,12303,12326],{"className":12302},[163,164],[100,12304,12306,12323],{"className":12305},[168],[100,12307,12309],{"className":12308,"style":426},[172],[100,12310,12311,12314],{"style":429},[100,12312],{"className":12313,"style":433},[180],[100,12315,12317],{"className":12316},[185,437,438,188],[100,12318,12320],{"className":12319},[150,188],[100,12321,127],{"className":12322,"style":195},[150,154,188],[100,12324,200],{"className":12325},[199],[100,12327,12329],{"className":12328},[168],[100,12330,12332],{"className":12331,"style":454},[172],[100,12333],{},[100,12335,12336,12339],{"style":459},[100,12337],{"className":12338,"style":394},[180],[100,12340,12341],{},[100,12342,253],{"className":12343},[376,468],[100,12345,200],{"className":12346},[199],[100,12348,12350],{"className":12349},[168],[100,12351,12353],{"className":12352,"style":478},[172],[100,12354],{},[100,12356],{"className":12357,"style":777},[354],[100,12359,12361,12367,12434,12437,12480,12483,12488,12491,12534,12537,12577,12580,12620,12623,12626,12629,12632,12635],{"className":12360},[781],[100,12362,12364],{"className":12363,"style":889},[303,888],[100,12365,270],{"className":12366},[2830,6847],[100,12368,12370],{"className":12369},[376,377],[100,12371,12373,12426],{"className":12372},[163,164],[100,12374,12376,12423],{"className":12375},[168],[100,12377,12379,12399,12409],{"className":12378,"style":1727},[172],[100,12380,12381,12384],{"style":1730},[100,12382],{"className":12383,"style":1734},[180],[100,12385,12387],{"className":12386},[185,186,187,188],[100,12388,12390,12393,12396],{"className":12389},[150,188],[100,12391,1515],{"className":12392},[150,154,188],[100,12394,242],{"className":12395},[359,188],[100,12397,1229],{"className":12398},[150,188],[100,12400,12401,12404],{"style":1752},[100,12402],{"className":12403,"style":1734},[180],[100,12405,12406],{},[100,12407,2508],{"className":12408},[376,1761,1762],[100,12410,12411,12414],{"style":1765},[100,12412],{"className":12413,"style":1734},[180],[100,12415,12417],{"className":12416},[185,186,187,188],[100,12418,12420],{"className":12419},[150,188],[100,12421,1258],{"className":12422,"style":1436},[150,154,188],[100,12424,200],{"className":12425},[199],[100,12427,12429],{"className":12428},[168],[100,12430,12432],{"className":12431,"style":1787},[172],[100,12433],{},[100,12435],{"className":12436,"style":777},[354],[100,12438,12440,12443],{"className":12439},[150],[100,12441,607],{"className":12442},[150],[100,12444,12446],{"className":12445},[159],[100,12447,12449,12472],{"className":12448},[163,164],[100,12450,12452,12469],{"className":12451},[168],[100,12453,12455],{"className":12454,"style":173},[172],[100,12456,12457,12460],{"style":809},[100,12458],{"className":12459,"style":181},[180],[100,12461,12463],{"className":12462},[185,186,187,188],[100,12464,12466],{"className":12465},[150,188],[100,12467,127],{"className":12468,"style":195},[150,154,188],[100,12470,200],{"className":12471},[199],[100,12473,12475],{"className":12474},[168],[100,12476,12478],{"className":12477,"style":207},[172],[100,12479],{},[100,12481],{"className":12482,"style":777},[354],[100,12484,2116,12486],{"className":12485},[376],[100,12487,2120],{"style":2119},[100,12489],{"className":12490,"style":777},[354],[100,12492,12494,12497],{"className":12493},[150],[100,12495,124],{"className":12496,"style":155},[150,154],[100,12498,12500],{"className":12499},[159],[100,12501,12503,12526],{"className":12502},[163,164],[100,12504,12506,12523],{"className":12505},[168],[100,12507,12509],{"className":12508,"style":173},[172],[100,12510,12511,12514],{"style":176},[100,12512],{"className":12513,"style":181},[180],[100,12515,12517],{"className":12516},[185,186,187,188],[100,12518,12520],{"className":12519},[150,188],[100,12521,127],{"className":12522,"style":195},[150,154,188],[100,12524,200],{"className":12525},[199],[100,12527,12529],{"className":12528},[168],[100,12530,12532],{"className":12531,"style":207},[172],[100,12533],{},[100,12535,230],{"className":12536},[303],[100,12538,12540,12543],{"className":12539},[150],[100,12541,18],{"className":12542},[150,154],[100,12544,12546],{"className":12545},[159],[100,12547,12549,12569],{"className":12548},[163,164],[100,12550,12552,12566],{"className":12551},[168],[100,12553,12555],{"className":12554,"style":1869},[172],[100,12556,12557,12560],{"style":809},[100,12558],{"className":12559,"style":181},[180],[100,12561,12563],{"className":12562},[185,186,187,188],[100,12564,1515],{"className":12565},[150,154,188],[100,12567,200],{"className":12568},[199],[100,12570,12572],{"className":12571},[168],[100,12573,12575],{"className":12574,"style":207},[172],[100,12576],{},[100,12578,624],{"className":12579},[150],[100,12581,12583,12586],{"className":12582},[150],[100,12584,1226],{"className":12585},[150,154],[100,12587,12589],{"className":12588},[159],[100,12590,12592,12612],{"className":12591},[163,164],[100,12593,12595,12609],{"className":12594},[168],[100,12596,12598],{"className":12597,"style":1869},[172],[100,12599,12600,12603],{"style":809},[100,12601],{"className":12602,"style":181},[180],[100,12604,12606],{"className":12605},[185,186,187,188],[100,12607,1515],{"className":12608},[150,154,188],[100,12610,200],{"className":12611},[199],[100,12613,12615],{"className":12614},[168],[100,12616,12618],{"className":12617,"style":207},[172],[100,12619],{},[100,12621,239],{"className":12622},[350],[100,12624,273],{"className":12625,"style":487},[150,154],[100,12627,230],{"className":12628},[303],[100,12630,258],{"className":12631,"style":404},[150,154],[100,12633,239],{"className":12634},[350],[100,12636,12638],{"className":12637,"style":889},[350,888],[100,12639,282],{"className":12640},[2830,6847],[14,12642,12643,12644,12687,12688,12691],{},"沿这个梯度走一步，会把每个动作的对数概率往上推，推的幅度正比于 ",[100,12645,12647,12666],{"className":12646},[103],[100,12648,12650],{"className":12649},[107],[109,12651,12652],{"xmlns":111},[113,12653,12654,12664],{},[116,12655,12656,12658,12660,12662],{},[122,12657,273],{},[227,12659,230],{"stretchy":229},[122,12661,258],{},[227,12663,239],{"stretchy":229},[129,12665,521],{"encoding":131},[100,12667,12669],{"className":12668,"ariaHidden":137},[136],[100,12670,12672,12675,12678,12681,12684],{"className":12671},[141],[100,12673],{"className":12674,"style":295},[145],[100,12676,273],{"className":12677,"style":487},[150,154],[100,12679,230],{"className":12680},[303],[100,12682,258],{"className":12683,"style":404},[150,154],[100,12685,239],{"className":12686},[350],"，也就是",[9291,12689,12690],{},"曾经拿到的所有奖励","的总和。但这不太讲道理。",[14,12693,12694,12695,12698,12699,12702,12703,12706],{},"智能体强化一个动作，理应只看它的",[9291,12696,12697],{},"后果","。采取动作",[9291,12700,12701],{},"之前","拿到的奖励与这个动作的好坏毫无关系：有关系的只有",[9291,12704,12705],{},"之后","来的奖励。",[14,12708,12709],{},"这个直觉其实藏在数学里：可以证明，策略梯度也能写成",[100,12711,12713],{"className":12712},[555],[100,12714,12716,12888],{"className":12715},[103],[100,12717,12719],{"className":12718},[107],[109,12720,12721],{"xmlns":111,"display":564},[113,12722,12723,12885],{},[116,12724,12725,12731,12733,12735,12741,12743,12745,12765],{},[119,12726,12727,12729],{},[122,12728,607],{"mathvariant":606},[122,12730,127],{},[122,12732,225],{},[227,12734,230],{"stretchy":229},[119,12736,12737,12739],{},[122,12738,124],{},[122,12740,127],{},[227,12742,239],{"stretchy":229},[227,12744,242],{},[122,12746,12747],{},[246,12748,12749,12753],{},[227,12750,12751],{},[122,12752,253],{"mathvariant":252},[116,12754,12755,12757,12759],{},[122,12756,258],{},[227,12758,261],{},[119,12760,12761,12763],{},[122,12762,124],{},[122,12764,127],{},[116,12766,12767,12769,12783,12789,12791,12793,12799,12801,12807,12809,12815,12817,12837,12839,12841,12851,12853,12863,12865,12881,12883],{},[227,12768,270],{"fence":137},[1506,12770,12771,12773,12781],{},[227,12772,2508],{},[116,12774,12775,12777,12779],{},[122,12776,1515],{},[227,12778,242],{},[582,12780,1229],{},[122,12782,1258],{},[119,12784,12785,12787],{},[122,12786,607],{"mathvariant":606},[122,12788,127],{},[122,12790,2093],{},[227,12792,2096],{},[119,12794,12795,12797],{},[122,12796,124],{},[122,12798,127],{},[227,12800,230],{"stretchy":229},[119,12802,12803,12805],{},[122,12804,18],{},[122,12806,1515],{},[122,12808,624],{"mathvariant":606},[119,12810,12811,12813],{},[122,12812,1226],{},[122,12814,1515],{},[227,12816,239],{"stretchy":229},[1506,12818,12819,12821,12835],{},[227,12820,2508],{},[116,12822,12823,12831,12833],{},[12824,12825,12826,12828],"msup",{},[122,12827,1515],{},[227,12829,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},"′",[227,12832,242],{},[122,12834,1515],{},[122,12836,1258],{},[122,12838,273],{},[227,12840,230],{"stretchy":229},[119,12842,12843,12845],{},[122,12844,1226],{},[12824,12846,12847,12849],{},[122,12848,1515],{},[227,12850,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,12852,1232],{"separator":137},[119,12854,12855,12857],{},[122,12856,18],{},[12824,12858,12859,12861],{},[122,12860,1515],{},[227,12862,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,12864,1232],{"separator":137},[119,12866,12867,12869],{},[122,12868,1226],{},[116,12870,12871,12877,12879],{},[12824,12872,12873,12875],{},[122,12874,1515],{},[227,12876,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,12878,580],{},[582,12880,584],{},[227,12882,239],{"stretchy":229},[227,12884,282],{"fence":137},[129,12886,12887],{"encoding":131},"\\nabla_{\\theta} J(\\pi_{\\theta}) = \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t) \\sum_{t'=t}^T R(s_{t'}, a_{t'}, s_{t'+1})\\right]",[100,12889,12891,13001],{"className":12890,"ariaHidden":137},[136],[100,12892,12894,12897,12940,12943,12946,12989,12992,12995,12998],{"className":12893},[141],[100,12895],{"className":12896,"style":295},[145],[100,12898,12900,12903],{"className":12899},[150],[100,12901,607],{"className":12902},[150],[100,12904,12906],{"className":12905},[159],[100,12907,12909,12932],{"className":12908},[163,164],[100,12910,12912,12929],{"className":12911},[168],[100,12913,12915],{"className":12914,"style":173},[172],[100,12916,12917,12920],{"style":809},[100,12918],{"className":12919,"style":181},[180],[100,12921,12923],{"className":12922},[185,186,187,188],[100,12924,12926],{"className":12925},[150,188],[100,12927,127],{"className":12928,"style":195},[150,154,188],[100,12930,200],{"className":12931},[199],[100,12933,12935],{"className":12934},[168],[100,12936,12938],{"className":12937,"style":207},[172],[100,12939],{},[100,12941,225],{"className":12942,"style":299},[150,154],[100,12944,230],{"className":12945},[303],[100,12947,12949,12952],{"className":12948},[150],[100,12950,124],{"className":12951,"style":155},[150,154],[100,12953,12955],{"className":12954},[159],[100,12956,12958,12981],{"className":12957},[163,164],[100,12959,12961,12978],{"className":12960},[168],[100,12962,12964],{"className":12963,"style":173},[172],[100,12965,12966,12969],{"style":176},[100,12967],{"className":12968,"style":181},[180],[100,12970,12972],{"className":12971},[185,186,187,188],[100,12973,12975],{"className":12974},[150,188],[100,12976,127],{"className":12977,"style":195},[150,154,188],[100,12979,200],{"className":12980},[199],[100,12982,12984],{"className":12983},[168],[100,12985,12987],{"className":12986,"style":207},[172],[100,12988],{},[100,12990,239],{"className":12991},[350],[100,12993],{"className":12994,"style":355},[354],[100,12996,242],{"className":12997},[359],[100,12999],{"className":13000,"style":355},[354],[100,13002,13004,13008,13104,13107],{"className":13003},[141],[100,13005],{"className":13006,"style":13007},[145],"height:3.1223em;vertical-align:-1.294em;",[100,13009,13011],{"className":13010},[150],[100,13012,13014],{"className":13013},[376,377],[100,13015,13017,13096],{"className":13016},[163,164],[100,13018,13020,13093],{"className":13019},[168],[100,13021,13023,13083],{"className":13022,"style":387},[172],[100,13024,13025,13028],{"style":390},[100,13026],{"className":13027,"style":394},[180],[100,13029,13031],{"className":13030},[185,186,187,188],[100,13032,13034,13037,13040],{"className":13033},[150,188],[100,13035,258],{"className":13036,"style":404},[150,154,188],[100,13038,261],{"className":13039},[359,188],[100,13041,13043,13046],{"className":13042},[150,188],[100,13044,124],{"className":13045,"style":155},[150,154,188],[100,13047,13049],{"className":13048},[159],[100,13050,13052,13075],{"className":13051},[163,164],[100,13053,13055,13072],{"className":13054},[168],[100,13056,13058],{"className":13057,"style":426},[172],[100,13059,13060,13063],{"style":429},[100,13061],{"className":13062,"style":433},[180],[100,13064,13066],{"className":13065},[185,437,438,188],[100,13067,13069],{"className":13068},[150,188],[100,13070,127],{"className":13071,"style":195},[150,154,188],[100,13073,200],{"className":13074},[199],[100,13076,13078],{"className":13077},[168],[100,13079,13081],{"className":13080,"style":454},[172],[100,13082],{},[100,13084,13085,13088],{"style":459},[100,13086],{"className":13087,"style":394},[180],[100,13089,13090],{},[100,13091,253],{"className":13092},[376,468],[100,13094,200],{"className":13095},[199],[100,13097,13099],{"className":13098},[168],[100,13100,13102],{"className":13101,"style":478},[172],[100,13103],{},[100,13105],{"className":13106,"style":777},[354],[100,13108,13110,13116,13183,13186,13229,13232,13237,13240,13283,13286,13326,13329,13369,13372,13375,13472,13475,13478,13481,13554,13557,13560,13632,13635,13638,13716,13719],{"className":13109},[781],[100,13111,13113],{"className":13112,"style":889},[303,888],[100,13114,270],{"className":13115},[2830,6847],[100,13117,13119],{"className":13118},[376,377],[100,13120,13122,13175],{"className":13121},[163,164],[100,13123,13125,13172],{"className":13124},[168],[100,13126,13128,13148,13158],{"className":13127,"style":1727},[172],[100,13129,13130,13133],{"style":1730},[100,13131],{"className":13132,"style":1734},[180],[100,13134,13136],{"className":13135},[185,186,187,188],[100,13137,13139,13142,13145],{"className":13138},[150,188],[100,13140,1515],{"className":13141},[150,154,188],[100,13143,242],{"className":13144},[359,188],[100,13146,1229],{"className":13147},[150,188],[100,13149,13150,13153],{"style":1752},[100,13151],{"className":13152,"style":1734},[180],[100,13154,13155],{},[100,13156,2508],{"className":13157},[376,1761,1762],[100,13159,13160,13163],{"style":1765},[100,13161],{"className":13162,"style":1734},[180],[100,13164,13166],{"className":13165},[185,186,187,188],[100,13167,13169],{"className":13168},[150,188],[100,13170,1258],{"className":13171,"style":1436},[150,154,188],[100,13173,200],{"className":13174},[199],[100,13176,13178],{"className":13177},[168],[100,13179,13181],{"className":13180,"style":1787},[172],[100,13182],{},[100,13184],{"className":13185,"style":777},[354],[100,13187,13189,13192],{"className":13188},[150],[100,13190,607],{"className":13191},[150],[100,13193,13195],{"className":13194},[159],[100,13196,13198,13221],{"className":13197},[163,164],[100,13199,13201,13218],{"className":13200},[168],[100,13202,13204],{"className":13203,"style":173},[172],[100,13205,13206,13209],{"style":809},[100,13207],{"className":13208,"style":181},[180],[100,13210,13212],{"className":13211},[185,186,187,188],[100,13213,13215],{"className":13214},[150,188],[100,13216,127],{"className":13217,"style":195},[150,154,188],[100,13219,200],{"className":13220},[199],[100,13222,13224],{"className":13223},[168],[100,13225,13227],{"className":13226,"style":207},[172],[100,13228],{},[100,13230],{"className":13231,"style":777},[354],[100,13233,2116,13235],{"className":13234},[376],[100,13236,2120],{"style":2119},[100,13238],{"className":13239,"style":777},[354],[100,13241,13243,13246],{"className":13242},[150],[100,13244,124],{"className":13245,"style":155},[150,154],[100,13247,13249],{"className":13248},[159],[100,13250,13252,13275],{"className":13251},[163,164],[100,13253,13255,13272],{"className":13254},[168],[100,13256,13258],{"className":13257,"style":173},[172],[100,13259,13260,13263],{"style":176},[100,13261],{"className":13262,"style":181},[180],[100,13264,13266],{"className":13265},[185,186,187,188],[100,13267,13269],{"className":13268},[150,188],[100,13270,127],{"className":13271,"style":195},[150,154,188],[100,13273,200],{"className":13274},[199],[100,13276,13278],{"className":13277},[168],[100,13279,13281],{"className":13280,"style":207},[172],[100,13282],{},[100,13284,230],{"className":13285},[303],[100,13287,13289,13292],{"className":13288},[150],[100,13290,18],{"className":13291},[150,154],[100,13293,13295],{"className":13294},[159],[100,13296,13298,13318],{"className":13297},[163,164],[100,13299,13301,13315],{"className":13300},[168],[100,13302,13304],{"className":13303,"style":1869},[172],[100,13305,13306,13309],{"style":809},[100,13307],{"className":13308,"style":181},[180],[100,13310,13312],{"className":13311},[185,186,187,188],[100,13313,1515],{"className":13314},[150,154,188],[100,13316,200],{"className":13317},[199],[100,13319,13321],{"className":13320},[168],[100,13322,13324],{"className":13323,"style":207},[172],[100,13325],{},[100,13327,624],{"className":13328},[150],[100,13330,13332,13335],{"className":13331},[150],[100,13333,1226],{"className":13334},[150,154],[100,13336,13338],{"className":13337},[159],[100,13339,13341,13361],{"className":13340},[163,164],[100,13342,13344,13358],{"className":13343},[168],[100,13345,13347],{"className":13346,"style":1869},[172],[100,13348,13349,13352],{"style":809},[100,13350],{"className":13351,"style":181},[180],[100,13353,13355],{"className":13354},[185,186,187,188],[100,13356,1515],{"className":13357},[150,154,188],[100,13359,200],{"className":13360},[199],[100,13362,13364],{"className":13363},[168],[100,13365,13367],{"className":13366,"style":207},[172],[100,13368],{},[100,13370,239],{"className":13371},[350],[100,13373],{"className":13374,"style":777},[354],[100,13376,13378],{"className":13377},[376,377],[100,13379,13381,13463],{"className":13380},[163,164],[100,13382,13384,13460],{"className":13383},[168],[100,13385,13387,13439,13449],{"className":13386,"style":1727},[172],[100,13388,13390,13393],{"style":13389},"top:-1.856em;margin-left:0em;",[100,13391],{"className":13392,"style":1734},[180],[100,13394,13396],{"className":13395},[185,186,187,188],[100,13397,13399,13433,13436],{"className":13398},[150,188],[100,13400,13402,13405],{"className":13401},[150,188],[100,13403,1515],{"className":13404},[150,154,188],[100,13406,13408],{"className":13407},[159],[100,13409,13411],{"className":13410},[163],[100,13412,13414],{"className":13413},[168],[100,13415,13418],{"className":13416,"style":13417},[172],"height:0.6828em;",[100,13419,13421,13424],{"style":13420},"top:-2.786em;margin-right:0.0714em;",[100,13422],{"className":13423,"style":433},[180],[100,13425,13427],{"className":13426},[185,437,438,188],[100,13428,13430],{"className":13429},[150,188],[100,13431,12830],{"className":13432},[150,188],[100,13434,242],{"className":13435},[359,188],[100,13437,1515],{"className":13438},[150,154,188],[100,13440,13441,13444],{"style":1752},[100,13442],{"className":13443,"style":1734},[180],[100,13445,13446],{},[100,13447,2508],{"className":13448},[376,1761,1762],[100,13450,13451,13454],{"style":1765},[100,13452],{"className":13453,"style":1734},[180],[100,13455,13457],{"className":13456},[185,186,187,188],[100,13458,1258],{"className":13459,"style":1436},[150,154,188],[100,13461,200],{"className":13462},[199],[100,13464,13466],{"className":13465},[168],[100,13467,13470],{"className":13468,"style":13469},[172],"height:1.294em;",[100,13471],{},[100,13473],{"className":13474,"style":777},[354],[100,13476,273],{"className":13477,"style":487},[150,154],[100,13479,230],{"className":13480},[303],[100,13482,13484,13487],{"className":13483},[150],[100,13485,1226],{"className":13486},[150,154],[100,13488,13490],{"className":13489},[159],[100,13491,13493,13546],{"className":13492},[163,164],[100,13494,13496,13543],{"className":13495},[168],[100,13497,13500],{"className":13498,"style":13499},[172],"height:0.328em;",[100,13501,13502,13505],{"style":809},[100,13503],{"className":13504,"style":181},[180],[100,13506,13508],{"className":13507},[185,186,187,188],[100,13509,13511],{"className":13510},[150,188],[100,13512,13514,13517],{"className":13513},[150,188],[100,13515,1515],{"className":13516},[150,154,188],[100,13518,13520],{"className":13519},[159],[100,13521,13523],{"className":13522},[163],[100,13524,13526],{"className":13525},[168],[100,13527,13529],{"className":13528,"style":13417},[172],[100,13530,13531,13534],{"style":13420},[100,13532],{"className":13533,"style":433},[180],[100,13535,13537],{"className":13536},[185,437,438,188],[100,13538,13540],{"className":13539},[150,188],[100,13541,12830],{"className":13542},[150,188],[100,13544,200],{"className":13545},[199],[100,13547,13549],{"className":13548},[168],[100,13550,13552],{"className":13551,"style":207},[172],[100,13553],{},[100,13555,1232],{"className":13556},[1343],[100,13558],{"className":13559,"style":777},[354],[100,13561,13563,13566],{"className":13562},[150],[100,13564,18],{"className":13565},[150,154],[100,13567,13569],{"className":13568},[159],[100,13570,13572,13624],{"className":13571},[163,164],[100,13573,13575,13621],{"className":13574},[168],[100,13576,13578],{"className":13577,"style":13499},[172],[100,13579,13580,13583],{"style":809},[100,13581],{"className":13582,"style":181},[180],[100,13584,13586],{"className":13585},[185,186,187,188],[100,13587,13589],{"className":13588},[150,188],[100,13590,13592,13595],{"className":13591},[150,188],[100,13593,1515],{"className":13594},[150,154,188],[100,13596,13598],{"className":13597},[159],[100,13599,13601],{"className":13600},[163],[100,13602,13604],{"className":13603},[168],[100,13605,13607],{"className":13606,"style":13417},[172],[100,13608,13609,13612],{"style":13420},[100,13610],{"className":13611,"style":433},[180],[100,13613,13615],{"className":13614},[185,437,438,188],[100,13616,13618],{"className":13617},[150,188],[100,13619,12830],{"className":13620},[150,188],[100,13622,200],{"className":13623},[199],[100,13625,13627],{"className":13626},[168],[100,13628,13630],{"className":13629,"style":207},[172],[100,13631],{},[100,13633,1232],{"className":13634},[1343],[100,13636],{"className":13637,"style":777},[354],[100,13639,13641,13644],{"className":13640},[150],[100,13642,1226],{"className":13643},[150,154],[100,13645,13647],{"className":13646},[159],[100,13648,13650,13708],{"className":13649},[163,164],[100,13651,13653,13705],{"className":13652},[168],[100,13654,13656],{"className":13655,"style":13499},[172],[100,13657,13658,13661],{"style":809},[100,13659],{"className":13660,"style":181},[180],[100,13662,13664],{"className":13663},[185,186,187,188],[100,13665,13667,13699,13702],{"className":13666},[150,188],[100,13668,13670,13673],{"className":13669},[150,188],[100,13671,1515],{"className":13672},[150,154,188],[100,13674,13676],{"className":13675},[159],[100,13677,13679],{"className":13678},[163],[100,13680,13682],{"className":13681},[168],[100,13683,13685],{"className":13684,"style":13417},[172],[100,13686,13687,13690],{"style":13420},[100,13688],{"className":13689,"style":433},[180],[100,13691,13693],{"className":13692},[185,437,438,188],[100,13694,13696],{"className":13695},[150,188],[100,13697,12830],{"className":13698},[150,188],[100,13700,580],{"className":13701},[681,188],[100,13703,584],{"className":13704},[150,188],[100,13706,200],{"className":13707},[199],[100,13709,13711],{"className":13710},[168],[100,13712,13714],{"className":13713,"style":694},[172],[100,13715],{},[100,13717,239],{"className":13718},[350],[100,13720,13722],{"className":13721,"style":889},[350,888],[100,13723,282],{"className":13724},[2830,6847],[14,13726,13727,13728,13731],{},"在这个形式里，动作只根据它",[9291,13729,13730],{},"被采取之后","获得的奖励来强化。",[14,13733,13734],{},"我们把这个形式叫作「reward-to-go 策略梯度」，因为轨迹中某一点之后的奖励总和",[100,13736,13738],{"className":13737},[555],[100,13739,13741,13830],{"className":13740},[103],[100,13742,13744],{"className":13743},[107],[109,13745,13746],{"xmlns":111,"display":564},[113,13747,13748,13827],{},[116,13749,13750,13760,13763,13781,13783,13785,13795,13797,13807,13809,13825],{},[119,13751,13752,13758],{},[7623,13753,13754,13756],{"accent":137},[122,13755,273],{},[227,13757,7629],{},[122,13759,1515],{},[227,13761,13762],{},"≐",[1506,13764,13765,13767,13779],{},[227,13766,2508],{},[116,13768,13769,13775,13777],{},[12824,13770,13771,13773],{},[122,13772,1515],{},[227,13774,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,13776,242],{},[122,13778,1515],{},[122,13780,1258],{},[122,13782,273],{},[227,13784,230],{"stretchy":229},[119,13786,13787,13789],{},[122,13788,1226],{},[12824,13790,13791,13793],{},[122,13792,1515],{},[227,13794,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,13796,1232],{"separator":137},[119,13798,13799,13801],{},[122,13800,18],{},[12824,13802,13803,13805],{},[122,13804,1515],{},[227,13806,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,13808,1232],{"separator":137},[119,13810,13811,13813],{},[122,13812,1226],{},[116,13814,13815,13821,13823],{},[12824,13816,13817,13819],{},[122,13818,1515],{},[227,13820,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,13822,580],{},[582,13824,584],{},[227,13826,239],{"stretchy":229},[129,13828,13829],{"encoding":131},"\\hat{R}_t \\doteq \\sum_{t'=t}^T R(s_{t'}, a_{t'}, s_{t'+1})",[100,13831,13833,13921],{"className":13832,"ariaHidden":137},[136],[100,13834,13836,13840,13912,13915,13918],{"className":13835},[141],[100,13837],{"className":13838,"style":13839},[145],"height:1.0968em;vertical-align:-0.15em;",[100,13841,13843,13877],{"className":13842},[150],[100,13844,13846],{"className":13845},[150,7729],[100,13847,13849],{"className":13848},[163],[100,13850,13852],{"className":13851},[168],[100,13853,13856,13864],{"className":13854,"style":13855},[172],"height:0.9468em;",[100,13857,13858,13861],{"style":459},[100,13859],{"className":13860,"style":394},[180],[100,13862,273],{"className":13863,"style":487},[150,154],[100,13865,13867,13870],{"style":13866},"top:-3.2523em;",[100,13868],{"className":13869,"style":394},[180],[100,13871,13874],{"className":13872,"style":13873},[7755],"left:-0.1667em;",[100,13875,7629],{"className":13876},[150],[100,13878,13880],{"className":13879},[159],[100,13881,13883,13904],{"className":13882},[163,164],[100,13884,13886,13901],{"className":13885},[168],[100,13887,13889],{"className":13888,"style":1869},[172],[100,13890,13892,13895],{"style":13891},"top:-2.55em;margin-left:-0.0077em;margin-right:0.05em;",[100,13893],{"className":13894,"style":181},[180],[100,13896,13898],{"className":13897},[185,186,187,188],[100,13899,1515],{"className":13900},[150,154,188],[100,13902,200],{"className":13903},[199],[100,13905,13907],{"className":13906},[168],[100,13908,13910],{"className":13909,"style":207},[172],[100,13911],{},[100,13913],{"className":13914,"style":355},[354],[100,13916,13762],{"className":13917},[359],[100,13919],{"className":13920,"style":355},[354],[100,13922,13924,13927,14020,14023,14026,14029,14101,14104,14107,14179,14182,14185,14263],{"className":13923},[141],[100,13925],{"className":13926,"style":13007},[145],[100,13928,13930],{"className":13929},[376,377],[100,13931,13933,14012],{"className":13932},[163,164],[100,13934,13936,14009],{"className":13935},[168],[100,13937,13939,13988,13998],{"className":13938,"style":1727},[172],[100,13940,13941,13944],{"style":13389},[100,13942],{"className":13943,"style":1734},[180],[100,13945,13947],{"className":13946},[185,186,187,188],[100,13948,13950,13982,13985],{"className":13949},[150,188],[100,13951,13953,13956],{"className":13952},[150,188],[100,13954,1515],{"className":13955},[150,154,188],[100,13957,13959],{"className":13958},[159],[100,13960,13962],{"className":13961},[163],[100,13963,13965],{"className":13964},[168],[100,13966,13968],{"className":13967,"style":13417},[172],[100,13969,13970,13973],{"style":13420},[100,13971],{"className":13972,"style":433},[180],[100,13974,13976],{"className":13975},[185,437,438,188],[100,13977,13979],{"className":13978},[150,188],[100,13980,12830],{"className":13981},[150,188],[100,13983,242],{"className":13984},[359,188],[100,13986,1515],{"className":13987},[150,154,188],[100,13989,13990,13993],{"style":1752},[100,13991],{"className":13992,"style":1734},[180],[100,13994,13995],{},[100,13996,2508],{"className":13997},[376,1761,1762],[100,13999,14000,14003],{"style":1765},[100,14001],{"className":14002,"style":1734},[180],[100,14004,14006],{"className":14005},[185,186,187,188],[100,14007,1258],{"className":14008,"style":1436},[150,154,188],[100,14010,200],{"className":14011},[199],[100,14013,14015],{"className":14014},[168],[100,14016,14018],{"className":14017,"style":13469},[172],[100,14019],{},[100,14021],{"className":14022,"style":777},[354],[100,14024,273],{"className":14025,"style":487},[150,154],[100,14027,230],{"className":14028},[303],[100,14030,14032,14035],{"className":14031},[150],[100,14033,1226],{"className":14034},[150,154],[100,14036,14038],{"className":14037},[159],[100,14039,14041,14093],{"className":14040},[163,164],[100,14042,14044,14090],{"className":14043},[168],[100,14045,14047],{"className":14046,"style":13499},[172],[100,14048,14049,14052],{"style":809},[100,14050],{"className":14051,"style":181},[180],[100,14053,14055],{"className":14054},[185,186,187,188],[100,14056,14058],{"className":14057},[150,188],[100,14059,14061,14064],{"className":14060},[150,188],[100,14062,1515],{"className":14063},[150,154,188],[100,14065,14067],{"className":14066},[159],[100,14068,14070],{"className":14069},[163],[100,14071,14073],{"className":14072},[168],[100,14074,14076],{"className":14075,"style":13417},[172],[100,14077,14078,14081],{"style":13420},[100,14079],{"className":14080,"style":433},[180],[100,14082,14084],{"className":14083},[185,437,438,188],[100,14085,14087],{"className":14086},[150,188],[100,14088,12830],{"className":14089},[150,188],[100,14091,200],{"className":14092},[199],[100,14094,14096],{"className":14095},[168],[100,14097,14099],{"className":14098,"style":207},[172],[100,14100],{},[100,14102,1232],{"className":14103},[1343],[100,14105],{"className":14106,"style":777},[354],[100,14108,14110,14113],{"className":14109},[150],[100,14111,18],{"className":14112},[150,154],[100,14114,14116],{"className":14115},[159],[100,14117,14119,14171],{"className":14118},[163,164],[100,14120,14122,14168],{"className":14121},[168],[100,14123,14125],{"className":14124,"style":13499},[172],[100,14126,14127,14130],{"style":809},[100,14128],{"className":14129,"style":181},[180],[100,14131,14133],{"className":14132},[185,186,187,188],[100,14134,14136],{"className":14135},[150,188],[100,14137,14139,14142],{"className":14138},[150,188],[100,14140,1515],{"className":14141},[150,154,188],[100,14143,14145],{"className":14144},[159],[100,14146,14148],{"className":14147},[163],[100,14149,14151],{"className":14150},[168],[100,14152,14154],{"className":14153,"style":13417},[172],[100,14155,14156,14159],{"style":13420},[100,14157],{"className":14158,"style":433},[180],[100,14160,14162],{"className":14161},[185,437,438,188],[100,14163,14165],{"className":14164},[150,188],[100,14166,12830],{"className":14167},[150,188],[100,14169,200],{"className":14170},[199],[100,14172,14174],{"className":14173},[168],[100,14175,14177],{"className":14176,"style":207},[172],[100,14178],{},[100,14180,1232],{"className":14181},[1343],[100,14183],{"className":14184,"style":777},[354],[100,14186,14188,14191],{"className":14187},[150],[100,14189,1226],{"className":14190},[150,154],[100,14192,14194],{"className":14193},[159],[100,14195,14197,14255],{"className":14196},[163,164],[100,14198,14200,14252],{"className":14199},[168],[100,14201,14203],{"className":14202,"style":13499},[172],[100,14204,14205,14208],{"style":809},[100,14206],{"className":14207,"style":181},[180],[100,14209,14211],{"className":14210},[185,186,187,188],[100,14212,14214,14246,14249],{"className":14213},[150,188],[100,14215,14217,14220],{"className":14216},[150,188],[100,14218,1515],{"className":14219},[150,154,188],[100,14221,14223],{"className":14222},[159],[100,14224,14226],{"className":14225},[163],[100,14227,14229],{"className":14228},[168],[100,14230,14232],{"className":14231,"style":13417},[172],[100,14233,14234,14237],{"style":13420},[100,14235],{"className":14236,"style":433},[180],[100,14238,14240],{"className":14239},[185,437,438,188],[100,14241,14243],{"className":14242},[150,188],[100,14244,12830],{"className":14245},[150,188],[100,14247,580],{"className":14248},[681,188],[100,14250,584],{"className":14251},[150,188],[100,14253,200],{"className":14254},[199],[100,14256,14258],{"className":14257},[168],[100,14259,14261],{"className":14260,"style":694},[172],[100,14262],{},[100,14264,239],{"className":14265},[350],[14,14267,14268,14269,14272],{},"就叫作从该点起的 ",[51,14270,14271],{},"reward-to-go","（只往后算的奖励），而这个策略梯度表达式依赖的正是各状态-动作对的 reward-to-go。",[10,14274,14275],{"title":8480},[14,14276,14277,14280],{},[51,14278,14279],{},"但这为什么更好","？策略梯度的一个关键问题是：要把它的样本估计做到低方差，需要多少条采样轨迹。我们出发的那个公式里，强化动作的项正比于过去的奖励，这些项均值为零、方差非零，只会给策略梯度的样本估计添噪音。去掉它们，所需的采样轨迹就少了。",[14,14282,14283,14284,14288],{},"这个论断的（选读）证明见",[18,14285,8488],{"href":14286,"rel":14287},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Fextra_pg_proof1.html",[22],"，最终靠的是 EGLP 引理。",[93,14290,14292],{"id":14291},"实现-reward-to-go-策略梯度","实现 reward-to-go 策略梯度",[14,14294,14295,14296,8471,14299,8294],{},"reward-to-go 策略梯度的 PyTorch 短实现放在 ",[8468,14297,14298],{},"spinup\u002Fexamples\u002Fpytorch\u002Fpg_math\u002F2_rtg_pg.py",[18,14300,8476],{"href":14301,"rel":14302},"https:\u002F\u002Fgithub.com\u002Fopenai\u002Fspinningup\u002Fblob\u002Fmaster\u002Fspinup\u002Fexamples\u002Fpytorch\u002Fpg_math\u002F2_rtg_pg.py",[22],[14,14304,14305,14306,14309],{},"相比 ",[8468,14307,14308],{},"1_simple_pg.py","，唯一的变化是损失函数里用了不同的权重。代码改动非常小：加一个新函数，再改两行。新函数是：",[8496,14311,14313],{"className":8498,"code":14312,"language":8500,"meta":8501,"style":8501},"def reward_to_go(rews):\n    n = len(rews)\n    rtgs = np.zeros_like(rews)\n    for i in reversed(range(n)):\n        rtgs[i] = rews[i] + (rtgs[i+1] if i+1 \u003C n else 0)\n    return rtgs\n",[8468,14314,14315,14325,14337,14347,14369,14415],{"__ignoreMap":8501},[100,14316,14317,14319,14322],{"class":4184,"line":8506},[100,14318,8561],{"class":8520},[100,14320,14321],{"class":8564}," reward_to_go",[100,14323,14324],{"class":8516},"(rews):\n",[100,14326,14327,14330,14332,14334],{"class":4184,"line":8513},[100,14328,14329],{"class":8516},"    n ",[100,14331,242],{"class":8520},[100,14333,9858],{"class":9399},[100,14335,14336],{"class":8516},"(rews)\n",[100,14338,14339,14342,14344],{"class":4184,"line":8545},[100,14340,14341],{"class":8516},"    rtgs ",[100,14343,242],{"class":8520},[100,14345,14346],{"class":8516}," np.zeros_like(rews)\n",[100,14348,14349,14352,14355,14358,14361,14363,14366],{"class":4184,"line":8552},[100,14350,14351],{"class":8520},"    for",[100,14353,14354],{"class":8516}," i ",[100,14356,14357],{"class":8520},"in",[100,14359,14360],{"class":9399}," reversed",[100,14362,230],{"class":8516},[100,14364,14365],{"class":9399},"range",[100,14367,14368],{"class":8516},"(n)):\n",[100,14370,14371,14374,14376,14379,14381,14384,14386,14388,14391,14394,14397,14399,14401,14404,14407,14410,14413],{"class":4184,"line":8558},[100,14372,14373],{"class":8516},"        rtgs[i] ",[100,14375,242],{"class":8520},[100,14377,14378],{"class":8516}," rews[i] ",[100,14380,580],{"class":8520},[100,14382,14383],{"class":8516}," (rtgs[i",[100,14385,580],{"class":8520},[100,14387,584],{"class":9399},[100,14389,14390],{"class":8516},"] ",[100,14392,14393],{"class":8520},"if",[100,14395,14396],{"class":8516}," i",[100,14398,580],{"class":8520},[100,14400,584],{"class":9399},[100,14402,14403],{"class":8520}," \u003C",[100,14405,14406],{"class":8516}," n ",[100,14408,14409],{"class":8520},"else",[100,14411,14412],{"class":9399}," 0",[100,14414,9402],{"class":8516},[100,14416,14417,14419],{"class":4184,"line":8571},[100,14418,8585],{"class":8520},[100,14420,14421],{"class":8516}," rtgs\n",[14,14423,14424],{},"然后把原来的第 91、92 行从：",[8496,14426,14428],{"className":8498,"code":14427,"language":8500,"meta":8501,"style":8501},"                # 每个 logprob(a|s) 的权重是 R(tau)\n                batch_weights += [ep_ret] * ep_len\n",[8468,14429,14430,14435],{"__ignoreMap":8501},[100,14431,14432],{"class":4184,"line":8506},[100,14433,14434],{"class":8509},"                # 每个 logprob(a|s) 的权重是 R(tau)\n",[100,14436,14437,14440,14442,14444,14446],{"class":4184,"line":8513},[100,14438,14439],{"class":8516},"                batch_weights ",[100,14441,9780],{"class":8520},[100,14443,9783],{"class":8516},[100,14445,9170],{"class":8520},[100,14447,9788],{"class":8516},[14,14449,14450],{},"改成：",[8496,14452,14454],{"className":8498,"code":14453,"language":8500,"meta":8501,"style":8501},"                # 每个 logprob(a_t|s_t) 的权重是从 t 时刻起的 reward-to-go\n                batch_weights += list(reward_to_go(ep_rews))\n",[8468,14455,14456,14461],{"__ignoreMap":8501},[100,14457,14458],{"class":4184,"line":8506},[100,14459,14460],{"class":8509},"                # 每个 logprob(a_t|s_t) 的权重是从 t 时刻起的 reward-to-go\n",[100,14462,14463,14465,14467,14470],{"class":4184,"line":8513},[100,14464,14439],{"class":8516},[100,14466,9780],{"class":8520},[100,14468,14469],{"class":9399}," list",[100,14471,14472],{"class":8516},"(reward_to_go(ep_rews))\n",[10,14474,14475],{"title":8480},[14,14476,8483,14477,8489],{},[18,14478,8488],{"href":14479,"rel":14480},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Fextra_tf_pg_implementation.html#implementing-reward-to-go-policy-gradient",[22],[93,14482,14483],{"id":14483},"策略梯度中的基线",[14,14485,14486,14487,14516],{},"EGLP 引理的一个直接推论：对任何只依赖状态的函数 ",[100,14488,14490,14504],{"className":14489},[103],[100,14491,14493],{"className":14492},[107],[109,14494,14495],{"xmlns":111},[113,14496,14497,14502],{},[116,14498,14499],{},[122,14500,14501],{},"b",[129,14503,14501],{"encoding":131},[100,14505,14507],{"className":14506,"ariaHidden":137},[136],[100,14508,14510,14513],{"className":14509},[141],[100,14511],{"className":14512,"style":3183},[145],[100,14514,14501],{"className":14515},[150,154],"，有",[100,14518,14520],{"className":14519},[555],[100,14521,14523,14615],{"className":14522},[103],[100,14524,14526],{"className":14525},[107],[109,14527,14528],{"xmlns":111,"display":564},[113,14529,14530,14612],{},[116,14531,14532,14556,14608,14610],{},[122,14533,14534],{},[246,14535,14536,14540],{},[227,14537,14538],{},[122,14539,253],{"mathvariant":252},[116,14541,14542,14548,14550],{},[119,14543,14544,14546],{},[122,14545,18],{},[122,14547,1515],{},[227,14549,261],{},[119,14551,14552,14554],{},[122,14553,124],{},[122,14555,127],{},[116,14557,14558,14560,14566,14568,14570,14576,14578,14584,14586,14592,14594,14596,14598,14604,14606],{},[227,14559,270],{"fence":137},[119,14561,14562,14564],{},[122,14563,607],{"mathvariant":606},[122,14565,127],{},[122,14567,2093],{},[227,14569,2096],{},[119,14571,14572,14574],{},[122,14573,124],{},[122,14575,127],{},[227,14577,230],{"stretchy":229},[119,14579,14580,14582],{},[122,14581,18],{},[122,14583,1515],{},[122,14585,624],{"mathvariant":606},[119,14587,14588,14590],{},[122,14589,1226],{},[122,14591,1515],{},[227,14593,239],{"stretchy":229},[122,14595,14501],{},[227,14597,230],{"stretchy":229},[119,14599,14600,14602],{},[122,14601,1226],{},[122,14603,1515],{},[227,14605,239],{"stretchy":229},[227,14607,282],{"fence":137},[227,14609,242],{},[582,14611,1229],{},[129,14613,14614],{"encoding":131},"\\underset{a_t \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\nabla_{\\theta} \\log \\pi_{\\theta}(a_t|s_t) b(s_t)\\right] = 0",[100,14616,14618,15016],{"className":14617,"ariaHidden":137},[136],[100,14619,14621,14624,14760,14763,15007,15010,15013],{"className":14620},[141],[100,14622],{"className":14623,"style":369},[145],[100,14625,14627],{"className":14626},[150],[100,14628,14630],{"className":14629},[376,377],[100,14631,14633,14752],{"className":14632},[163,164],[100,14634,14636,14749],{"className":14635},[168],[100,14637,14639,14739],{"className":14638,"style":387},[172],[100,14640,14641,14644],{"style":390},[100,14642],{"className":14643,"style":394},[180],[100,14645,14647],{"className":14646},[185,186,187,188],[100,14648,14650,14693,14696],{"className":14649},[150,188],[100,14651,14653,14656],{"className":14652},[150,188],[100,14654,18],{"className":14655},[150,154,188],[100,14657,14659],{"className":14658},[159],[100,14660,14662,14684],{"className":14661},[163,164],[100,14663,14665,14681],{"className":14664},[168],[100,14666,14669],{"className":14667,"style":14668},[172],"height:0.2963em;",[100,14670,14672,14675],{"style":14671},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[100,14673],{"className":14674,"style":433},[180],[100,14676,14678],{"className":14677},[185,437,438,188],[100,14679,1515],{"className":14680},[150,154,188],[100,14682,200],{"className":14683},[199],[100,14685,14687],{"className":14686},[168],[100,14688,14691],{"className":14689,"style":14690},[172],"height:0.143em;",[100,14692],{},[100,14694,261],{"className":14695},[359,188],[100,14697,14699,14702],{"className":14698},[150,188],[100,14700,124],{"className":14701,"style":155},[150,154,188],[100,14703,14705],{"className":14704},[159],[100,14706,14708,14731],{"className":14707},[163,164],[100,14709,14711,14728],{"className":14710},[168],[100,14712,14714],{"className":14713,"style":426},[172],[100,14715,14716,14719],{"style":429},[100,14717],{"className":14718,"style":433},[180],[100,14720,14722],{"className":14721},[185,437,438,188],[100,14723,14725],{"className":14724},[150,188],[100,14726,127],{"className":14727,"style":195},[150,154,188],[100,14729,200],{"className":14730},[199],[100,14732,14734],{"className":14733},[168],[100,14735,14737],{"className":14736,"style":454},[172],[100,14738],{},[100,14740,14741,14744],{"style":459},[100,14742],{"className":14743,"style":394},[180],[100,14745,14746],{},[100,14747,253],{"className":14748},[376,468],[100,14750,200],{"className":14751},[199],[100,14753,14755],{"className":14754},[168],[100,14756,14758],{"className":14757,"style":478},[172],[100,14759],{},[100,14761],{"className":14762,"style":777},[354],[100,14764,14766,14769,14812,14815,14820,14823,14866,14869,14909,14912,14952,14955,14958,14961,15001,15004],{"className":14765},[781],[100,14767,270],{"className":14768,"style":889},[303,888],[100,14770,14772,14775],{"className":14771},[150],[100,14773,607],{"className":14774},[150],[100,14776,14778],{"className":14777},[159],[100,14779,14781,14804],{"className":14780},[163,164],[100,14782,14784,14801],{"className":14783},[168],[100,14785,14787],{"className":14786,"style":173},[172],[100,14788,14789,14792],{"style":809},[100,14790],{"className":14791,"style":181},[180],[100,14793,14795],{"className":14794},[185,186,187,188],[100,14796,14798],{"className":14797},[150,188],[100,14799,127],{"className":14800,"style":195},[150,154,188],[100,14802,200],{"className":14803},[199],[100,14805,14807],{"className":14806},[168],[100,14808,14810],{"className":14809,"style":207},[172],[100,14811],{},[100,14813],{"className":14814,"style":777},[354],[100,14816,2116,14818],{"className":14817},[376],[100,14819,2120],{"style":2119},[100,14821],{"className":14822,"style":777},[354],[100,14824,14826,14829],{"className":14825},[150],[100,14827,124],{"className":14828,"style":155},[150,154],[100,14830,14832],{"className":14831},[159],[100,14833,14835,14858],{"className":14834},[163,164],[100,14836,14838,14855],{"className":14837},[168],[100,14839,14841],{"className":14840,"style":173},[172],[100,14842,14843,14846],{"style":176},[100,14844],{"className":14845,"style":181},[180],[100,14847,14849],{"className":14848},[185,186,187,188],[100,14850,14852],{"className":14851},[150,188],[100,14853,127],{"className":14854,"style":195},[150,154,188],[100,14856,200],{"className":14857},[199],[100,14859,14861],{"className":14860},[168],[100,14862,14864],{"className":14863,"style":207},[172],[100,14865],{},[100,14867,230],{"className":14868},[303],[100,14870,14872,14875],{"className":14871},[150],[100,14873,18],{"className":14874},[150,154],[100,14876,14878],{"className":14877},[159],[100,14879,14881,14901],{"className":14880},[163,164],[100,14882,14884,14898],{"className":14883},[168],[100,14885,14887],{"className":14886,"style":1869},[172],[100,14888,14889,14892],{"style":809},[100,14890],{"className":14891,"style":181},[180],[100,14893,14895],{"className":14894},[185,186,187,188],[100,14896,1515],{"className":14897},[150,154,188],[100,14899,200],{"className":14900},[199],[100,14902,14904],{"className":14903},[168],[100,14905,14907],{"className":14906,"style":207},[172],[100,14908],{},[100,14910,624],{"className":14911},[150],[100,14913,14915,14918],{"className":14914},[150],[100,14916,1226],{"className":14917},[150,154],[100,14919,14921],{"className":14920},[159],[100,14922,14924,14944],{"className":14923},[163,164],[100,14925,14927,14941],{"className":14926},[168],[100,14928,14930],{"className":14929,"style":1869},[172],[100,14931,14932,14935],{"style":809},[100,14933],{"className":14934,"style":181},[180],[100,14936,14938],{"className":14937},[185,186,187,188],[100,14939,1515],{"className":14940},[150,154,188],[100,14942,200],{"className":14943},[199],[100,14945,14947],{"className":14946},[168],[100,14948,14950],{"className":14949,"style":207},[172],[100,14951],{},[100,14953,239],{"className":14954},[350],[100,14956,14501],{"className":14957},[150,154],[100,14959,230],{"className":14960},[303],[100,14962,14964,14967],{"className":14963},[150],[100,14965,1226],{"className":14966},[150,154],[100,14968,14970],{"className":14969},[159],[100,14971,14973,14993],{"className":14972},[163,164],[100,14974,14976,14990],{"className":14975},[168],[100,14977,14979],{"className":14978,"style":1869},[172],[100,14980,14981,14984],{"style":809},[100,14982],{"className":14983,"style":181},[180],[100,14985,14987],{"className":14986},[185,186,187,188],[100,14988,1515],{"className":14989},[150,154,188],[100,14991,200],{"className":14992},[199],[100,14994,14996],{"className":14995},[168],[100,14997,14999],{"className":14998,"style":207},[172],[100,15000],{},[100,15002,239],{"className":15003},[350],[100,15005,282],{"className":15006,"style":889},[350,888],[100,15008],{"className":15009,"style":355},[354],[100,15011,242],{"className":15012},[359],[100,15014],{"className":15015,"style":355},[354],[100,15017,15019,15022],{"className":15018},[141],[100,15020],{"className":15021,"style":10423},[145],[100,15023,1229],{"className":15024},[150],[14,15026,15027],{},"这让我们可以在策略梯度表达式里加上或减去任意多个这样的项，而期望不变：",[100,15029,15031],{"className":15030},[555],[100,15032,15034,15224],{"className":15033},[103],[100,15035,15037],{"className":15036},[107],[109,15038,15039],{"xmlns":111,"display":564},[113,15040,15041,15221],{},[116,15042,15043,15049,15051,15053,15059,15061,15063,15083],{},[119,15044,15045,15047],{},[122,15046,607],{"mathvariant":606},[122,15048,127],{},[122,15050,225],{},[227,15052,230],{"stretchy":229},[119,15054,15055,15057],{},[122,15056,124],{},[122,15058,127],{},[227,15060,239],{"stretchy":229},[227,15062,242],{},[122,15064,15065],{},[246,15066,15067,15071],{},[227,15068,15069],{},[122,15070,253],{"mathvariant":252},[116,15072,15073,15075,15077],{},[122,15074,258],{},[227,15076,261],{},[119,15078,15079,15081],{},[122,15080,124],{},[122,15082,127],{},[116,15084,15085,15087,15101,15107,15109,15111,15117,15119,15125,15127,15133,15135,15219],{},[227,15086,270],{"fence":137},[1506,15088,15089,15091,15099],{},[227,15090,2508],{},[116,15092,15093,15095,15097],{},[122,15094,1515],{},[227,15096,242],{},[582,15098,1229],{},[122,15100,1258],{},[119,15102,15103,15105],{},[122,15104,607],{"mathvariant":606},[122,15106,127],{},[122,15108,2093],{},[227,15110,2096],{},[119,15112,15113,15115],{},[122,15114,124],{},[122,15116,127],{},[227,15118,230],{"stretchy":229},[119,15120,15121,15123],{},[122,15122,18],{},[122,15124,1515],{},[122,15126,624],{"mathvariant":606},[119,15128,15129,15131],{},[122,15130,1226],{},[122,15132,1515],{},[227,15134,239],{"stretchy":229},[116,15136,15137,15139,15157,15159,15161,15171,15173,15183,15185,15201,15203,15205,15207,15209,15215,15217],{},[227,15138,230],{"fence":137},[1506,15140,15141,15143,15155],{},[227,15142,2508],{},[116,15144,15145,15151,15153],{},[12824,15146,15147,15149],{},[122,15148,1515],{},[227,15150,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,15152,242],{},[122,15154,1515],{},[122,15156,1258],{},[122,15158,273],{},[227,15160,230],{"stretchy":229},[119,15162,15163,15165],{},[122,15164,1226],{},[12824,15166,15167,15169],{},[122,15168,1515],{},[227,15170,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,15172,1232],{"separator":137},[119,15174,15175,15177],{},[122,15176,18],{},[12824,15178,15179,15181],{},[122,15180,1515],{},[227,15182,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,15184,1232],{"separator":137},[119,15186,15187,15189],{},[122,15188,1226],{},[116,15190,15191,15197,15199],{},[12824,15192,15193,15195],{},[122,15194,1515],{},[227,15196,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,15198,580],{},[582,15200,584],{},[227,15202,239],{"stretchy":229},[227,15204,9309],{},[122,15206,14501],{},[227,15208,230],{"stretchy":229},[119,15210,15211,15213],{},[122,15212,1226],{},[122,15214,1515],{},[227,15216,239],{"stretchy":229},[227,15218,239],{"fence":137},[227,15220,282],{"fence":137},[129,15222,15223],{"encoding":131},"\\nabla_{\\theta} J(\\pi_{\\theta}) = \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t) \\left(\\sum_{t'=t}^T R(s_{t'}, a_{t'}, s_{t'+1}) - b(s_t)\\right)\\right]",[100,15225,15227,15337],{"className":15226,"ariaHidden":137},[136],[100,15228,15230,15233,15276,15279,15282,15325,15328,15331,15334],{"className":15229},[141],[100,15231],{"className":15232,"style":295},[145],[100,15234,15236,15239],{"className":15235},[150],[100,15237,607],{"className":15238},[150],[100,15240,15242],{"className":15241},[159],[100,15243,15245,15268],{"className":15244},[163,164],[100,15246,15248,15265],{"className":15247},[168],[100,15249,15251],{"className":15250,"style":173},[172],[100,15252,15253,15256],{"style":809},[100,15254],{"className":15255,"style":181},[180],[100,15257,15259],{"className":15258},[185,186,187,188],[100,15260,15262],{"className":15261},[150,188],[100,15263,127],{"className":15264,"style":195},[150,154,188],[100,15266,200],{"className":15267},[199],[100,15269,15271],{"className":15270},[168],[100,15272,15274],{"className":15273,"style":207},[172],[100,15275],{},[100,15277,225],{"className":15278,"style":299},[150,154],[100,15280,230],{"className":15281},[303],[100,15283,15285,15288],{"className":15284},[150],[100,15286,124],{"className":15287,"style":155},[150,154],[100,15289,15291],{"className":15290},[159],[100,15292,15294,15317],{"className":15293},[163,164],[100,15295,15297,15314],{"className":15296},[168],[100,15298,15300],{"className":15299,"style":173},[172],[100,15301,15302,15305],{"style":176},[100,15303],{"className":15304,"style":181},[180],[100,15306,15308],{"className":15307},[185,186,187,188],[100,15309,15311],{"className":15310},[150,188],[100,15312,127],{"className":15313,"style":195},[150,154,188],[100,15315,200],{"className":15316},[199],[100,15318,15320],{"className":15319},[168],[100,15321,15323],{"className":15322,"style":207},[172],[100,15324],{},[100,15326,239],{"className":15327},[350],[100,15329],{"className":15330,"style":355},[354],[100,15332,242],{"className":15333},[359],[100,15335],{"className":15336,"style":355},[354],[100,15338,15340,15343,15439,15442],{"className":15339},[141],[100,15341],{"className":15342,"style":13007},[145],[100,15344,15346],{"className":15345},[150],[100,15347,15349],{"className":15348},[376,377],[100,15350,15352,15431],{"className":15351},[163,164],[100,15353,15355,15428],{"className":15354},[168],[100,15356,15358,15418],{"className":15357,"style":387},[172],[100,15359,15360,15363],{"style":390},[100,15361],{"className":15362,"style":394},[180],[100,15364,15366],{"className":15365},[185,186,187,188],[100,15367,15369,15372,15375],{"className":15368},[150,188],[100,15370,258],{"className":15371,"style":404},[150,154,188],[100,15373,261],{"className":15374},[359,188],[100,15376,15378,15381],{"className":15377},[150,188],[100,15379,124],{"className":15380,"style":155},[150,154,188],[100,15382,15384],{"className":15383},[159],[100,15385,15387,15410],{"className":15386},[163,164],[100,15388,15390,15407],{"className":15389},[168],[100,15391,15393],{"className":15392,"style":426},[172],[100,15394,15395,15398],{"style":429},[100,15396],{"className":15397,"style":433},[180],[100,15399,15401],{"className":15400},[185,437,438,188],[100,15402,15404],{"className":15403},[150,188],[100,15405,127],{"className":15406,"style":195},[150,154,188],[100,15408,200],{"className":15409},[199],[100,15411,15413],{"className":15412},[168],[100,15414,15416],{"className":15415,"style":454},[172],[100,15417],{},[100,15419,15420,15423],{"style":459},[100,15421],{"className":15422,"style":394},[180],[100,15424,15425],{},[100,15426,253],{"className":15427},[376,468],[100,15429,200],{"className":15430},[199],[100,15432,15434],{"className":15433},[168],[100,15435,15437],{"className":15436,"style":478},[172],[100,15438],{},[100,15440],{"className":15441,"style":777},[354],[100,15443,15445,15451,15518,15521,15564,15567,15572,15575,15618,15621,15661,15664,15704,15707,15710,16122],{"className":15444},[781],[100,15446,15448],{"className":15447,"style":889},[303,888],[100,15449,270],{"className":15450},[2830,6847],[100,15452,15454],{"className":15453},[376,377],[100,15455,15457,15510],{"className":15456},[163,164],[100,15458,15460,15507],{"className":15459},[168],[100,15461,15463,15483,15493],{"className":15462,"style":1727},[172],[100,15464,15465,15468],{"style":1730},[100,15466],{"className":15467,"style":1734},[180],[100,15469,15471],{"className":15470},[185,186,187,188],[100,15472,15474,15477,15480],{"className":15473},[150,188],[100,15475,1515],{"className":15476},[150,154,188],[100,15478,242],{"className":15479},[359,188],[100,15481,1229],{"className":15482},[150,188],[100,15484,15485,15488],{"style":1752},[100,15486],{"className":15487,"style":1734},[180],[100,15489,15490],{},[100,15491,2508],{"className":15492},[376,1761,1762],[100,15494,15495,15498],{"style":1765},[100,15496],{"className":15497,"style":1734},[180],[100,15499,15501],{"className":15500},[185,186,187,188],[100,15502,15504],{"className":15503},[150,188],[100,15505,1258],{"className":15506,"style":1436},[150,154,188],[100,15508,200],{"className":15509},[199],[100,15511,15513],{"className":15512},[168],[100,15514,15516],{"className":15515,"style":1787},[172],[100,15517],{},[100,15519],{"className":15520,"style":777},[354],[100,15522,15524,15527],{"className":15523},[150],[100,15525,607],{"className":15526},[150],[100,15528,15530],{"className":15529},[159],[100,15531,15533,15556],{"className":15532},[163,164],[100,15534,15536,15553],{"className":15535},[168],[100,15537,15539],{"className":15538,"style":173},[172],[100,15540,15541,15544],{"style":809},[100,15542],{"className":15543,"style":181},[180],[100,15545,15547],{"className":15546},[185,186,187,188],[100,15548,15550],{"className":15549},[150,188],[100,15551,127],{"className":15552,"style":195},[150,154,188],[100,15554,200],{"className":15555},[199],[100,15557,15559],{"className":15558},[168],[100,15560,15562],{"className":15561,"style":207},[172],[100,15563],{},[100,15565],{"className":15566,"style":777},[354],[100,15568,2116,15570],{"className":15569},[376],[100,15571,2120],{"style":2119},[100,15573],{"className":15574,"style":777},[354],[100,15576,15578,15581],{"className":15577},[150],[100,15579,124],{"className":15580,"style":155},[150,154],[100,15582,15584],{"className":15583},[159],[100,15585,15587,15610],{"className":15586},[163,164],[100,15588,15590,15607],{"className":15589},[168],[100,15591,15593],{"className":15592,"style":173},[172],[100,15594,15595,15598],{"style":176},[100,15596],{"className":15597,"style":181},[180],[100,15599,15601],{"className":15600},[185,186,187,188],[100,15602,15604],{"className":15603},[150,188],[100,15605,127],{"className":15606,"style":195},[150,154,188],[100,15608,200],{"className":15609},[199],[100,15611,15613],{"className":15612},[168],[100,15614,15616],{"className":15615,"style":207},[172],[100,15617],{},[100,15619,230],{"className":15620},[303],[100,15622,15624,15627],{"className":15623},[150],[100,15625,18],{"className":15626},[150,154],[100,15628,15630],{"className":15629},[159],[100,15631,15633,15653],{"className":15632},[163,164],[100,15634,15636,15650],{"className":15635},[168],[100,15637,15639],{"className":15638,"style":1869},[172],[100,15640,15641,15644],{"style":809},[100,15642],{"className":15643,"style":181},[180],[100,15645,15647],{"className":15646},[185,186,187,188],[100,15648,1515],{"className":15649},[150,154,188],[100,15651,200],{"className":15652},[199],[100,15654,15656],{"className":15655},[168],[100,15657,15659],{"className":15658,"style":207},[172],[100,15660],{},[100,15662,624],{"className":15663},[150],[100,15665,15667,15670],{"className":15666},[150],[100,15668,1226],{"className":15669},[150,154],[100,15671,15673],{"className":15672},[159],[100,15674,15676,15696],{"className":15675},[163,164],[100,15677,15679,15693],{"className":15678},[168],[100,15680,15682],{"className":15681,"style":1869},[172],[100,15683,15684,15687],{"style":809},[100,15685],{"className":15686,"style":181},[180],[100,15688,15690],{"className":15689},[185,186,187,188],[100,15691,1515],{"className":15692},[150,154,188],[100,15694,200],{"className":15695},[199],[100,15697,15699],{"className":15698},[168],[100,15700,15702],{"className":15701,"style":207},[172],[100,15703],{},[100,15705,239],{"className":15706},[350],[100,15708],{"className":15709,"style":777},[354],[100,15711,15713,15719,15812,15815,15818,15821,15893,15896,15899,15971,15974,15977,16055,16058,16061,16064,16067,16070,16073,16113,16116],{"className":15712},[781],[100,15714,15716],{"className":15715,"style":889},[303,888],[100,15717,230],{"className":15718},[2830,6847],[100,15720,15722],{"className":15721},[376,377],[100,15723,15725,15804],{"className":15724},[163,164],[100,15726,15728,15801],{"className":15727},[168],[100,15729,15731,15780,15790],{"className":15730,"style":1727},[172],[100,15732,15733,15736],{"style":13389},[100,15734],{"className":15735,"style":1734},[180],[100,15737,15739],{"className":15738},[185,186,187,188],[100,15740,15742,15774,15777],{"className":15741},[150,188],[100,15743,15745,15748],{"className":15744},[150,188],[100,15746,1515],{"className":15747},[150,154,188],[100,15749,15751],{"className":15750},[159],[100,15752,15754],{"className":15753},[163],[100,15755,15757],{"className":15756},[168],[100,15758,15760],{"className":15759,"style":13417},[172],[100,15761,15762,15765],{"style":13420},[100,15763],{"className":15764,"style":433},[180],[100,15766,15768],{"className":15767},[185,437,438,188],[100,15769,15771],{"className":15770},[150,188],[100,15772,12830],{"className":15773},[150,188],[100,15775,242],{"className":15776},[359,188],[100,15778,1515],{"className":15779},[150,154,188],[100,15781,15782,15785],{"style":1752},[100,15783],{"className":15784,"style":1734},[180],[100,15786,15787],{},[100,15788,2508],{"className":15789},[376,1761,1762],[100,15791,15792,15795],{"style":1765},[100,15793],{"className":15794,"style":1734},[180],[100,15796,15798],{"className":15797},[185,186,187,188],[100,15799,1258],{"className":15800,"style":1436},[150,154,188],[100,15802,200],{"className":15803},[199],[100,15805,15807],{"className":15806},[168],[100,15808,15810],{"className":15809,"style":13469},[172],[100,15811],{},[100,15813],{"className":15814,"style":777},[354],[100,15816,273],{"className":15817,"style":487},[150,154],[100,15819,230],{"className":15820},[303],[100,15822,15824,15827],{"className":15823},[150],[100,15825,1226],{"className":15826},[150,154],[100,15828,15830],{"className":15829},[159],[100,15831,15833,15885],{"className":15832},[163,164],[100,15834,15836,15882],{"className":15835},[168],[100,15837,15839],{"className":15838,"style":13499},[172],[100,15840,15841,15844],{"style":809},[100,15842],{"className":15843,"style":181},[180],[100,15845,15847],{"className":15846},[185,186,187,188],[100,15848,15850],{"className":15849},[150,188],[100,15851,15853,15856],{"className":15852},[150,188],[100,15854,1515],{"className":15855},[150,154,188],[100,15857,15859],{"className":15858},[159],[100,15860,15862],{"className":15861},[163],[100,15863,15865],{"className":15864},[168],[100,15866,15868],{"className":15867,"style":13417},[172],[100,15869,15870,15873],{"style":13420},[100,15871],{"className":15872,"style":433},[180],[100,15874,15876],{"className":15875},[185,437,438,188],[100,15877,15879],{"className":15878},[150,188],[100,15880,12830],{"className":15881},[150,188],[100,15883,200],{"className":15884},[199],[100,15886,15888],{"className":15887},[168],[100,15889,15891],{"className":15890,"style":207},[172],[100,15892],{},[100,15894,1232],{"className":15895},[1343],[100,15897],{"className":15898,"style":777},[354],[100,15900,15902,15905],{"className":15901},[150],[100,15903,18],{"className":15904},[150,154],[100,15906,15908],{"className":15907},[159],[100,15909,15911,15963],{"className":15910},[163,164],[100,15912,15914,15960],{"className":15913},[168],[100,15915,15917],{"className":15916,"style":13499},[172],[100,15918,15919,15922],{"style":809},[100,15920],{"className":15921,"style":181},[180],[100,15923,15925],{"className":15924},[185,186,187,188],[100,15926,15928],{"className":15927},[150,188],[100,15929,15931,15934],{"className":15930},[150,188],[100,15932,1515],{"className":15933},[150,154,188],[100,15935,15937],{"className":15936},[159],[100,15938,15940],{"className":15939},[163],[100,15941,15943],{"className":15942},[168],[100,15944,15946],{"className":15945,"style":13417},[172],[100,15947,15948,15951],{"style":13420},[100,15949],{"className":15950,"style":433},[180],[100,15952,15954],{"className":15953},[185,437,438,188],[100,15955,15957],{"className":15956},[150,188],[100,15958,12830],{"className":15959},[150,188],[100,15961,200],{"className":15962},[199],[100,15964,15966],{"className":15965},[168],[100,15967,15969],{"className":15968,"style":207},[172],[100,15970],{},[100,15972,1232],{"className":15973},[1343],[100,15975],{"className":15976,"style":777},[354],[100,15978,15980,15983],{"className":15979},[150],[100,15981,1226],{"className":15982},[150,154],[100,15984,15986],{"className":15985},[159],[100,15987,15989,16047],{"className":15988},[163,164],[100,15990,15992,16044],{"className":15991},[168],[100,15993,15995],{"className":15994,"style":13499},[172],[100,15996,15997,16000],{"style":809},[100,15998],{"className":15999,"style":181},[180],[100,16001,16003],{"className":16002},[185,186,187,188],[100,16004,16006,16038,16041],{"className":16005},[150,188],[100,16007,16009,16012],{"className":16008},[150,188],[100,16010,1515],{"className":16011},[150,154,188],[100,16013,16015],{"className":16014},[159],[100,16016,16018],{"className":16017},[163],[100,16019,16021],{"className":16020},[168],[100,16022,16024],{"className":16023,"style":13417},[172],[100,16025,16026,16029],{"style":13420},[100,16027],{"className":16028,"style":433},[180],[100,16030,16032],{"className":16031},[185,437,438,188],[100,16033,16035],{"className":16034},[150,188],[100,16036,12830],{"className":16037},[150,188],[100,16039,580],{"className":16040},[681,188],[100,16042,584],{"className":16043},[150,188],[100,16045,200],{"className":16046},[199],[100,16048,16050],{"className":16049},[168],[100,16051,16053],{"className":16052,"style":694},[172],[100,16054],{},[100,16056,239],{"className":16057},[350],[100,16059],{"className":16060,"style":756},[354],[100,16062,9309],{"className":16063},[681],[100,16065],{"className":16066,"style":756},[354],[100,16068,14501],{"className":16069},[150,154],[100,16071,230],{"className":16072},[303],[100,16074,16076,16079],{"className":16075},[150],[100,16077,1226],{"className":16078},[150,154],[100,16080,16082],{"className":16081},[159],[100,16083,16085,16105],{"className":16084},[163,164],[100,16086,16088,16102],{"className":16087},[168],[100,16089,16091],{"className":16090,"style":1869},[172],[100,16092,16093,16096],{"style":809},[100,16094],{"className":16095,"style":181},[180],[100,16097,16099],{"className":16098},[185,186,187,188],[100,16100,1515],{"className":16101},[150,154,188],[100,16103,200],{"className":16104},[199],[100,16106,16108],{"className":16107},[168],[100,16109,16111],{"className":16110,"style":207},[172],[100,16112],{},[100,16114,239],{"className":16115},[350],[100,16117,16119],{"className":16118,"style":889},[350,888],[100,16120,239],{"className":16121},[2830,6847],[100,16123,16125],{"className":16124,"style":889},[350,888],[100,16126,282],{"className":16127},[2830,6847],[14,16129,16130,16131,16159,16160,8489],{},"这样使用的任何函数 ",[100,16132,16134,16147],{"className":16133},[103],[100,16135,16137],{"className":16136},[107],[109,16138,16139],{"xmlns":111},[113,16140,16141,16145],{},[116,16142,16143],{},[122,16144,14501],{},[129,16146,14501],{"encoding":131},[100,16148,16150],{"className":16149,"ariaHidden":137},[136],[100,16151,16153,16156],{"className":16152},[141],[100,16154],{"className":16155,"style":3183},[145],[100,16157,14501],{"className":16158},[150,154]," 都叫作",[51,16161,16162],{},"基线（baseline）",[14,16164,16165,16166,16170,16171,16292,16293,16363,16364,16393],{},"最常见的基线选择是",[18,16167,16169],{"href":16168},"\u002Fposts\u002Fspinning-up-rl-intro#%E4%BB%B7%E5%80%BC%E5%87%BD%E6%95%B0","同策略价值函数"," ",[100,16172,16174,16203],{"className":16173},[103],[100,16175,16177],{"className":16176},[107],[109,16178,16179],{"xmlns":111},[113,16180,16181,16200],{},[116,16182,16183,16190,16192,16198],{},[12824,16184,16185,16188],{},[122,16186,16187],{},"V",[122,16189,124],{},[227,16191,230],{"stretchy":229},[119,16193,16194,16196],{},[122,16195,1226],{},[122,16197,1515],{},[227,16199,239],{"stretchy":229},[129,16201,16202],{"encoding":131},"V^{\\pi}(s_t)",[100,16204,16206],{"className":16205,"ariaHidden":137},[136],[100,16207,16209,16212,16246,16249,16289],{"className":16208},[141],[100,16210],{"className":16211,"style":295},[145],[100,16213,16215,16218],{"className":16214},[150],[100,16216,16187],{"className":16217,"style":756},[150,154],[100,16219,16221],{"className":16220},[159],[100,16222,16224],{"className":16223},[163],[100,16225,16227],{"className":16226},[168],[100,16228,16231],{"className":16229,"style":16230},[172],"height:0.6644em;",[100,16232,16234,16237],{"style":16233},"top:-3.063em;margin-right:0.05em;",[100,16235],{"className":16236,"style":181},[180],[100,16238,16240],{"className":16239},[185,186,187,188],[100,16241,16243],{"className":16242},[150,188],[100,16244,124],{"className":16245,"style":155},[150,154,188],[100,16247,230],{"className":16248},[303],[100,16250,16252,16255],{"className":16251},[150],[100,16253,1226],{"className":16254},[150,154],[100,16256,16258],{"className":16257},[159],[100,16259,16261,16281],{"className":16260},[163,164],[100,16262,16264,16278],{"className":16263},[168],[100,16265,16267],{"className":16266,"style":1869},[172],[100,16268,16269,16272],{"style":809},[100,16270],{"className":16271,"style":181},[180],[100,16273,16275],{"className":16274},[185,186,187,188],[100,16276,1515],{"className":16277},[150,154,188],[100,16279,200],{"className":16280},[199],[100,16282,16284],{"className":16283},[168],[100,16285,16287],{"className":16286,"style":207},[172],[100,16288],{},[100,16290,239],{"className":16291},[350],"。回忆一下：它指智能体从状态 ",[100,16294,16296,16314],{"className":16295},[103],[100,16297,16299],{"className":16298},[107],[109,16300,16301],{"xmlns":111},[113,16302,16303,16311],{},[116,16304,16305],{},[119,16306,16307,16309],{},[122,16308,1226],{},[122,16310,1515],{},[129,16312,16313],{"encoding":131},"s_t",[100,16315,16317],{"className":16316,"ariaHidden":137},[136],[100,16318,16320,16323],{"className":16319},[141],[100,16321],{"className":16322,"style":146},[145],[100,16324,16326,16329],{"className":16325},[150],[100,16327,1226],{"className":16328},[150,154],[100,16330,16332],{"className":16331},[159],[100,16333,16335,16355],{"className":16334},[163,164],[100,16336,16338,16352],{"className":16337},[168],[100,16339,16341],{"className":16340,"style":1869},[172],[100,16342,16343,16346],{"style":809},[100,16344],{"className":16345,"style":181},[180],[100,16347,16349],{"className":16348},[185,186,187,188],[100,16350,1515],{"className":16351},[150,154,188],[100,16353,200],{"className":16354},[199],[100,16356,16358],{"className":16357},[168],[100,16359,16361],{"className":16360,"style":207},[172],[100,16362],{}," 出发、之后一直按策略 ",[100,16365,16367,16381],{"className":16366},[103],[100,16368,16370],{"className":16369},[107],[109,16371,16372],{"xmlns":111},[113,16373,16374,16378],{},[116,16375,16376],{},[122,16377,124],{},[129,16379,16380],{"encoding":131},"\\pi",[100,16382,16384],{"className":16383,"ariaHidden":137},[136],[100,16385,16387,16390],{"className":16386},[141],[100,16388],{"className":16389,"style":1277},[145],[100,16391,124],{"className":16392,"style":155},[150,154]," 行动所能得到的平均回报。",[14,16395,16396,16397,16593],{},"经验上，取 ",[100,16398,16400,16442],{"className":16399},[103],[100,16401,16403],{"className":16402},[107],[109,16404,16405],{"xmlns":111},[113,16406,16407,16439],{},[116,16408,16409,16411,16413,16419,16421,16423,16429,16431,16437],{},[122,16410,14501],{},[227,16412,230],{"stretchy":229},[119,16414,16415,16417],{},[122,16416,1226],{},[122,16418,1515],{},[227,16420,239],{"stretchy":229},[227,16422,242],{},[12824,16424,16425,16427],{},[122,16426,16187],{},[122,16428,124],{},[227,16430,230],{"stretchy":229},[119,16432,16433,16435],{},[122,16434,1226],{},[122,16436,1515],{},[227,16438,239],{"stretchy":229},[129,16440,16441],{"encoding":131},"b(s_t) = V^{\\pi}(s_t)",[100,16443,16445,16509],{"className":16444,"ariaHidden":137},[136],[100,16446,16448,16451,16454,16457,16497,16500,16503,16506],{"className":16447},[141],[100,16449],{"className":16450,"style":295},[145],[100,16452,14501],{"className":16453},[150,154],[100,16455,230],{"className":16456},[303],[100,16458,16460,16463],{"className":16459},[150],[100,16461,1226],{"className":16462},[150,154],[100,16464,16466],{"className":16465},[159],[100,16467,16469,16489],{"className":16468},[163,164],[100,16470,16472,16486],{"className":16471},[168],[100,16473,16475],{"className":16474,"style":1869},[172],[100,16476,16477,16480],{"style":809},[100,16478],{"className":16479,"style":181},[180],[100,16481,16483],{"className":16482},[185,186,187,188],[100,16484,1515],{"className":16485},[150,154,188],[100,16487,200],{"className":16488},[199],[100,16490,16492],{"className":16491},[168],[100,16493,16495],{"className":16494,"style":207},[172],[100,16496],{},[100,16498,239],{"className":16499},[350],[100,16501],{"className":16502,"style":355},[354],[100,16504,242],{"className":16505},[359],[100,16507],{"className":16508,"style":355},[354],[100,16510,16512,16515,16547,16550,16590],{"className":16511},[141],[100,16513],{"className":16514,"style":295},[145],[100,16516,16518,16521],{"className":16517},[150],[100,16519,16187],{"className":16520,"style":756},[150,154],[100,16522,16524],{"className":16523},[159],[100,16525,16527],{"className":16526},[163],[100,16528,16530],{"className":16529},[168],[100,16531,16533],{"className":16532,"style":16230},[172],[100,16534,16535,16538],{"style":16233},[100,16536],{"className":16537,"style":181},[180],[100,16539,16541],{"className":16540},[185,186,187,188],[100,16542,16544],{"className":16543},[150,188],[100,16545,124],{"className":16546,"style":155},[150,154,188],[100,16548,230],{"className":16549},[303],[100,16551,16553,16556],{"className":16552},[150],[100,16554,1226],{"className":16555},[150,154],[100,16557,16559],{"className":16558},[159],[100,16560,16562,16582],{"className":16561},[163,164],[100,16563,16565,16579],{"className":16564},[168],[100,16566,16568],{"className":16567,"style":1869},[172],[100,16569,16570,16573],{"style":809},[100,16571],{"className":16572,"style":181},[180],[100,16574,16576],{"className":16575},[185,186,187,188],[100,16577,1515],{"className":16578},[150,154,188],[100,16580,200],{"className":16581},[199],[100,16583,16585],{"className":16584},[168],[100,16586,16588],{"className":16587,"style":207},[172],[100,16589],{},[100,16591,239],{"className":16592},[350]," 能切实降低策略梯度样本估计的方差，策略学得更快也更稳。从概念上讲，这个选择也有吸引力：它编码了这样一个直觉，如果智能体得到的就是它预期的，它对此应该「无感」。",[10,16595,16596,16849,16927,17611],{"title":8480},[14,16597,16598,16599,16716,16717,16848],{},"实践中 ",[100,16600,16602,16629],{"className":16601},[103],[100,16603,16605],{"className":16604},[107],[109,16606,16607],{"xmlns":111},[113,16608,16609,16627],{},[116,16610,16611,16617,16619,16625],{},[12824,16612,16613,16615],{},[122,16614,16187],{},[122,16616,124],{},[227,16618,230],{"stretchy":229},[119,16620,16621,16623],{},[122,16622,1226],{},[122,16624,1515],{},[227,16626,239],{"stretchy":229},[129,16628,16202],{"encoding":131},[100,16630,16632],{"className":16631,"ariaHidden":137},[136],[100,16633,16635,16638,16670,16673,16713],{"className":16634},[141],[100,16636],{"className":16637,"style":295},[145],[100,16639,16641,16644],{"className":16640},[150],[100,16642,16187],{"className":16643,"style":756},[150,154],[100,16645,16647],{"className":16646},[159],[100,16648,16650],{"className":16649},[163],[100,16651,16653],{"className":16652},[168],[100,16654,16656],{"className":16655,"style":16230},[172],[100,16657,16658,16661],{"style":16233},[100,16659],{"className":16660,"style":181},[180],[100,16662,16664],{"className":16663},[185,186,187,188],[100,16665,16667],{"className":16666},[150,188],[100,16668,124],{"className":16669,"style":155},[150,154,188],[100,16671,230],{"className":16672},[303],[100,16674,16676,16679],{"className":16675},[150],[100,16677,1226],{"className":16678},[150,154],[100,16680,16682],{"className":16681},[159],[100,16683,16685,16705],{"className":16684},[163,164],[100,16686,16688,16702],{"className":16687},[168],[100,16689,16691],{"className":16690,"style":1869},[172],[100,16692,16693,16696],{"style":809},[100,16694],{"className":16695,"style":181},[180],[100,16697,16699],{"className":16698},[185,186,187,188],[100,16700,1515],{"className":16701},[150,154,188],[100,16703,200],{"className":16704},[199],[100,16706,16708],{"className":16707},[168],[100,16709,16711],{"className":16710,"style":207},[172],[100,16712],{},[100,16714,239],{"className":16715},[350]," 算不出来，只能近似。通常用一个神经网络 ",[100,16718,16720,16749],{"className":16719},[103],[100,16721,16723],{"className":16722},[107],[109,16724,16725],{"xmlns":111},[113,16726,16727,16746],{},[116,16728,16729,16736,16738,16744],{},[119,16730,16731,16733],{},[122,16732,16187],{},[122,16734,16735],{},"ϕ",[227,16737,230],{"stretchy":229},[119,16739,16740,16742],{},[122,16741,1226],{},[122,16743,1515],{},[227,16745,239],{"stretchy":229},[129,16747,16748],{"encoding":131},"V_{\\phi}(s_t)",[100,16750,16752],{"className":16751,"ariaHidden":137},[136],[100,16753,16755,16758,16802,16805,16845],{"className":16754},[141],[100,16756],{"className":16757,"style":7424},[145],[100,16759,16761,16764],{"className":16760},[150],[100,16762,16187],{"className":16763,"style":756},[150,154],[100,16765,16767],{"className":16766},[159],[100,16768,16770,16794],{"className":16769},[163,164],[100,16771,16773,16791],{"className":16772},[168],[100,16774,16776],{"className":16775,"style":173},[172],[100,16777,16779,16782],{"style":16778},"top:-2.55em;margin-left:-0.2222em;margin-right:0.05em;",[100,16780],{"className":16781,"style":181},[180],[100,16783,16785],{"className":16784},[185,186,187,188],[100,16786,16788],{"className":16787},[150,188],[100,16789,16735],{"className":16790},[150,154,188],[100,16792,200],{"className":16793},[199],[100,16795,16797],{"className":16796},[168],[100,16798,16800],{"className":16799,"style":7530},[172],[100,16801],{},[100,16803,230],{"className":16804},[303],[100,16806,16808,16811],{"className":16807},[150],[100,16809,1226],{"className":16810},[150,154],[100,16812,16814],{"className":16813},[159],[100,16815,16817,16837],{"className":16816},[163,164],[100,16818,16820,16834],{"className":16819},[168],[100,16821,16823],{"className":16822,"style":1869},[172],[100,16824,16825,16828],{"style":809},[100,16826],{"className":16827,"style":181},[180],[100,16829,16831],{"className":16830},[185,186,187,188],[100,16832,1515],{"className":16833},[150,154,188],[100,16835,200],{"className":16836},[199],[100,16838,16840],{"className":16839},[168],[100,16841,16843],{"className":16842,"style":207},[172],[100,16844],{},[100,16846,239],{"className":16847},[350],"，与策略同时更新（让价值网络始终近似最新策略的价值函数）。",[14,16850,16851,16852,16926],{},"学习 ",[100,16853,16855,16873],{"className":16854},[103],[100,16856,16858],{"className":16857},[107],[109,16859,16860],{"xmlns":111},[113,16861,16862,16870],{},[116,16863,16864],{},[119,16865,16866,16868],{},[122,16867,16187],{},[122,16869,16735],{},[129,16871,16872],{"encoding":131},"V_{\\phi}",[100,16874,16876],{"className":16875,"ariaHidden":137},[136],[100,16877,16879,16883],{"className":16878},[141],[100,16880],{"className":16881,"style":16882},[145],"height:0.9694em;vertical-align:-0.2861em;",[100,16884,16886,16889],{"className":16885},[150],[100,16887,16187],{"className":16888,"style":756},[150,154],[100,16890,16892],{"className":16891},[159],[100,16893,16895,16918],{"className":16894},[163,164],[100,16896,16898,16915],{"className":16897},[168],[100,16899,16901],{"className":16900,"style":173},[172],[100,16902,16903,16906],{"style":16778},[100,16904],{"className":16905,"style":181},[180],[100,16907,16909],{"className":16908},[185,186,187,188],[100,16910,16912],{"className":16911},[150,188],[100,16913,16735],{"className":16914},[150,154,188],[100,16916,200],{"className":16917},[199],[100,16919,16921],{"className":16920},[168],[100,16922,16924],{"className":16923,"style":7530},[172],[100,16925],{}," 最简单的方法（VPG、TRPO、PPO 和 A2C 的大多数实现都这么做）是最小化均方误差目标：",[100,16928,16930],{"className":16929},[555],[100,16931,16933,17050],{"className":16932},[103],[100,16934,16936],{"className":16935},[107],[109,16937,16938],{"xmlns":111,"display":564},[113,16939,16940,17047],{},[116,16941,16942,16948,16950,16953,16955,16966,17002],{},[119,16943,16944,16946],{},[122,16945,16735],{},[122,16947,577],{},[227,16949,242],{},[122,16951,16952],{},"arg",[227,16954,2096],{},[246,16956,16957,16964],{},[116,16958,16959,16962],{},[122,16960,16961],{},"min",[227,16963,2096],{},[122,16965,16735],{},[122,16967,16968],{},[246,16969,16970,16974],{},[227,16971,16972],{},[122,16973,253],{"mathvariant":252},[116,16975,16976,16982,16984,16994,16996],{},[119,16977,16978,16980],{},[122,16979,1226],{},[122,16981,1515],{},[227,16983,1232],{"separator":137},[119,16985,16986,16992],{},[7623,16987,16988,16990],{"accent":137},[122,16989,273],{},[227,16991,7629],{},[122,16993,1515],{},[227,16995,261],{},[119,16997,16998,17000],{},[122,16999,124],{},[122,17001,577],{},[116,17003,17004,17006,17045],{},[227,17005,270],{"fence":137},[12824,17007,17008,17042],{},[116,17009,17010,17012,17018,17020,17026,17028,17030,17040],{},[227,17011,230],{"fence":137},[119,17013,17014,17016],{},[122,17015,16187],{},[122,17017,16735],{},[227,17019,230],{"stretchy":229},[119,17021,17022,17024],{},[122,17023,1226],{},[122,17025,1515],{},[227,17027,239],{"stretchy":229},[227,17029,9309],{},[119,17031,17032,17038],{},[7623,17033,17034,17036],{"accent":137},[122,17035,273],{},[227,17037,7629],{},[122,17039,1515],{},[227,17041,239],{"fence":137},[582,17043,17044],{},"2",[227,17046,282],{"fence":137},[129,17048,17049],{"encoding":131},"\\phi_k = \\arg \\min_{\\phi} \\underset{s_t, \\hat{R}_t \\sim \\pi_k}{\\mathbb{E}}\\left[\\left( V_{\\phi}(s_t) - \\hat{R}_t \\right)^2\\right]",[100,17051,17053,17108],{"className":17052,"ariaHidden":137},[136],[100,17054,17056,17059,17099,17102,17105],{"className":17055},[141],[100,17057],{"className":17058,"style":2112},[145],[100,17060,17062,17065],{"className":17061},[150],[100,17063,16735],{"className":17064},[150,154],[100,17066,17068],{"className":17067},[159],[100,17069,17071,17091],{"className":17070},[163,164],[100,17072,17074,17088],{"className":17073},[168],[100,17075,17077],{"className":17076,"style":173},[172],[100,17078,17079,17082],{"style":809},[100,17080],{"className":17081,"style":181},[180],[100,17083,17085],{"className":17084},[185,186,187,188],[100,17086,577],{"className":17087,"style":677},[150,154,188],[100,17089,200],{"className":17090},[199],[100,17092,17094],{"className":17093},[168],[100,17095,17097],{"className":17096,"style":207},[172],[100,17098],{},[100,17100],{"className":17101,"style":355},[354],[100,17103,242],{"className":17104},[359],[100,17106],{"className":17107,"style":355},[354],[100,17109,17111,17115,17121,17124,17174,17177,17383,17386],{"className":17110},[141],[100,17112],{"className":17113,"style":17114},[145],"height:2.5148em;vertical-align:-1.0648em;",[100,17116,17118,17119],{"className":17117},[376],"ar",[100,17120,2120],{"style":2119},[100,17122],{"className":17123,"style":777},[354],[100,17125,17127],{"className":17126},[376,377],[100,17128,17130,17165],{"className":17129},[163,164],[100,17131,17133,17162],{"className":17132},[168],[100,17134,17137,17152],{"className":17135,"style":17136},[172],"height:0.6679em;",[100,17138,17140,17143],{"style":17139},"top:-2.3479em;margin-left:0em;",[100,17141],{"className":17142,"style":394},[180],[100,17144,17146],{"className":17145},[185,186,187,188],[100,17147,17149],{"className":17148},[150,188],[100,17150,16735],{"className":17151},[150,154,188],[100,17153,17154,17157],{"style":459},[100,17155],{"className":17156,"style":394},[180],[100,17158,17159],{},[100,17160,16961],{"className":17161},[376],[100,17163,200],{"className":17164},[199],[100,17166,17168],{"className":17167},[168],[100,17169,17172],{"className":17170,"style":17171},[172],"height:0.8882em;",[100,17173],{},[100,17175],{"className":17176,"style":777},[354],[100,17178,17180],{"className":17179},[150],[100,17181,17183],{"className":17182},[376,377],[100,17184,17186,17374],{"className":17185},[163,164],[100,17187,17189,17371],{"className":17188},[168],[100,17190,17192,17361],{"className":17191,"style":387},[172],[100,17193,17195,17198],{"style":17194},"top:-2.1713em;margin-left:0em;",[100,17196],{"className":17197,"style":394},[180],[100,17199,17201],{"className":17200},[185,186,187,188],[100,17202,17204,17244,17247,17318,17321],{"className":17203},[150,188],[100,17205,17207,17210],{"className":17206},[150,188],[100,17208,1226],{"className":17209},[150,154,188],[100,17211,17213],{"className":17212},[159],[100,17214,17216,17236],{"className":17215},[163,164],[100,17217,17219,17233],{"className":17218},[168],[100,17220,17222],{"className":17221,"style":14668},[172],[100,17223,17224,17227],{"style":14671},[100,17225],{"className":17226,"style":433},[180],[100,17228,17230],{"className":17229},[185,437,438,188],[100,17231,1515],{"className":17232},[150,154,188],[100,17234,200],{"className":17235},[199],[100,17237,17239],{"className":17238},[168],[100,17240,17242],{"className":17241,"style":14690},[172],[100,17243],{},[100,17245,1232],{"className":17246},[1343,188],[100,17248,17250,17283],{"className":17249},[150,188],[100,17251,17253],{"className":17252},[150,7729,188],[100,17254,17256],{"className":17255},[163],[100,17257,17259],{"className":17258},[168],[100,17260,17262,17271],{"className":17261,"style":13855},[172],[100,17263,17265,17268],{"style":17264},"top:-2.7em;",[100,17266],{"className":17267,"style":181},[180],[100,17269,273],{"className":17270,"style":487},[150,154,188],[100,17272,17274,17277],{"style":17273},"top:-2.9523em;",[100,17275],{"className":17276,"style":181},[180],[100,17278,17280],{"className":17279,"style":13873},[7755],[100,17281,7629],{"className":17282},[150,188],[100,17284,17286],{"className":17285},[159],[100,17287,17289,17310],{"className":17288},[163,164],[100,17290,17292,17307],{"className":17291},[168],[100,17293,17295],{"className":17294,"style":14668},[172],[100,17296,17298,17301],{"style":17297},"top:-2.357em;margin-left:-0.0077em;margin-right:0.0714em;",[100,17299],{"className":17300,"style":433},[180],[100,17302,17304],{"className":17303},[185,437,438,188],[100,17305,1515],{"className":17306},[150,154,188],[100,17308,200],{"className":17309},[199],[100,17311,17313],{"className":17312},[168],[100,17314,17316],{"className":17315,"style":14690},[172],[100,17317],{},[100,17319,261],{"className":17320},[359,188],[100,17322,17324,17327],{"className":17323},[150,188],[100,17325,124],{"className":17326,"style":155},[150,154,188],[100,17328,17330],{"className":17329},[159],[100,17331,17333,17353],{"className":17332},[163,164],[100,17334,17336,17350],{"className":17335},[168],[100,17337,17339],{"className":17338,"style":426},[172],[100,17340,17341,17344],{"style":429},[100,17342],{"className":17343,"style":433},[180],[100,17345,17347],{"className":17346},[185,437,438,188],[100,17348,577],{"className":17349,"style":677},[150,154,188],[100,17351,200],{"className":17352},[199],[100,17354,17356],{"className":17355},[168],[100,17357,17359],{"className":17358,"style":454},[172],[100,17360],{},[100,17362,17363,17366],{"style":459},[100,17364],{"className":17365,"style":394},[180],[100,17367,17368],{},[100,17369,253],{"className":17370},[376,468],[100,17372,200],{"className":17373},[199],[100,17375,17377],{"className":17376},[168],[100,17378,17381],{"className":17379,"style":17380},[172],"height:1.0648em;",[100,17382],{},[100,17384],{"className":17385,"style":777},[354],[100,17387,17389,17395,17605],{"className":17388},[781],[100,17390,17392],{"className":17391,"style":889},[303,888],[100,17393,270],{"className":17394},[2830,187],[100,17396,17398,17580],{"className":17397},[781],[100,17399,17401,17408,17451,17454,17494,17497,17500,17503,17506,17574],{"className":17400},[781],[100,17402,17404],{"className":17403,"style":889},[303,888],[100,17405,230],{"className":17406},[2830,17407],"size2",[100,17409,17411,17414],{"className":17410},[150],[100,17412,16187],{"className":17413,"style":756},[150,154],[100,17415,17417],{"className":17416},[159],[100,17418,17420,17443],{"className":17419},[163,164],[100,17421,17423,17440],{"className":17422},[168],[100,17424,17426],{"className":17425,"style":173},[172],[100,17427,17428,17431],{"style":16778},[100,17429],{"className":17430,"style":181},[180],[100,17432,17434],{"className":17433},[185,186,187,188],[100,17435,17437],{"className":17436},[150,188],[100,17438,16735],{"className":17439},[150,154,188],[100,17441,200],{"className":17442},[199],[100,17444,17446],{"className":17445},[168],[100,17447,17449],{"className":17448,"style":7530},[172],[100,17450],{},[100,17452,230],{"className":17453},[303],[100,17455,17457,17460],{"className":17456},[150],[100,17458,1226],{"className":17459},[150,154],[100,17461,17463],{"className":17462},[159],[100,17464,17466,17486],{"className":17465},[163,164],[100,17467,17469,17483],{"className":17468},[168],[100,17470,17472],{"className":17471,"style":1869},[172],[100,17473,17474,17477],{"style":809},[100,17475],{"className":17476,"style":181},[180],[100,17478,17480],{"className":17479},[185,186,187,188],[100,17481,1515],{"className":17482},[150,154,188],[100,17484,200],{"className":17485},[199],[100,17487,17489],{"className":17488},[168],[100,17490,17492],{"className":17491,"style":207},[172],[100,17493],{},[100,17495,239],{"className":17496},[350],[100,17498],{"className":17499,"style":756},[354],[100,17501,9309],{"className":17502},[681],[100,17504],{"className":17505,"style":756},[354],[100,17507,17509,17540],{"className":17508},[150],[100,17510,17512],{"className":17511},[150,7729],[100,17513,17515],{"className":17514},[163],[100,17516,17518],{"className":17517},[168],[100,17519,17521,17529],{"className":17520,"style":13855},[172],[100,17522,17523,17526],{"style":459},[100,17524],{"className":17525,"style":394},[180],[100,17527,273],{"className":17528,"style":487},[150,154],[100,17530,17531,17534],{"style":13866},[100,17532],{"className":17533,"style":394},[180],[100,17535,17537],{"className":17536,"style":13873},[7755],[100,17538,7629],{"className":17539},[150],[100,17541,17543],{"className":17542},[159],[100,17544,17546,17566],{"className":17545},[163,164],[100,17547,17549,17563],{"className":17548},[168],[100,17550,17552],{"className":17551,"style":1869},[172],[100,17553,17554,17557],{"style":13891},[100,17555],{"className":17556,"style":181},[180],[100,17558,17560],{"className":17559},[185,186,187,188],[100,17561,1515],{"className":17562},[150,154,188],[100,17564,200],{"className":17565},[199],[100,17567,17569],{"className":17568},[168],[100,17570,17572],{"className":17571,"style":207},[172],[100,17573],{},[100,17575,17577],{"className":17576,"style":889},[350,888],[100,17578,239],{"className":17579},[2830,17407],[100,17581,17583],{"className":17582},[159],[100,17584,17586],{"className":17585},[163],[100,17587,17589],{"className":17588},[168],[100,17590,17593],{"className":17591,"style":17592},[172],"height:1.354em;",[100,17594,17596,17599],{"style":17595},"top:-3.6029em;margin-right:0.05em;",[100,17597],{"className":17598,"style":181},[180],[100,17600,17602],{"className":17601},[185,186,187,188],[100,17603,17044],{"className":17604},[150,188],[100,17606,17608],{"className":17607,"style":889},[350,888],[100,17609,282],{"className":17610},[2830,187],[14,17612,8195,17613,17683,17684,17712,17713,17798],{},[100,17614,17616,17634],{"className":17615},[103],[100,17617,17619],{"className":17618},[107],[109,17620,17621],{"xmlns":111},[113,17622,17623,17631],{},[116,17624,17625],{},[119,17626,17627,17629],{},[122,17628,124],{},[122,17630,577],{},[129,17632,17633],{"encoding":131},"\\pi_k",[100,17635,17637],{"className":17636,"ariaHidden":137},[136],[100,17638,17640,17643],{"className":17639},[141],[100,17641],{"className":17642,"style":146},[145],[100,17644,17646,17649],{"className":17645},[150],[100,17647,124],{"className":17648,"style":155},[150,154],[100,17650,17652],{"className":17651},[159],[100,17653,17655,17675],{"className":17654},[163,164],[100,17656,17658,17672],{"className":17657},[168],[100,17659,17661],{"className":17660,"style":173},[172],[100,17662,17663,17666],{"style":176},[100,17664],{"className":17665,"style":181},[180],[100,17667,17669],{"className":17668},[185,186,187,188],[100,17670,577],{"className":17671,"style":677},[150,154,188],[100,17673,200],{"className":17674},[199],[100,17676,17678],{"className":17677},[168],[100,17679,17681],{"className":17680,"style":207},[172],[100,17682],{}," 是第 ",[100,17685,17687,17700],{"className":17686},[103],[100,17688,17690],{"className":17689},[107],[109,17691,17692],{"xmlns":111},[113,17693,17694,17698],{},[116,17695,17696],{},[122,17697,577],{},[129,17699,577],{"encoding":131},[100,17701,17703],{"className":17702,"ariaHidden":137},[136],[100,17704,17706,17709],{"className":17705},[141],[100,17707],{"className":17708,"style":3183},[145],[100,17710,577],{"className":17711,"style":677},[150,154]," 轮的策略。具体做法是从上一轮的价值参数 ",[100,17714,17716,17740],{"className":17715},[103],[100,17717,17719],{"className":17718},[107],[109,17720,17721],{"xmlns":111},[113,17722,17723,17737],{},[116,17724,17725],{},[119,17726,17727,17729],{},[122,17728,16735],{},[116,17730,17731,17733,17735],{},[122,17732,577],{},[227,17734,9309],{},[582,17736,584],{},[129,17738,17739],{"encoding":131},"\\phi_{k-1}",[100,17741,17743],{"className":17742,"ariaHidden":137},[136],[100,17744,17746,17749],{"className":17745},[141],[100,17747],{"className":17748,"style":643},[145],[100,17750,17752,17755],{"className":17751},[150],[100,17753,16735],{"className":17754},[150,154],[100,17756,17758],{"className":17757},[159],[100,17759,17761,17790],{"className":17760},[163,164],[100,17762,17764,17787],{"className":17763},[168],[100,17765,17767],{"className":17766,"style":173},[172],[100,17768,17769,17772],{"style":809},[100,17770],{"className":17771,"style":181},[180],[100,17773,17775],{"className":17774},[185,186,187,188],[100,17776,17778,17781,17784],{"className":17777},[150,188],[100,17779,577],{"className":17780,"style":677},[150,154,188],[100,17782,9309],{"className":17783},[681,188],[100,17785,584],{"className":17786},[150,188],[100,17788,200],{"className":17789},[199],[100,17791,17793],{"className":17792},[168],[100,17794,17796],{"className":17795,"style":694},[172],[100,17797],{}," 出发，做一步或多步梯度下降。",[93,17800,17801],{"id":17801},"策略梯度的其他形式",[14,17803,17804],{},"到目前为止我们看到，策略梯度有如下一般形式：",[100,17806,17808],{"className":17807},[555],[100,17809,17811,17924],{"className":17810},[103],[100,17812,17814],{"className":17813},[107],[109,17815,17816],{"xmlns":111,"display":564},[113,17817,17818,17921],{},[116,17819,17820,17826,17828,17830,17836,17838,17840,17860],{},[119,17821,17822,17824],{},[122,17823,607],{"mathvariant":606},[122,17825,127],{},[122,17827,225],{},[227,17829,230],{"stretchy":229},[119,17831,17832,17834],{},[122,17833,124],{},[122,17835,127],{},[227,17837,239],{"stretchy":229},[227,17839,242],{},[122,17841,17842],{},[246,17843,17844,17848],{},[227,17845,17846],{},[122,17847,253],{"mathvariant":252},[116,17849,17850,17852,17854],{},[122,17851,258],{},[227,17853,261],{},[119,17855,17856,17858],{},[122,17857,124],{},[122,17859,127],{},[116,17861,17862,17864,17878,17884,17886,17888,17894,17896,17902,17904,17910,17912,17919],{},[227,17863,270],{"fence":137},[1506,17865,17866,17868,17876],{},[227,17867,2508],{},[116,17869,17870,17872,17874],{},[122,17871,1515],{},[227,17873,242],{},[582,17875,1229],{},[122,17877,1258],{},[119,17879,17880,17882],{},[122,17881,607],{"mathvariant":606},[122,17883,127],{},[122,17885,2093],{},[227,17887,2096],{},[119,17889,17890,17892],{},[122,17891,124],{},[122,17893,127],{},[227,17895,230],{"stretchy":229},[119,17897,17898,17900],{},[122,17899,18],{},[122,17901,1515],{},[122,17903,624],{"mathvariant":606},[119,17905,17906,17908],{},[122,17907,1226],{},[122,17909,1515],{},[227,17911,239],{"stretchy":229},[119,17913,17914,17917],{},[122,17915,17916],{"mathvariant":606},"Φ",[122,17918,1515],{},[227,17920,282],{"fence":137},[129,17922,17923],{"encoding":131},"\\nabla_{\\theta} J(\\pi_{\\theta}) = \\underset{\\tau \\sim \\pi_{\\theta}}{\\mathbb{E}}\\left[\\sum_{t=0}^{T} \\nabla_{\\theta} \\log \\pi_{\\theta}(a_t |s_t) \\Phi_t\\right]",[100,17925,17927,18037],{"className":17926,"ariaHidden":137},[136],[100,17928,17930,17933,17976,17979,17982,18025,18028,18031,18034],{"className":17929},[141],[100,17931],{"className":17932,"style":295},[145],[100,17934,17936,17939],{"className":17935},[150],[100,17937,607],{"className":17938},[150],[100,17940,17942],{"className":17941},[159],[100,17943,17945,17968],{"className":17944},[163,164],[100,17946,17948,17965],{"className":17947},[168],[100,17949,17951],{"className":17950,"style":173},[172],[100,17952,17953,17956],{"style":809},[100,17954],{"className":17955,"style":181},[180],[100,17957,17959],{"className":17958},[185,186,187,188],[100,17960,17962],{"className":17961},[150,188],[100,17963,127],{"className":17964,"style":195},[150,154,188],[100,17966,200],{"className":17967},[199],[100,17969,17971],{"className":17970},[168],[100,17972,17974],{"className":17973,"style":207},[172],[100,17975],{},[100,17977,225],{"className":17978,"style":299},[150,154],[100,17980,230],{"className":17981},[303],[100,17983,17985,17988],{"className":17984},[150],[100,17986,124],{"className":17987,"style":155},[150,154],[100,17989,17991],{"className":17990},[159],[100,17992,17994,18017],{"className":17993},[163,164],[100,17995,17997,18014],{"className":17996},[168],[100,17998,18000],{"className":17999,"style":173},[172],[100,18001,18002,18005],{"style":176},[100,18003],{"className":18004,"style":181},[180],[100,18006,18008],{"className":18007},[185,186,187,188],[100,18009,18011],{"className":18010},[150,188],[100,18012,127],{"className":18013,"style":195},[150,154,188],[100,18015,200],{"className":18016},[199],[100,18018,18020],{"className":18019},[168],[100,18021,18023],{"className":18022,"style":207},[172],[100,18024],{},[100,18026,239],{"className":18027},[350],[100,18029],{"className":18030,"style":355},[354],[100,18032,242],{"className":18033},[359],[100,18035],{"className":18036,"style":355},[354],[100,18038,18040,18043,18139,18142],{"className":18039},[141],[100,18041],{"className":18042,"style":1625},[145],[100,18044,18046],{"className":18045},[150],[100,18047,18049],{"className":18048},[376,377],[100,18050,18052,18131],{"className":18051},[163,164],[100,18053,18055,18128],{"className":18054},[168],[100,18056,18058,18118],{"className":18057,"style":387},[172],[100,18059,18060,18063],{"style":390},[100,18061],{"className":18062,"style":394},[180],[100,18064,18066],{"className":18065},[185,186,187,188],[100,18067,18069,18072,18075],{"className":18068},[150,188],[100,18070,258],{"className":18071,"style":404},[150,154,188],[100,18073,261],{"className":18074},[359,188],[100,18076,18078,18081],{"className":18077},[150,188],[100,18079,124],{"className":18080,"style":155},[150,154,188],[100,18082,18084],{"className":18083},[159],[100,18085,18087,18110],{"className":18086},[163,164],[100,18088,18090,18107],{"className":18089},[168],[100,18091,18093],{"className":18092,"style":426},[172],[100,18094,18095,18098],{"style":429},[100,18096],{"className":18097,"style":433},[180],[100,18099,18101],{"className":18100},[185,437,438,188],[100,18102,18104],{"className":18103},[150,188],[100,18105,127],{"className":18106,"style":195},[150,154,188],[100,18108,200],{"className":18109},[199],[100,18111,18113],{"className":18112},[168],[100,18114,18116],{"className":18115,"style":454},[172],[100,18117],{},[100,18119,18120,18123],{"style":459},[100,18121],{"className":18122,"style":394},[180],[100,18124,18125],{},[100,18126,253],{"className":18127},[376,468],[100,18129,200],{"className":18130},[199],[100,18132,18134],{"className":18133},[168],[100,18135,18137],{"className":18136,"style":478},[172],[100,18138],{},[100,18140],{"className":18141,"style":777},[354],[100,18143,18145,18151,18218,18221,18264,18267,18272,18275,18318,18321,18361,18364,18404,18407,18447],{"className":18144},[781],[100,18146,18148],{"className":18147,"style":889},[303,888],[100,18149,270],{"className":18150},[2830,6847],[100,18152,18154],{"className":18153},[376,377],[100,18155,18157,18210],{"className":18156},[163,164],[100,18158,18160,18207],{"className":18159},[168],[100,18161,18163,18183,18193],{"className":18162,"style":1727},[172],[100,18164,18165,18168],{"style":1730},[100,18166],{"className":18167,"style":1734},[180],[100,18169,18171],{"className":18170},[185,186,187,188],[100,18172,18174,18177,18180],{"className":18173},[150,188],[100,18175,1515],{"className":18176},[150,154,188],[100,18178,242],{"className":18179},[359,188],[100,18181,1229],{"className":18182},[150,188],[100,18184,18185,18188],{"style":1752},[100,18186],{"className":18187,"style":1734},[180],[100,18189,18190],{},[100,18191,2508],{"className":18192},[376,1761,1762],[100,18194,18195,18198],{"style":1765},[100,18196],{"className":18197,"style":1734},[180],[100,18199,18201],{"className":18200},[185,186,187,188],[100,18202,18204],{"className":18203},[150,188],[100,18205,1258],{"className":18206,"style":1436},[150,154,188],[100,18208,200],{"className":18209},[199],[100,18211,18213],{"className":18212},[168],[100,18214,18216],{"className":18215,"style":1787},[172],[100,18217],{},[100,18219],{"className":18220,"style":777},[354],[100,18222,18224,18227],{"className":18223},[150],[100,18225,607],{"className":18226},[150],[100,18228,18230],{"className":18229},[159],[100,18231,18233,18256],{"className":18232},[163,164],[100,18234,18236,18253],{"className":18235},[168],[100,18237,18239],{"className":18238,"style":173},[172],[100,18240,18241,18244],{"style":809},[100,18242],{"className":18243,"style":181},[180],[100,18245,18247],{"className":18246},[185,186,187,188],[100,18248,18250],{"className":18249},[150,188],[100,18251,127],{"className":18252,"style":195},[150,154,188],[100,18254,200],{"className":18255},[199],[100,18257,18259],{"className":18258},[168],[100,18260,18262],{"className":18261,"style":207},[172],[100,18263],{},[100,18265],{"className":18266,"style":777},[354],[100,18268,2116,18270],{"className":18269},[376],[100,18271,2120],{"style":2119},[100,18273],{"className":18274,"style":777},[354],[100,18276,18278,18281],{"className":18277},[150],[100,18279,124],{"className":18280,"style":155},[150,154],[100,18282,18284],{"className":18283},[159],[100,18285,18287,18310],{"className":18286},[163,164],[100,18288,18290,18307],{"className":18289},[168],[100,18291,18293],{"className":18292,"style":173},[172],[100,18294,18295,18298],{"style":176},[100,18296],{"className":18297,"style":181},[180],[100,18299,18301],{"className":18300},[185,186,187,188],[100,18302,18304],{"className":18303},[150,188],[100,18305,127],{"className":18306,"style":195},[150,154,188],[100,18308,200],{"className":18309},[199],[100,18311,18313],{"className":18312},[168],[100,18314,18316],{"className":18315,"style":207},[172],[100,18317],{},[100,18319,230],{"className":18320},[303],[100,18322,18324,18327],{"className":18323},[150],[100,18325,18],{"className":18326},[150,154],[100,18328,18330],{"className":18329},[159],[100,18331,18333,18353],{"className":18332},[163,164],[100,18334,18336,18350],{"className":18335},[168],[100,18337,18339],{"className":18338,"style":1869},[172],[100,18340,18341,18344],{"style":809},[100,18342],{"className":18343,"style":181},[180],[100,18345,18347],{"className":18346},[185,186,187,188],[100,18348,1515],{"className":18349},[150,154,188],[100,18351,200],{"className":18352},[199],[100,18354,18356],{"className":18355},[168],[100,18357,18359],{"className":18358,"style":207},[172],[100,18360],{},[100,18362,624],{"className":18363},[150],[100,18365,18367,18370],{"className":18366},[150],[100,18368,1226],{"className":18369},[150,154],[100,18371,18373],{"className":18372},[159],[100,18374,18376,18396],{"className":18375},[163,164],[100,18377,18379,18393],{"className":18378},[168],[100,18380,18382],{"className":18381,"style":1869},[172],[100,18383,18384,18387],{"style":809},[100,18385],{"className":18386,"style":181},[180],[100,18388,18390],{"className":18389},[185,186,187,188],[100,18391,1515],{"className":18392},[150,154,188],[100,18394,200],{"className":18395},[199],[100,18397,18399],{"className":18398},[168],[100,18400,18402],{"className":18401,"style":207},[172],[100,18403],{},[100,18405,239],{"className":18406},[350],[100,18408,18410,18413],{"className":18409},[150],[100,18411,17916],{"className":18412},[150],[100,18414,18416],{"className":18415},[159],[100,18417,18419,18439],{"className":18418},[163,164],[100,18420,18422,18436],{"className":18421},[168],[100,18423,18425],{"className":18424,"style":1869},[172],[100,18426,18427,18430],{"style":809},[100,18428],{"className":18429,"style":181},[180],[100,18431,18433],{"className":18432},[185,186,187,188],[100,18434,1515],{"className":18435},[150,154,188],[100,18437,200],{"className":18438},[199],[100,18440,18442],{"className":18441},[168],[100,18443,18445],{"className":18444,"style":207},[172],[100,18446],{},[100,18448,18450],{"className":18449,"style":889},[350,888],[100,18451,282],{"className":18452},[2830,6847],[14,18454,8195,18455,18525],{},[100,18456,18458,18476],{"className":18457},[103],[100,18459,18461],{"className":18460},[107],[109,18462,18463],{"xmlns":111},[113,18464,18465,18473],{},[116,18466,18467],{},[119,18468,18469,18471],{},[122,18470,17916],{"mathvariant":606},[122,18472,1515],{},[129,18474,18475],{"encoding":131},"\\Phi_t",[100,18477,18479],{"className":18478,"ariaHidden":137},[136],[100,18480,18482,18485],{"className":18481},[141],[100,18483],{"className":18484,"style":10036},[145],[100,18486,18488,18491],{"className":18487},[150],[100,18489,17916],{"className":18490},[150],[100,18492,18494],{"className":18493},[159],[100,18495,18497,18517],{"className":18496},[163,164],[100,18498,18500,18514],{"className":18499},[168],[100,18501,18503],{"className":18502,"style":1869},[172],[100,18504,18505,18508],{"style":809},[100,18506],{"className":18507,"style":181},[180],[100,18509,18511],{"className":18510},[185,186,187,188],[100,18512,1515],{"className":18513},[150,154,188],[100,18515,200],{"className":18516},[199],[100,18518,18520],{"className":18519},[168],[100,18521,18523],{"className":18522,"style":207},[172],[100,18524],{}," 可以取",[100,18527,18529],{"className":18528},[555],[100,18530,18532,18560],{"className":18531},[103],[100,18533,18535],{"className":18534},[107],[109,18536,18537],{"xmlns":111,"display":564},[113,18538,18539,18557],{},[116,18540,18541,18547,18549,18551,18553,18555],{},[119,18542,18543,18545],{},[122,18544,17916],{"mathvariant":606},[122,18546,1515],{},[227,18548,242],{},[122,18550,273],{},[227,18552,230],{"stretchy":229},[122,18554,258],{},[227,18556,239],{"stretchy":229},[129,18558,18559],{"encoding":131},"\\Phi_t = R(\\tau)",[100,18561,18563,18618],{"className":18562,"ariaHidden":137},[136],[100,18564,18566,18569,18609,18612,18615],{"className":18565},[141],[100,18567],{"className":18568,"style":10036},[145],[100,18570,18572,18575],{"className":18571},[150],[100,18573,17916],{"className":18574},[150],[100,18576,18578],{"className":18577},[159],[100,18579,18581,18601],{"className":18580},[163,164],[100,18582,18584,18598],{"className":18583},[168],[100,18585,18587],{"className":18586,"style":1869},[172],[100,18588,18589,18592],{"style":809},[100,18590],{"className":18591,"style":181},[180],[100,18593,18595],{"className":18594},[185,186,187,188],[100,18596,1515],{"className":18597},[150,154,188],[100,18599,200],{"className":18600},[199],[100,18602,18604],{"className":18603},[168],[100,18605,18607],{"className":18606,"style":207},[172],[100,18608],{},[100,18610],{"className":18611,"style":355},[354],[100,18613,242],{"className":18614},[359],[100,18616],{"className":18617,"style":355},[354],[100,18619,18621,18624,18627,18630,18633],{"className":18620},[141],[100,18622],{"className":18623,"style":295},[145],[100,18625,273],{"className":18626,"style":487},[150,154],[100,18628,230],{"className":18629},[303],[100,18631,258],{"className":18632,"style":404},[150,154],[100,18634,239],{"className":18635},[350],[14,18637,18638],{},"或",[100,18640,18642],{"className":18641},[555],[100,18643,18645,18729],{"className":18644},[103],[100,18646,18648],{"className":18647},[107],[109,18649,18650],{"xmlns":111,"display":564},[113,18651,18652,18726],{},[116,18653,18654,18660,18662,18680,18682,18684,18694,18696,18706,18708,18724],{},[119,18655,18656,18658],{},[122,18657,17916],{"mathvariant":606},[122,18659,1515],{},[227,18661,242],{},[1506,18663,18664,18666,18678],{},[227,18665,2508],{},[116,18667,18668,18674,18676],{},[12824,18669,18670,18672],{},[122,18671,1515],{},[227,18673,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,18675,242],{},[122,18677,1515],{},[122,18679,1258],{},[122,18681,273],{},[227,18683,230],{"stretchy":229},[119,18685,18686,18688],{},[122,18687,1226],{},[12824,18689,18690,18692],{},[122,18691,1515],{},[227,18693,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,18695,1232],{"separator":137},[119,18697,18698,18700],{},[122,18699,18],{},[12824,18701,18702,18704],{},[122,18703,1515],{},[227,18705,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,18707,1232],{"separator":137},[119,18709,18710,18712],{},[122,18711,1226],{},[116,18713,18714,18720,18722],{},[12824,18715,18716,18718],{},[122,18717,1515],{},[227,18719,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,18721,580],{},[582,18723,584],{},[227,18725,239],{"stretchy":229},[129,18727,18728],{"encoding":131},"\\Phi_t = \\sum_{t'=t}^T R(s_{t'}, a_{t'}, s_{t'+1})",[100,18730,18732,18787],{"className":18731,"ariaHidden":137},[136],[100,18733,18735,18738,18778,18781,18784],{"className":18734},[141],[100,18736],{"className":18737,"style":10036},[145],[100,18739,18741,18744],{"className":18740},[150],[100,18742,17916],{"className":18743},[150],[100,18745,18747],{"className":18746},[159],[100,18748,18750,18770],{"className":18749},[163,164],[100,18751,18753,18767],{"className":18752},[168],[100,18754,18756],{"className":18755,"style":1869},[172],[100,18757,18758,18761],{"style":809},[100,18759],{"className":18760,"style":181},[180],[100,18762,18764],{"className":18763},[185,186,187,188],[100,18765,1515],{"className":18766},[150,154,188],[100,18768,200],{"className":18769},[199],[100,18771,18773],{"className":18772},[168],[100,18774,18776],{"className":18775,"style":207},[172],[100,18777],{},[100,18779],{"className":18780,"style":355},[354],[100,18782,242],{"className":18783},[359],[100,18785],{"className":18786,"style":355},[354],[100,18788,18790,18793,18886,18889,18892,18895,18967,18970,18973,19045,19048,19051,19129],{"className":18789},[141],[100,18791],{"className":18792,"style":13007},[145],[100,18794,18796],{"className":18795},[376,377],[100,18797,18799,18878],{"className":18798},[163,164],[100,18800,18802,18875],{"className":18801},[168],[100,18803,18805,18854,18864],{"className":18804,"style":1727},[172],[100,18806,18807,18810],{"style":13389},[100,18808],{"className":18809,"style":1734},[180],[100,18811,18813],{"className":18812},[185,186,187,188],[100,18814,18816,18848,18851],{"className":18815},[150,188],[100,18817,18819,18822],{"className":18818},[150,188],[100,18820,1515],{"className":18821},[150,154,188],[100,18823,18825],{"className":18824},[159],[100,18826,18828],{"className":18827},[163],[100,18829,18831],{"className":18830},[168],[100,18832,18834],{"className":18833,"style":13417},[172],[100,18835,18836,18839],{"style":13420},[100,18837],{"className":18838,"style":433},[180],[100,18840,18842],{"className":18841},[185,437,438,188],[100,18843,18845],{"className":18844},[150,188],[100,18846,12830],{"className":18847},[150,188],[100,18849,242],{"className":18850},[359,188],[100,18852,1515],{"className":18853},[150,154,188],[100,18855,18856,18859],{"style":1752},[100,18857],{"className":18858,"style":1734},[180],[100,18860,18861],{},[100,18862,2508],{"className":18863},[376,1761,1762],[100,18865,18866,18869],{"style":1765},[100,18867],{"className":18868,"style":1734},[180],[100,18870,18872],{"className":18871},[185,186,187,188],[100,18873,1258],{"className":18874,"style":1436},[150,154,188],[100,18876,200],{"className":18877},[199],[100,18879,18881],{"className":18880},[168],[100,18882,18884],{"className":18883,"style":13469},[172],[100,18885],{},[100,18887],{"className":18888,"style":777},[354],[100,18890,273],{"className":18891,"style":487},[150,154],[100,18893,230],{"className":18894},[303],[100,18896,18898,18901],{"className":18897},[150],[100,18899,1226],{"className":18900},[150,154],[100,18902,18904],{"className":18903},[159],[100,18905,18907,18959],{"className":18906},[163,164],[100,18908,18910,18956],{"className":18909},[168],[100,18911,18913],{"className":18912,"style":13499},[172],[100,18914,18915,18918],{"style":809},[100,18916],{"className":18917,"style":181},[180],[100,18919,18921],{"className":18920},[185,186,187,188],[100,18922,18924],{"className":18923},[150,188],[100,18925,18927,18930],{"className":18926},[150,188],[100,18928,1515],{"className":18929},[150,154,188],[100,18931,18933],{"className":18932},[159],[100,18934,18936],{"className":18935},[163],[100,18937,18939],{"className":18938},[168],[100,18940,18942],{"className":18941,"style":13417},[172],[100,18943,18944,18947],{"style":13420},[100,18945],{"className":18946,"style":433},[180],[100,18948,18950],{"className":18949},[185,437,438,188],[100,18951,18953],{"className":18952},[150,188],[100,18954,12830],{"className":18955},[150,188],[100,18957,200],{"className":18958},[199],[100,18960,18962],{"className":18961},[168],[100,18963,18965],{"className":18964,"style":207},[172],[100,18966],{},[100,18968,1232],{"className":18969},[1343],[100,18971],{"className":18972,"style":777},[354],[100,18974,18976,18979],{"className":18975},[150],[100,18977,18],{"className":18978},[150,154],[100,18980,18982],{"className":18981},[159],[100,18983,18985,19037],{"className":18984},[163,164],[100,18986,18988,19034],{"className":18987},[168],[100,18989,18991],{"className":18990,"style":13499},[172],[100,18992,18993,18996],{"style":809},[100,18994],{"className":18995,"style":181},[180],[100,18997,18999],{"className":18998},[185,186,187,188],[100,19000,19002],{"className":19001},[150,188],[100,19003,19005,19008],{"className":19004},[150,188],[100,19006,1515],{"className":19007},[150,154,188],[100,19009,19011],{"className":19010},[159],[100,19012,19014],{"className":19013},[163],[100,19015,19017],{"className":19016},[168],[100,19018,19020],{"className":19019,"style":13417},[172],[100,19021,19022,19025],{"style":13420},[100,19023],{"className":19024,"style":433},[180],[100,19026,19028],{"className":19027},[185,437,438,188],[100,19029,19031],{"className":19030},[150,188],[100,19032,12830],{"className":19033},[150,188],[100,19035,200],{"className":19036},[199],[100,19038,19040],{"className":19039},[168],[100,19041,19043],{"className":19042,"style":207},[172],[100,19044],{},[100,19046,1232],{"className":19047},[1343],[100,19049],{"className":19050,"style":777},[354],[100,19052,19054,19057],{"className":19053},[150],[100,19055,1226],{"className":19056},[150,154],[100,19058,19060],{"className":19059},[159],[100,19061,19063,19121],{"className":19062},[163,164],[100,19064,19066,19118],{"className":19065},[168],[100,19067,19069],{"className":19068,"style":13499},[172],[100,19070,19071,19074],{"style":809},[100,19072],{"className":19073,"style":181},[180],[100,19075,19077],{"className":19076},[185,186,187,188],[100,19078,19080,19112,19115],{"className":19079},[150,188],[100,19081,19083,19086],{"className":19082},[150,188],[100,19084,1515],{"className":19085},[150,154,188],[100,19087,19089],{"className":19088},[159],[100,19090,19092],{"className":19091},[163],[100,19093,19095],{"className":19094},[168],[100,19096,19098],{"className":19097,"style":13417},[172],[100,19099,19100,19103],{"style":13420},[100,19101],{"className":19102,"style":433},[180],[100,19104,19106],{"className":19105},[185,437,438,188],[100,19107,19109],{"className":19108},[150,188],[100,19110,12830],{"className":19111},[150,188],[100,19113,580],{"className":19114},[681,188],[100,19116,584],{"className":19117},[150,188],[100,19119,200],{"className":19120},[199],[100,19122,19124],{"className":19123},[168],[100,19125,19127],{"className":19126,"style":694},[172],[100,19128],{},[100,19130,239],{"className":19131},[350],[14,19133,18638],{},[100,19135,19137],{"className":19136},[555],[100,19138,19140,19238],{"className":19139},[103],[100,19141,19143],{"className":19142},[107],[109,19144,19145],{"xmlns":111,"display":564},[113,19146,19147,19235],{},[116,19148,19149,19155,19157,19175,19177,19179,19189,19191,19201,19203,19219,19221,19223,19225,19227,19233],{},[119,19150,19151,19153],{},[122,19152,17916],{"mathvariant":606},[122,19154,1515],{},[227,19156,242],{},[1506,19158,19159,19161,19173],{},[227,19160,2508],{},[116,19162,19163,19169,19171],{},[12824,19164,19165,19167],{},[122,19166,1515],{},[227,19168,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,19170,242],{},[122,19172,1515],{},[122,19174,1258],{},[122,19176,273],{},[227,19178,230],{"stretchy":229},[119,19180,19181,19183],{},[122,19182,1226],{},[12824,19184,19185,19187],{},[122,19186,1515],{},[227,19188,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,19190,1232],{"separator":137},[119,19192,19193,19195],{},[122,19194,18],{},[12824,19196,19197,19199],{},[122,19198,1515],{},[227,19200,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,19202,1232],{"separator":137},[119,19204,19205,19207],{},[122,19206,1226],{},[116,19208,19209,19215,19217],{},[12824,19210,19211,19213],{},[122,19212,1515],{},[227,19214,12830],{"mathvariant":606,"lspace":3588,"rspace":3588},[227,19216,580],{},[582,19218,584],{},[227,19220,239],{"stretchy":229},[227,19222,9309],{},[122,19224,14501],{},[227,19226,230],{"stretchy":229},[119,19228,19229,19231],{},[122,19230,1226],{},[122,19232,1515],{},[227,19234,239],{"stretchy":229},[129,19236,19237],{"encoding":131},"\\Phi_t = \\sum_{t'=t}^T R(s_{t'}, a_{t'}, s_{t'+1}) - b(s_t)",[100,19239,19241,19296,19650],{"className":19240,"ariaHidden":137},[136],[100,19242,19244,19247,19287,19290,19293],{"className":19243},[141],[100,19245],{"className":19246,"style":10036},[145],[100,19248,19250,19253],{"className":19249},[150],[100,19251,17916],{"className":19252},[150],[100,19254,19256],{"className":19255},[159],[100,19257,19259,19279],{"className":19258},[163,164],[100,19260,19262,19276],{"className":19261},[168],[100,19263,19265],{"className":19264,"style":1869},[172],[100,19266,19267,19270],{"style":809},[100,19268],{"className":19269,"style":181},[180],[100,19271,19273],{"className":19272},[185,186,187,188],[100,19274,1515],{"className":19275},[150,154,188],[100,19277,200],{"className":19278},[199],[100,19280,19282],{"className":19281},[168],[100,19283,19285],{"className":19284,"style":207},[172],[100,19286],{},[100,19288],{"className":19289,"style":355},[354],[100,19291,242],{"className":19292},[359],[100,19294],{"className":19295,"style":355},[354],[100,19297,19299,19302,19395,19398,19401,19404,19476,19479,19482,19554,19557,19560,19638,19641,19644,19647],{"className":19298},[141],[100,19300],{"className":19301,"style":13007},[145],[100,19303,19305],{"className":19304},[376,377],[100,19306,19308,19387],{"className":19307},[163,164],[100,19309,19311,19384],{"className":19310},[168],[100,19312,19314,19363,19373],{"className":19313,"style":1727},[172],[100,19315,19316,19319],{"style":13389},[100,19317],{"className":19318,"style":1734},[180],[100,19320,19322],{"className":19321},[185,186,187,188],[100,19323,19325,19357,19360],{"className":19324},[150,188],[100,19326,19328,19331],{"className":19327},[150,188],[100,19329,1515],{"className":19330},[150,154,188],[100,19332,19334],{"className":19333},[159],[100,19335,19337],{"className":19336},[163],[100,19338,19340],{"className":19339},[168],[100,19341,19343],{"className":19342,"style":13417},[172],[100,19344,19345,19348],{"style":13420},[100,19346],{"className":19347,"style":433},[180],[100,19349,19351],{"className":19350},[185,437,438,188],[100,19352,19354],{"className":19353},[150,188],[100,19355,12830],{"className":19356},[150,188],[100,19358,242],{"className":19359},[359,188],[100,19361,1515],{"className":19362},[150,154,188],[100,19364,19365,19368],{"style":1752},[100,19366],{"className":19367,"style":1734},[180],[100,19369,19370],{},[100,19371,2508],{"className":19372},[376,1761,1762],[100,19374,19375,19378],{"style":1765},[100,19376],{"className":19377,"style":1734},[180],[100,19379,19381],{"className":19380},[185,186,187,188],[100,19382,1258],{"className":19383,"style":1436},[150,154,188],[100,19385,200],{"className":19386},[199],[100,19388,19390],{"className":19389},[168],[100,19391,19393],{"className":19392,"style":13469},[172],[100,19394],{},[100,19396],{"className":19397,"style":777},[354],[100,19399,273],{"className":19400,"style":487},[150,154],[100,19402,230],{"className":19403},[303],[100,19405,19407,19410],{"className":19406},[150],[100,19408,1226],{"className":19409},[150,154],[100,19411,19413],{"className":19412},[159],[100,19414,19416,19468],{"className":19415},[163,164],[100,19417,19419,19465],{"className":19418},[168],[100,19420,19422],{"className":19421,"style":13499},[172],[100,19423,19424,19427],{"style":809},[100,19425],{"className":19426,"style":181},[180],[100,19428,19430],{"className":19429},[185,186,187,188],[100,19431,19433],{"className":19432},[150,188],[100,19434,19436,19439],{"className":19435},[150,188],[100,19437,1515],{"className":19438},[150,154,188],[100,19440,19442],{"className":19441},[159],[100,19443,19445],{"className":19444},[163],[100,19446,19448],{"className":19447},[168],[100,19449,19451],{"className":19450,"style":13417},[172],[100,19452,19453,19456],{"style":13420},[100,19454],{"className":19455,"style":433},[180],[100,19457,19459],{"className":19458},[185,437,438,188],[100,19460,19462],{"className":19461},[150,188],[100,19463,12830],{"className":19464},[150,188],[100,19466,200],{"className":19467},[199],[100,19469,19471],{"className":19470},[168],[100,19472,19474],{"className":19473,"style":207},[172],[100,19475],{},[100,19477,1232],{"className":19478},[1343],[100,19480],{"className":19481,"style":777},[354],[100,19483,19485,19488],{"className":19484},[150],[100,19486,18],{"className":19487},[150,154],[100,19489,19491],{"className":19490},[159],[100,19492,19494,19546],{"className":19493},[163,164],[100,19495,19497,19543],{"className":19496},[168],[100,19498,19500],{"className":19499,"style":13499},[172],[100,19501,19502,19505],{"style":809},[100,19503],{"className":19504,"style":181},[180],[100,19506,19508],{"className":19507},[185,186,187,188],[100,19509,19511],{"className":19510},[150,188],[100,19512,19514,19517],{"className":19513},[150,188],[100,19515,1515],{"className":19516},[150,154,188],[100,19518,19520],{"className":19519},[159],[100,19521,19523],{"className":19522},[163],[100,19524,19526],{"className":19525},[168],[100,19527,19529],{"className":19528,"style":13417},[172],[100,19530,19531,19534],{"style":13420},[100,19532],{"className":19533,"style":433},[180],[100,19535,19537],{"className":19536},[185,437,438,188],[100,19538,19540],{"className":19539},[150,188],[100,19541,12830],{"className":19542},[150,188],[100,19544,200],{"className":19545},[199],[100,19547,19549],{"className":19548},[168],[100,19550,19552],{"className":19551,"style":207},[172],[100,19553],{},[100,19555,1232],{"className":19556},[1343],[100,19558],{"className":19559,"style":777},[354],[100,19561,19563,19566],{"className":19562},[150],[100,19564,1226],{"className":19565},[150,154],[100,19567,19569],{"className":19568},[159],[100,19570,19572,19630],{"className":19571},[163,164],[100,19573,19575,19627],{"className":19574},[168],[100,19576,19578],{"className":19577,"style":13499},[172],[100,19579,19580,19583],{"style":809},[100,19581],{"className":19582,"style":181},[180],[100,19584,19586],{"className":19585},[185,186,187,188],[100,19587,19589,19621,19624],{"className":19588},[150,188],[100,19590,19592,19595],{"className":19591},[150,188],[100,19593,1515],{"className":19594},[150,154,188],[100,19596,19598],{"className":19597},[159],[100,19599,19601],{"className":19600},[163],[100,19602,19604],{"className":19603},[168],[100,19605,19607],{"className":19606,"style":13417},[172],[100,19608,19609,19612],{"style":13420},[100,19610],{"className":19611,"style":433},[180],[100,19613,19615],{"className":19614},[185,437,438,188],[100,19616,19618],{"className":19617},[150,188],[100,19619,12830],{"className":19620},[150,188],[100,19622,580],{"className":19623},[681,188],[100,19625,584],{"className":19626},[150,188],[100,19628,200],{"className":19629},[199],[100,19631,19633],{"className":19632},[168],[100,19634,19636],{"className":19635,"style":694},[172],[100,19637],{},[100,19639,239],{"className":19640},[350],[100,19642],{"className":19643,"style":756},[354],[100,19645,9309],{"className":19646},[681],[100,19648],{"className":19649,"style":756},[354],[100,19651,19653,19656,19659,19662,19702],{"className":19652},[141],[100,19654],{"className":19655,"style":295},[145],[100,19657,14501],{"className":19658},[150,154],[100,19660,230],{"className":19661},[303],[100,19663,19665,19668],{"className":19664},[150],[100,19666,1226],{"className":19667},[150,154],[100,19669,19671],{"className":19670},[159],[100,19672,19674,19694],{"className":19673},[163,164],[100,19675,19677,19691],{"className":19676},[168],[100,19678,19680],{"className":19679,"style":1869},[172],[100,19681,19682,19685],{"style":809},[100,19683],{"className":19684,"style":181},[180],[100,19686,19688],{"className":19687},[185,186,187,188],[100,19689,1515],{"className":19690},[150,154,188],[100,19692,200],{"className":19693},[199],[100,19695,19697],{"className":19696},[168],[100,19698,19700],{"className":19699,"style":207},[172],[100,19701],{},[100,19703,239],{"className":19704},[350],[14,19706,19707,19708,19777],{},"尽管方差不同，这些选择给出的策略梯度期望值全都相同。事实上，",[100,19709,19711,19728],{"className":19710},[103],[100,19712,19714],{"className":19713},[107],[109,19715,19716],{"xmlns":111},[113,19717,19718,19726],{},[116,19719,19720],{},[119,19721,19722,19724],{},[122,19723,17916],{"mathvariant":606},[122,19725,1515],{},[129,19727,18475],{"encoding":131},[100,19729,19731],{"className":19730,"ariaHidden":137},[136],[100,19732,19734,19737],{"className":19733},[141],[100,19735],{"className":19736,"style":10036},[145],[100,19738,19740,19743],{"className":19739},[150],[100,19741,17916],{"className":19742},[150],[100,19744,19746],{"className":19745},[159],[100,19747,19749,19769],{"className":19748},[163,164],[100,19750,19752,19766],{"className":19751},[168],[100,19753,19755],{"className":19754,"style":1869},[172],[100,19756,19757,19760],{"style":809},[100,19758],{"className":19759,"style":181},[180],[100,19761,19763],{"className":19762},[185,186,187,188],[100,19764,1515],{"className":19765},[150,154,188],[100,19767,200],{"className":19768},[199],[100,19770,19772],{"className":19771},[168],[100,19773,19775],{"className":19774,"style":207},[172],[100,19776],{}," 还有两个同样合法的选择，也很重要，值得知道。",[14,19779,19780,19783],{},[51,19781,19782],{},"1. 同策略动作价值函数","。取",[100,19785,19787],{"className":19786},[555],[100,19788,19790,19839],{"className":19789},[103],[100,19791,19793],{"className":19792},[107],[109,19794,19795],{"xmlns":111,"display":564},[113,19796,19797,19836],{},[116,19798,19799,19805,19807,19818,19820,19826,19828,19834],{},[119,19800,19801,19803],{},[122,19802,17916],{"mathvariant":606},[122,19804,1515],{},[227,19806,242],{},[12824,19808,19809,19812],{},[122,19810,19811],{},"Q",[119,19813,19814,19816],{},[122,19815,124],{},[122,19817,127],{},[227,19819,230],{"stretchy":229},[119,19821,19822,19824],{},[122,19823,1226],{},[122,19825,1515],{},[227,19827,1232],{"separator":137},[119,19829,19830,19832],{},[122,19831,18],{},[122,19833,1515],{},[227,19835,239],{"stretchy":229},[129,19837,19838],{"encoding":131},"\\Phi_t = Q^{\\pi_{\\theta}}(s_t, a_t)",[100,19840,19842,19897],{"className":19841,"ariaHidden":137},[136],[100,19843,19845,19848,19888,19891,19894],{"className":19844},[141],[100,19846],{"className":19847,"style":10036},[145],[100,19849,19851,19854],{"className":19850},[150],[100,19852,17916],{"className":19853},[150],[100,19855,19857],{"className":19856},[159],[100,19858,19860,19880],{"className":19859},[163,164],[100,19861,19863,19877],{"className":19862},[168],[100,19864,19866],{"className":19865,"style":1869},[172],[100,19867,19868,19871],{"style":809},[100,19869],{"className":19870,"style":181},[180],[100,19872,19874],{"className":19873},[185,186,187,188],[100,19875,1515],{"className":19876},[150,154,188],[100,19878,200],{"className":19879},[199],[100,19881,19883],{"className":19882},[168],[100,19884,19886],{"className":19885,"style":207},[172],[100,19887],{},[100,19889],{"className":19890,"style":355},[354],[100,19892,242],{"className":19893},[359],[100,19895],{"className":19896,"style":355},[354],[100,19898,19900,19903,19977,19980,20020,20023,20026,20066],{"className":19899},[141],[100,19901],{"className":19902,"style":295},[145],[100,19904,19906,19909],{"className":19905},[150],[100,19907,19811],{"className":19908},[150,154],[100,19910,19912],{"className":19911},[159],[100,19913,19915],{"className":19914},[163],[100,19916,19918],{"className":19917},[168],[100,19919,19922],{"className":19920,"style":19921},[172],"height:0.7144em;",[100,19923,19925,19928],{"style":19924},"top:-3.113em;margin-right:0.05em;",[100,19926],{"className":19927,"style":181},[180],[100,19929,19931],{"className":19930},[185,186,187,188],[100,19932,19934],{"className":19933},[150,188],[100,19935,19937,19940],{"className":19936},[150,188],[100,19938,124],{"className":19939,"style":155},[150,154,188],[100,19941,19943],{"className":19942},[159],[100,19944,19946,19969],{"className":19945},[163,164],[100,19947,19949,19966],{"className":19948},[168],[100,19950,19952],{"className":19951,"style":426},[172],[100,19953,19954,19957],{"style":429},[100,19955],{"className":19956,"style":433},[180],[100,19958,19960],{"className":19959},[185,437,438,188],[100,19961,19963],{"className":19962},[150,188],[100,19964,127],{"className":19965,"style":195},[150,154,188],[100,19967,200],{"className":19968},[199],[100,19970,19972],{"className":19971},[168],[100,19973,19975],{"className":19974,"style":454},[172],[100,19976],{},[100,19978,230],{"className":19979},[303],[100,19981,19983,19986],{"className":19982},[150],[100,19984,1226],{"className":19985},[150,154],[100,19987,19989],{"className":19988},[159],[100,19990,19992,20012],{"className":19991},[163,164],[100,19993,19995,20009],{"className":19994},[168],[100,19996,19998],{"className":19997,"style":1869},[172],[100,19999,20000,20003],{"style":809},[100,20001],{"className":20002,"style":181},[180],[100,20004,20006],{"className":20005},[185,186,187,188],[100,20007,1515],{"className":20008},[150,154,188],[100,20010,200],{"className":20011},[199],[100,20013,20015],{"className":20014},[168],[100,20016,20018],{"className":20017,"style":207},[172],[100,20019],{},[100,20021,1232],{"className":20022},[1343],[100,20024],{"className":20025,"style":777},[354],[100,20027,20029,20032],{"className":20028},[150],[100,20030,18],{"className":20031},[150,154],[100,20033,20035],{"className":20034},[159],[100,20036,20038,20058],{"className":20037},[163,164],[100,20039,20041,20055],{"className":20040},[168],[100,20042,20044],{"className":20043,"style":1869},[172],[100,20045,20046,20049],{"style":809},[100,20047],{"className":20048,"style":181},[180],[100,20050,20052],{"className":20051},[185,186,187,188],[100,20053,1515],{"className":20054},[150,154,188],[100,20056,200],{"className":20057},[199],[100,20059,20061],{"className":20060},[168],[100,20062,20064],{"className":20063,"style":207},[172],[100,20065],{},[100,20067,239],{"className":20068},[350],[14,20070,20071,20072,8489],{},"也是合法的。（选读）证明见",[18,20073,20076],{"href":20074,"rel":20075},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Fextra_pg_proof2.html",[22],"这个页面",[14,20078,20079,20082,20083,20087,20088,20537],{},[51,20080,20081],{},"2. 优势函数","。回忆一下，动作的",[18,20084,20086],{"href":20085},"\u002Fposts\u002Fspinning-up-rl-intro#%E4%BC%98%E5%8A%BF%E5%87%BD%E6%95%B0","优势","定义为 ",[100,20089,20091,20172],{"className":20090},[103],[100,20092,20094],{"className":20093},[107],[109,20095,20096],{"xmlns":111},[113,20097,20098,20169],{},[116,20099,20100,20107,20109,20115,20117,20123,20125,20127,20133,20135,20141,20143,20149,20151,20153,20159,20161,20167],{},[12824,20101,20102,20105],{},[122,20103,20104],{},"A",[122,20106,124],{},[227,20108,230],{"stretchy":229},[119,20110,20111,20113],{},[122,20112,1226],{},[122,20114,1515],{},[227,20116,1232],{"separator":137},[119,20118,20119,20121],{},[122,20120,18],{},[122,20122,1515],{},[227,20124,239],{"stretchy":229},[227,20126,242],{},[12824,20128,20129,20131],{},[122,20130,19811],{},[122,20132,124],{},[227,20134,230],{"stretchy":229},[119,20136,20137,20139],{},[122,20138,1226],{},[122,20140,1515],{},[227,20142,1232],{"separator":137},[119,20144,20145,20147],{},[122,20146,18],{},[122,20148,1515],{},[227,20150,239],{"stretchy":229},[227,20152,9309],{},[12824,20154,20155,20157],{},[122,20156,16187],{},[122,20158,124],{},[227,20160,230],{"stretchy":229},[119,20162,20163,20165],{},[122,20164,1226],{},[122,20166,1515],{},[227,20168,239],{"stretchy":229},[129,20170,20171],{"encoding":131},"A^{\\pi}(s_t,a_t) = Q^{\\pi}(s_t,a_t) - V^{\\pi}(s_t)",[100,20173,20175,20314,20453],{"className":20174,"ariaHidden":137},[136],[100,20176,20178,20181,20213,20216,20256,20259,20262,20302,20305,20308,20311],{"className":20177},[141],[100,20179],{"className":20180,"style":295},[145],[100,20182,20184,20187],{"className":20183},[150],[100,20185,20104],{"className":20186},[150,154],[100,20188,20190],{"className":20189},[159],[100,20191,20193],{"className":20192},[163],[100,20194,20196],{"className":20195},[168],[100,20197,20199],{"className":20198,"style":16230},[172],[100,20200,20201,20204],{"style":16233},[100,20202],{"className":20203,"style":181},[180],[100,20205,20207],{"className":20206},[185,186,187,188],[100,20208,20210],{"className":20209},[150,188],[100,20211,124],{"className":20212,"style":155},[150,154,188],[100,20214,230],{"className":20215},[303],[100,20217,20219,20222],{"className":20218},[150],[100,20220,1226],{"className":20221},[150,154],[100,20223,20225],{"className":20224},[159],[100,20226,20228,20248],{"className":20227},[163,164],[100,20229,20231,20245],{"className":20230},[168],[100,20232,20234],{"className":20233,"style":1869},[172],[100,20235,20236,20239],{"style":809},[100,20237],{"className":20238,"style":181},[180],[100,20240,20242],{"className":20241},[185,186,187,188],[100,20243,1515],{"className":20244},[150,154,188],[100,20246,200],{"className":20247},[199],[100,20249,20251],{"className":20250},[168],[100,20252,20254],{"className":20253,"style":207},[172],[100,20255],{},[100,20257,1232],{"className":20258},[1343],[100,20260],{"className":20261,"style":777},[354],[100,20263,20265,20268],{"className":20264},[150],[100,20266,18],{"className":20267},[150,154],[100,20269,20271],{"className":20270},[159],[100,20272,20274,20294],{"className":20273},[163,164],[100,20275,20277,20291],{"className":20276},[168],[100,20278,20280],{"className":20279,"style":1869},[172],[100,20281,20282,20285],{"style":809},[100,20283],{"className":20284,"style":181},[180],[100,20286,20288],{"className":20287},[185,186,187,188],[100,20289,1515],{"className":20290},[150,154,188],[100,20292,200],{"className":20293},[199],[100,20295,20297],{"className":20296},[168],[100,20298,20300],{"className":20299,"style":207},[172],[100,20301],{},[100,20303,239],{"className":20304},[350],[100,20306],{"className":20307,"style":355},[354],[100,20309,242],{"className":20310},[359],[100,20312],{"className":20313,"style":355},[354],[100,20315,20317,20320,20352,20355,20395,20398,20401,20441,20444,20447,20450],{"className":20316},[141],[100,20318],{"className":20319,"style":295},[145],[100,20321,20323,20326],{"className":20322},[150],[100,20324,19811],{"className":20325},[150,154],[100,20327,20329],{"className":20328},[159],[100,20330,20332],{"className":20331},[163],[100,20333,20335],{"className":20334},[168],[100,20336,20338],{"className":20337,"style":16230},[172],[100,20339,20340,20343],{"style":16233},[100,20341],{"className":20342,"style":181},[180],[100,20344,20346],{"className":20345},[185,186,187,188],[100,20347,20349],{"className":20348},[150,188],[100,20350,124],{"className":20351,"style":155},[150,154,188],[100,20353,230],{"className":20354},[303],[100,20356,20358,20361],{"className":20357},[150],[100,20359,1226],{"className":20360},[150,154],[100,20362,20364],{"className":20363},[159],[100,20365,20367,20387],{"className":20366},[163,164],[100,20368,20370,20384],{"className":20369},[168],[100,20371,20373],{"className":20372,"style":1869},[172],[100,20374,20375,20378],{"style":809},[100,20376],{"className":20377,"style":181},[180],[100,20379,20381],{"className":20380},[185,186,187,188],[100,20382,1515],{"className":20383},[150,154,188],[100,20385,200],{"className":20386},[199],[100,20388,20390],{"className":20389},[168],[100,20391,20393],{"className":20392,"style":207},[172],[100,20394],{},[100,20396,1232],{"className":20397},[1343],[100,20399],{"className":20400,"style":777},[354],[100,20402,20404,20407],{"className":20403},[150],[100,20405,18],{"className":20406},[150,154],[100,20408,20410],{"className":20409},[159],[100,20411,20413,20433],{"className":20412},[163,164],[100,20414,20416,20430],{"className":20415},[168],[100,20417,20419],{"className":20418,"style":1869},[172],[100,20420,20421,20424],{"style":809},[100,20422],{"className":20423,"style":181},[180],[100,20425,20427],{"className":20426},[185,186,187,188],[100,20428,1515],{"className":20429},[150,154,188],[100,20431,200],{"className":20432},[199],[100,20434,20436],{"className":20435},[168],[100,20437,20439],{"className":20438,"style":207},[172],[100,20440],{},[100,20442,239],{"className":20443},[350],[100,20445],{"className":20446,"style":756},[354],[100,20448,9309],{"className":20449},[681],[100,20451],{"className":20452,"style":756},[354],[100,20454,20456,20459,20491,20494,20534],{"className":20455},[141],[100,20457],{"className":20458,"style":295},[145],[100,20460,20462,20465],{"className":20461},[150],[100,20463,16187],{"className":20464,"style":756},[150,154],[100,20466,20468],{"className":20467},[159],[100,20469,20471],{"className":20470},[163],[100,20472,20474],{"className":20473},[168],[100,20475,20477],{"className":20476,"style":16230},[172],[100,20478,20479,20482],{"style":16233},[100,20480],{"className":20481,"style":181},[180],[100,20483,20485],{"className":20484},[185,186,187,188],[100,20486,20488],{"className":20487},[150,188],[100,20489,124],{"className":20490,"style":155},[150,154,188],[100,20492,230],{"className":20493},[303],[100,20495,20497,20500],{"className":20496},[150],[100,20498,1226],{"className":20499},[150,154],[100,20501,20503],{"className":20502},[159],[100,20504,20506,20526],{"className":20505},[163,164],[100,20507,20509,20523],{"className":20508},[168],[100,20510,20512],{"className":20511,"style":1869},[172],[100,20513,20514,20517],{"style":809},[100,20515],{"className":20516,"style":181},[180],[100,20518,20520],{"className":20519},[185,186,187,188],[100,20521,1515],{"className":20522},[150,154,188],[100,20524,200],{"className":20525},[199],[100,20527,20529],{"className":20528},[168],[100,20530,20532],{"className":20531,"style":207},[172],[100,20533],{},[100,20535,239],{"className":20536},[350],"，描述的是（相对当前策略）它平均而言比其他动作好多少。取",[100,20539,20541],{"className":20540},[555],[100,20542,20544,20592],{"className":20543},[103],[100,20545,20547],{"className":20546},[107],[109,20548,20549],{"xmlns":111,"display":564},[113,20550,20551,20589],{},[116,20552,20553,20559,20561,20571,20573,20579,20581,20587],{},[119,20554,20555,20557],{},[122,20556,17916],{"mathvariant":606},[122,20558,1515],{},[227,20560,242],{},[12824,20562,20563,20565],{},[122,20564,20104],{},[119,20566,20567,20569],{},[122,20568,124],{},[122,20570,127],{},[227,20572,230],{"stretchy":229},[119,20574,20575,20577],{},[122,20576,1226],{},[122,20578,1515],{},[227,20580,1232],{"separator":137},[119,20582,20583,20585],{},[122,20584,18],{},[122,20586,1515],{},[227,20588,239],{"stretchy":229},[129,20590,20591],{"encoding":131},"\\Phi_t = A^{\\pi_{\\theta}}(s_t, a_t)",[100,20593,20595,20650],{"className":20594,"ariaHidden":137},[136],[100,20596,20598,20601,20641,20644,20647],{"className":20597},[141],[100,20599],{"className":20600,"style":10036},[145],[100,20602,20604,20607],{"className":20603},[150],[100,20605,17916],{"className":20606},[150],[100,20608,20610],{"className":20609},[159],[100,20611,20613,20633],{"className":20612},[163,164],[100,20614,20616,20630],{"className":20615},[168],[100,20617,20619],{"className":20618,"style":1869},[172],[100,20620,20621,20624],{"style":809},[100,20622],{"className":20623,"style":181},[180],[100,20625,20627],{"className":20626},[185,186,187,188],[100,20628,1515],{"className":20629},[150,154,188],[100,20631,200],{"className":20632},[199],[100,20634,20636],{"className":20635},[168],[100,20637,20639],{"className":20638,"style":207},[172],[100,20640],{},[100,20642],{"className":20643,"style":355},[354],[100,20645,242],{"className":20646},[359],[100,20648],{"className":20649,"style":355},[354],[100,20651,20653,20656,20728,20731,20771,20774,20777,20817],{"className":20652},[141],[100,20654],{"className":20655,"style":295},[145],[100,20657,20659,20662],{"className":20658},[150],[100,20660,20104],{"className":20661},[150,154],[100,20663,20665],{"className":20664},[159],[100,20666,20668],{"className":20667},[163],[100,20669,20671],{"className":20670},[168],[100,20672,20674],{"className":20673,"style":19921},[172],[100,20675,20676,20679],{"style":19924},[100,20677],{"className":20678,"style":181},[180],[100,20680,20682],{"className":20681},[185,186,187,188],[100,20683,20685],{"className":20684},[150,188],[100,20686,20688,20691],{"className":20687},[150,188],[100,20689,124],{"className":20690,"style":155},[150,154,188],[100,20692,20694],{"className":20693},[159],[100,20695,20697,20720],{"className":20696},[163,164],[100,20698,20700,20717],{"className":20699},[168],[100,20701,20703],{"className":20702,"style":426},[172],[100,20704,20705,20708],{"style":429},[100,20706],{"className":20707,"style":433},[180],[100,20709,20711],{"className":20710},[185,437,438,188],[100,20712,20714],{"className":20713},[150,188],[100,20715,127],{"className":20716,"style":195},[150,154,188],[100,20718,200],{"className":20719},[199],[100,20721,20723],{"className":20722},[168],[100,20724,20726],{"className":20725,"style":454},[172],[100,20727],{},[100,20729,230],{"className":20730},[303],[100,20732,20734,20737],{"className":20733},[150],[100,20735,1226],{"className":20736},[150,154],[100,20738,20740],{"className":20739},[159],[100,20741,20743,20763],{"className":20742},[163,164],[100,20744,20746,20760],{"className":20745},[168],[100,20747,20749],{"className":20748,"style":1869},[172],[100,20750,20751,20754],{"style":809},[100,20752],{"className":20753,"style":181},[180],[100,20755,20757],{"className":20756},[185,186,187,188],[100,20758,1515],{"className":20759},[150,154,188],[100,20761,200],{"className":20762},[199],[100,20764,20766],{"className":20765},[168],[100,20767,20769],{"className":20768,"style":207},[172],[100,20770],{},[100,20772,1232],{"className":20773},[1343],[100,20775],{"className":20776,"style":777},[354],[100,20778,20780,20783],{"className":20779},[150],[100,20781,18],{"className":20782},[150,154],[100,20784,20786],{"className":20785},[159],[100,20787,20789,20809],{"className":20788},[163,164],[100,20790,20792,20806],{"className":20791},[168],[100,20793,20795],{"className":20794,"style":1869},[172],[100,20796,20797,20800],{"style":809},[100,20798],{"className":20799,"style":181},[180],[100,20801,20803],{"className":20802},[185,186,187,188],[100,20804,1515],{"className":20805},[150,154,188],[100,20807,200],{"className":20808},[199],[100,20810,20812],{"className":20811},[168],[100,20813,20815],{"className":20814,"style":207},[172],[100,20816],{},[100,20818,239],{"className":20819},[350],[14,20821,20822,20823,21101],{},"也合法。证明：它等价于先取 ",[100,20824,20826,20873],{"className":20825},[103],[100,20827,20829],{"className":20828},[107],[109,20830,20831],{"xmlns":111},[113,20832,20833,20871],{},[116,20834,20835,20841,20843,20853,20855,20861,20863,20869],{},[119,20836,20837,20839],{},[122,20838,17916],{"mathvariant":606},[122,20840,1515],{},[227,20842,242],{},[12824,20844,20845,20847],{},[122,20846,19811],{},[119,20848,20849,20851],{},[122,20850,124],{},[122,20852,127],{},[227,20854,230],{"stretchy":229},[119,20856,20857,20859],{},[122,20858,1226],{},[122,20860,1515],{},[227,20862,1232],{"separator":137},[119,20864,20865,20867],{},[122,20866,18],{},[122,20868,1515],{},[227,20870,239],{"stretchy":229},[129,20872,19838],{"encoding":131},[100,20874,20876,20931],{"className":20875,"ariaHidden":137},[136],[100,20877,20879,20882,20922,20925,20928],{"className":20878},[141],[100,20880],{"className":20881,"style":10036},[145],[100,20883,20885,20888],{"className":20884},[150],[100,20886,17916],{"className":20887},[150],[100,20889,20891],{"className":20890},[159],[100,20892,20894,20914],{"className":20893},[163,164],[100,20895,20897,20911],{"className":20896},[168],[100,20898,20900],{"className":20899,"style":1869},[172],[100,20901,20902,20905],{"style":809},[100,20903],{"className":20904,"style":181},[180],[100,20906,20908],{"className":20907},[185,186,187,188],[100,20909,1515],{"className":20910},[150,154,188],[100,20912,200],{"className":20913},[199],[100,20915,20917],{"className":20916},[168],[100,20918,20920],{"className":20919,"style":207},[172],[100,20921],{},[100,20923],{"className":20924,"style":355},[354],[100,20926,242],{"className":20927},[359],[100,20929],{"className":20930,"style":355},[354],[100,20932,20934,20937,21009,21012,21052,21055,21058,21098],{"className":20933},[141],[100,20935],{"className":20936,"style":295},[145],[100,20938,20940,20943],{"className":20939},[150],[100,20941,19811],{"className":20942},[150,154],[100,20944,20946],{"className":20945},[159],[100,20947,20949],{"className":20948},[163],[100,20950,20952],{"className":20951},[168],[100,20953,20955],{"className":20954,"style":16230},[172],[100,20956,20957,20960],{"style":16233},[100,20958],{"className":20959,"style":181},[180],[100,20961,20963],{"className":20962},[185,186,187,188],[100,20964,20966],{"className":20965},[150,188],[100,20967,20969,20972],{"className":20968},[150,188],[100,20970,124],{"className":20971,"style":155},[150,154,188],[100,20973,20975],{"className":20974},[159],[100,20976,20978,21001],{"className":20977},[163,164],[100,20979,20981,20998],{"className":20980},[168],[100,20982,20984],{"className":20983,"style":426},[172],[100,20985,20986,20989],{"style":429},[100,20987],{"className":20988,"style":433},[180],[100,20990,20992],{"className":20991},[185,437,438,188],[100,20993,20995],{"className":20994},[150,188],[100,20996,127],{"className":20997,"style":195},[150,154,188],[100,20999,200],{"className":21000},[199],[100,21002,21004],{"className":21003},[168],[100,21005,21007],{"className":21006,"style":454},[172],[100,21008],{},[100,21010,230],{"className":21011},[303],[100,21013,21015,21018],{"className":21014},[150],[100,21016,1226],{"className":21017},[150,154],[100,21019,21021],{"className":21020},[159],[100,21022,21024,21044],{"className":21023},[163,164],[100,21025,21027,21041],{"className":21026},[168],[100,21028,21030],{"className":21029,"style":1869},[172],[100,21031,21032,21035],{"style":809},[100,21033],{"className":21034,"style":181},[180],[100,21036,21038],{"className":21037},[185,186,187,188],[100,21039,1515],{"className":21040},[150,154,188],[100,21042,200],{"className":21043},[199],[100,21045,21047],{"className":21046},[168],[100,21048,21050],{"className":21049,"style":207},[172],[100,21051],{},[100,21053,1232],{"className":21054},[1343],[100,21056],{"className":21057,"style":777},[354],[100,21059,21061,21064],{"className":21060},[150],[100,21062,18],{"className":21063},[150,154],[100,21065,21067],{"className":21066},[159],[100,21068,21070,21090],{"className":21069},[163,164],[100,21071,21073,21087],{"className":21072},[168],[100,21074,21076],{"className":21075,"style":1869},[172],[100,21077,21078,21081],{"style":809},[100,21079],{"className":21080,"style":181},[180],[100,21082,21084],{"className":21083},[185,186,187,188],[100,21085,1515],{"className":21086},[150,154,188],[100,21088,200],{"className":21089},[199],[100,21091,21093],{"className":21092},[168],[100,21094,21096],{"className":21095,"style":207},[172],[100,21097],{},[100,21099,239],{"className":21100},[350],"，再加一个价值函数基线，而基线我们随时可以加。",[10,21103,21104],{"title":8480},[14,21105,21106],{},"用优势函数表述策略梯度极其常见，不同算法有各自估计优势函数的办法。",[10,21108,21109,21188],{"title":8480},[14,21110,21111,21112,21117,21118,21187],{},"想更深入这个主题，应该去读 ",[18,21113,21116],{"href":21114,"rel":21115},"https:\u002F\u002Farxiv.org\u002Fabs\u002F1506.02438",[22],"Generalized Advantage Estimation","（GAE）的论文，它的背景章节深入讨论了 ",[100,21119,21121,21138],{"className":21120},[103],[100,21122,21124],{"className":21123},[107],[109,21125,21126],{"xmlns":111},[113,21127,21128,21136],{},[116,21129,21130],{},[119,21131,21132,21134],{},[122,21133,17916],{"mathvariant":606},[122,21135,1515],{},[129,21137,18475],{"encoding":131},[100,21139,21141],{"className":21140,"ariaHidden":137},[136],[100,21142,21144,21147],{"className":21143},[141],[100,21145],{"className":21146,"style":10036},[145],[100,21148,21150,21153],{"className":21149},[150],[100,21151,17916],{"className":21152},[150],[100,21154,21156],{"className":21155},[159],[100,21157,21159,21179],{"className":21158},[163,164],[100,21160,21162,21176],{"className":21161},[168],[100,21163,21165],{"className":21164,"style":1869},[172],[100,21166,21167,21170],{"style":809},[100,21168],{"className":21169,"style":181},[180],[100,21171,21173],{"className":21172},[185,186,187,188],[100,21174,1515],{"className":21175},[150,154,188],[100,21177,200],{"className":21178},[199],[100,21180,21182],{"className":21181},[168],[100,21183,21185],{"className":21184,"style":207},[172],[100,21186],{}," 的各种选择。",[14,21189,21190],{},"那篇论文接着提出了 GAE，一种在策略优化算法中近似优势函数的方法，如今使用非常广泛。比如 Spinning Up 的 VPG、TRPO、PPO 实现都用它。所以我们强烈建议研究它。",[93,21192,21193],{"id":21193},"小结",[14,21195,21196,21197,21200],{},"本章讲了策略梯度方法的基本理论，并把其中一些早期结果对接到了代码示例上。有兴趣的读者可以从这里继续，去研究后面的结果（价值函数基线、策略梯度的优势形式）如何落实到 Spinning Up 的 ",[18,21198,90],{"href":88,"rel":21199},[22]," 实现里。",[21202,21203,21204],"style",{},"html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#6A737D;--shiki-dark:#6A737D}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .s4XuR, html code.shiki .s4XuR{--shiki-default:#E36209;--shiki-dark:#FFAB70}html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}",{"title":8501,"searchDepth":8513,"depth":8545,"links":21206},[21207,21208,21209,21210,21211,21212,21213,21214],{"id":95,"depth":8513,"text":95},{"id":8463,"depth":8513,"text":8463},{"id":9993,"depth":8513,"text":9994},{"id":12016,"depth":8513,"text":12016},{"id":14291,"depth":8513,"text":14292},{"id":14483,"depth":8513,"text":14483},{"id":17801,"depth":8513,"text":17801},{"id":21193,"depth":8513,"text":21193},"技术","2026-09-28 17:49:25","OpenAI Spinning Up 教程第三部分的全文翻译：推导最简单的策略梯度、log-derivative trick、EGLP 引理、reward-to-go 与基线，配 PyTorch 示例代码。",false,"md",{},"\u002Fposts\u002Fspinning-up-policy-optimization",{"title":5,"description":21217},"posts\u002Fspinning-up-policy-optimization",[21225,21226,21227,21228],"强化学习","RL","Spinning Up","翻译","V6xLB-hDlA_bX_ifP-I32H-jGeNBsYhVvy347DnGsIE",[21231,21241,21246,21250,21258,21264,21268,21278,21280,21285,21290,21299,21303,21307,21311,21321,21328,21334,21338,21343,21351,21358,21366,21371,21377,21383,21388,21393,21400,21407,21414,21420,21427,21435,21441,21447,21453,21459,21464,21472],{"path":21232,"title":21233,"description":21234,"date":21235,"categories":21236,"tags":21237,"draft":21218},"\u002Fposts\u002Fsurvival-welcome-to-university","生存手册——欢迎来到大学","大学同时生产三样东西：可验证技能、认证信号和驯化资本。你付了四年的钱，真正拿到手的是哪一样，取决于你毕业时要进的那个市场在为什么付费。","2026-10-12 20:00:00","生活杂谈",[21238,21239,21240],"大学","求职","方法论",{"path":21242,"title":21243,"description":8501,"date":21244,"categories":8501,"tags":21245,"draft":21218},"\u002Fposts\u002Fgp-tab-测试","gp-tab-测试","2026-10-12 10:51:22",null,{"path":21247,"title":21248,"description":8501,"date":21249,"categories":8501,"tags":21245,"draft":8548},"\u002Fposts\u002Fnetwork-clean","怎么让网络变得干净","2026-10-11 19:40:02",{"path":21251,"title":21252,"description":21253,"date":21254,"categories":21236,"tags":21255,"draft":21218},"\u002Fposts\u002Fthe-age-of-hesitation","[搬运] 不敢投入的时代","为什么我们现在越来越不敢投入了。很多时候我们以为的选择，比如你有两个工作offer，现在选择好像是我到底应该考公还是考编。明明这两个选项都不是真正的选项，因为你还没有开始，也不知道最后能不能成。这个时候我们就已经开始犹豫了。","2026-10-10 11:44:12",[21228,21256,21257],"搬运","心理学",{"path":21259,"title":21260,"description":21261,"date":21262,"categories":21236,"tags":21263,"draft":21218},"\u002Fposts\u002Fwhy-happiness-look-like-same","[搬运] 为什么大家的幸福都长一个样？","为什么大家的幸福都长一个样？一种认知被广泛传播后似乎变得不断扁平，这种感觉很奇怪。","2026-10-09 13:57:15",[21228,21256,21257],{"path":21265,"title":21266,"description":8501,"date":21267,"categories":21236,"tags":21245,"draft":21218},"\u002Fposts\u002Feverything-is-paid-twice","任何事情都要被消费两次","2026-10-06 12:30:06",{"path":21269,"title":21270,"description":21271,"date":21272,"categories":21273,"tags":21274,"draft":21218},"\u002Fposts\u002Fwhy-llm-seems-worse-after-launch","为什么模型上线后，体感像是“降智”了？","权重与 KV cache 的量化都能降低 serving 成本，也可能改变输出；但体感变弱不足以证明服务后来切换了精度。","2026-09-29 21:44:08","技术杂谈",[21275,21276,21277],"LLMs","Serving","量化",{"path":21221,"title":5,"description":21217,"date":21216,"categories":21215,"tags":21279,"draft":21218},[21225,21226,21227,21228],{"path":44,"title":21281,"description":21282,"date":21283,"categories":21215,"tags":21284,"draft":21218},"[译] 深度强化学习入门（二）：强化学习算法的分类","OpenAI Spinning Up 第二部分的全文翻译：无模型与基于模型的 RL、策略优化与 Q-learning 的取舍，以及规划、专家迭代和经验增强等方法。","2026-09-28 10:20:21",[21225,21226,21227,21228],{"path":39,"title":21286,"description":21287,"date":21288,"categories":21215,"tags":21289,"draft":21218},"[译] 深度强化学习入门（一）：RL 的关键概念","OpenAI Spinning Up 教程第一部分的全文翻译：智能体与环境、状态与观测、动作空间、策略、轨迹、奖励与回报、价值函数、贝尔曼方程、优势函数，附 MDP 形式化。","2026-09-28 09:29:40",[21225,21226,21227,21228],{"path":21291,"title":21292,"description":21293,"date":21294,"categories":21236,"tags":21295,"draft":8548},"\u002Fposts\u002Fwhen-machines-come-alive","冰箱之后，生活还会怎样改变？","从厨房里的果蝇想到冰箱、移动互联网与 AI 的多轮调用：AGI 已经到来，只是昂贵的智能会怎样改变生活？","2026-09-25 06:36:22",[21296,21297,21298],"AI","随笔","科技",{"path":21300,"title":21301,"description":8501,"date":21302,"categories":8501,"tags":21245,"draft":8548},"\u002Fposts\u002Ftest","test","2026-09-20 06:37:44",{"path":21304,"title":21305,"description":8501,"date":21306,"categories":8501,"tags":21245,"draft":8548},"\u002Fposts\u002Fmathematical-foundations-of-llms-1","LLM的数学基础（1）：","2026-09-19 22:59:55",{"path":21308,"title":21309,"description":8501,"date":21310,"categories":8501,"tags":21245,"draft":8548},"\u002Fposts\u002Ftowards-maximize-serving-capatity","如何在 offline bench 里最大化 LLM serving system 的吞吐","2026-09-19 05:45:05",{"path":21312,"title":21313,"description":21314,"date":21315,"categories":21273,"tags":21316,"draft":21218},"\u002Fposts\u002Fllm-memory-compute-tradeoff","LLM 的 Memory-Compute Trade-off：从 MHA 到 MLA、Sparse 与 Linear Attention","ds4.1f 上线了。它的架构 novel 在哪，便宜和快又是从哪省出来的。","2026-09-11 12:00:00",[21275,21317,21318,21319,21320],"Attention","MLA","DeepSeek","长上下文",{"path":21322,"title":21323,"description":21324,"date":21325,"categories":21273,"tags":21326,"draft":8548},"\u002Fposts\u002Fmarkdown-extensions","Markdown 扩展语法速查","本站 Markdown 支持的扩展语法演示与速查：admonition 提示块、GitHub 仓库卡片、spoiler 遮罩、ABC 乐谱、GP 乐谱、音频播放器、浏览器可运行代码块。","2026-09-01 15:30:00",[21327],"博客",{"path":21329,"title":21330,"description":21331,"date":21332,"categories":21236,"tags":21333,"draft":21218},"\u002Fposts\u002Fwhy-we-still-need-to-write-a-blog","为什么还需要手写博客？","AI这么强，还要古法写博客吗？","2026-09-01 13:17:53",[21327],{"path":21335,"title":21336,"description":8501,"date":21337,"categories":8501,"tags":21245,"draft":8548},"\u002Fposts\u002Fllm-serving-1","LLM Serving（一）:以在H200上部署GLM-5.2为例","2026-07-21 06:15:44",{"path":21339,"title":21340,"description":8501,"date":21341,"categories":21342,"tags":21245,"draft":21218},"\u002Fposts\u002Finteresting-music-rhythm","有趣的音乐节奏","2026-07-16 12:47:24","音乐",{"path":21344,"title":21345,"description":21346,"date":21347,"categories":21215,"tags":21348,"draft":21218},"\u002Fposts\u002Ftowards-monosemanticity","LLM 可解释性（一）：SAE与单语义","读 Anthropic《Towards Monosemanticity》。从难以解释的神经元出发, 讲清superposition为什么让单个神经元混成一团, SAE怎么把多语义的激活分解成单语义特征。","2026-07-12 22:00:00",[21275,21349,21350],"可解释性","SAE",{"path":21352,"title":21353,"description":21354,"date":21355,"categories":21236,"tags":21356,"draft":8548},"\u002Fposts\u002Fpiketty-ai-china-k-shaped","用皮凯蒂的镜头看 AI 与中国 K 型社会","一次凌晨的对话整理。从 r>g 出发，看 AI 时代的资本分配、中国的世袭化趋势、体制内的二元劳动力市场，以及一个普通 00 后在一线打拼的真实牌面。","2026-06-24 03:00:00",[21297,21357,21296],"经济",{"path":21359,"title":21360,"description":8501,"date":21361,"categories":8501,"tags":21362,"draft":8548},"\u002Fposts\u002Fclaude-code-compact-behavior","cluade code compact 行为","2026-06-23 20:17:42",[21363,21364,21365],"Agent","AI Agent","Claude Code",{"path":21367,"title":21368,"description":8501,"date":21369,"categories":21215,"tags":21370,"draft":21218},"\u002Fposts\u002Fclaude-code-settings","Claude Code 配置分享","2026-06-23 14:22:12",[21275,21364,21363,21365],{"path":21372,"title":21373,"description":21374,"date":21375,"categories":21273,"tags":21376,"draft":21218},"\u002Fposts\u002Fmy-thoughts-on-the-ai-industry","简单聊聊对 AI Agent 行业的理解","写这篇文章的初衷，是我发觉逐渐忘了前两个月的 AI\u002FLLMs 是什么样的能力和情况了。意识中似乎某些在突然加速，但理性和直觉告诉我这不对，作为行内人，我知道这是一个渐进的、累积的过程。我想自己回忆一下，Agent的发展。","2026-06-20 13:14:25",[21275,21364,21363],{"path":21378,"title":21379,"description":8501,"date":21380,"categories":21215,"tags":21381,"draft":21218},"\u002Fposts\u002Frotational-positional-encoding","RoPE","2026-06-17 18:31:40",[21275,21382,21379],"Transformer",{"path":21384,"title":21385,"description":8501,"date":21386,"categories":21215,"tags":21387,"draft":21218},"\u002Fposts\u002Fintroduction-to-large-language-model","大语言模型入门基础","2026-06-16 22:36:11",[21382,21275],{"path":21389,"title":21390,"description":8501,"date":21391,"categories":21273,"tags":21392,"draft":21218},"\u002Fposts\u002Fthought-about-coding-agent","关于 Coding Agent 使用的遐思","2026-06-03 17:54:20",[21363],{"path":21394,"title":21395,"description":21396,"date":21397,"categories":21215,"tags":21398,"draft":21218},"\u002Fposts\u002Fabout-ssh-usage","关于SSH的使用——客户端篇","介绍SSH客户端的用法。","2025-06-17 02:44:57",[21399],"工具",{"path":21401,"title":21402,"description":21403,"date":21404,"categories":21273,"tags":21405,"draft":21218},"\u002Fposts\u002Fopensourcesoftwarestarttutorial","如何开始一个开源项目？","浅谈我对开源项目的认知。","2025-04-17 18:00:00",[21406],"开源",{"path":21408,"title":21409,"description":21410,"date":21411,"categories":21412,"tags":21413,"draft":21218},"\u002Fposts\u002Fmotivation","渴求动力","有意识的生活并主动地行动会使得生活更美好。","2025-03-05 00:44:57","生活-情感",[21297],{"path":21415,"title":21416,"description":21417,"date":21418,"categories":21412,"tags":21419,"draft":21218},"\u002Fposts\u002F2024summary","2024年终总结","啊，痛彻的时光；啊，悲怆的人！","2025-01-01 01:00:00",[21297],{"path":21421,"title":21422,"description":21423,"date":21424,"categories":21215,"tags":21425,"draft":21218},"\u002Fposts\u002Fvps-crash-startup","快速搭建个人VPS","如何快速搭建个人VPS。","2024-12-31 19:00:00",[21426],"VPS",{"path":21428,"title":21429,"description":21430,"date":21431,"categories":21215,"tags":21432,"draft":21218},"\u002Fposts\u002Fdata-label-software","AI数据（自动）标注软件概述","对市面上目前的AI数据（自动）标注软件进行调研。","2024-12-20 14:00:00",[21433,21434],"数据标注","深度学习",{"path":21436,"title":21437,"description":21438,"date":21439,"categories":21412,"tags":21440,"draft":21218},"\u002Fposts\u002Fself-destruction","生活的崩坏与自我毁灭","是什么在牵引着我？我的行为，我的思想？我的感受？我所处的地方？它们在何处？","2024-12-19 12:00:00",[21297],{"path":21442,"title":21443,"description":21444,"date":21445,"categories":21215,"tags":21446,"draft":21218},"\u002Fposts\u002Fcs194-196_llmagents","LLM Agents 课程笔记","UCB CS294\u002F194-196 Large Language Model Agents 的课程笔记。","2024-11-30 20:00:00",[21363],{"path":21448,"title":21449,"description":21450,"date":21451,"categories":21412,"tags":21452,"draft":21218},"\u002Fposts\u002Fwhatsyourworth","价值、位置","你们的生存究竟有何价值？如果毫无价值，你们究竟为何存在？","2024-11-29 02:00:00",[21297],{"path":21454,"title":21455,"description":21456,"date":21457,"categories":21412,"tags":21458,"draft":21218},"\u002Fposts\u002Fblogrelifever2","重启博客v2","忙里偷闲学习前端，还是决定结合想法手搓一个自己的博客。","2024-11-25 00:00:00",[21327],{"path":21460,"title":21461,"description":21462,"date":21463,"categories":21412,"tags":21245,"draft":21218},"\u002Fposts\u002Fblogrelife","重启博客","在我的表达欲与现实所需(问题解决记录)驱动下，我决定重写博客。","2024-10-21 20:44:57",{"path":21465,"title":21466,"description":21467,"date":21468,"categories":21412,"tags":21469,"draft":21218},"\u002Fposts\u002Frevival","240205 复活","好吧，断更了许久，我又终于回来了，我又终于回来了。","2024-02-05 20:00:00",[21297,21470,21471],"生活","返乡",{"path":21473,"title":21474,"description":21475,"date":21476,"categories":21412,"tags":21477,"draft":21218},"\u002Fposts\u002Fclouds","「230717」云","我已经很久忘记了不我已经很少把自己的想法在lofter上发布的。","2023-07-17 20:00:00",[21297,21478],"散文",1791871062124]