[{"data":1,"prerenderedAt":15939},["ShallowReactive",2],{"content-\u002Fposts\u002Fspinning-up-rl-intro":3,"blog-posts":15688},{"id":4,"title":5,"body":6,"categories":15672,"date":15673,"description":15674,"draft":15675,"extension":15676,"meta":15677,"navigation":15678,"path":15679,"seo":15680,"stem":15681,"tags":15682,"__hash__":15687},"posts\u002Fposts\u002Fspinning-up-rl-intro.md","[译] 深度强化学习入门（一）：RL 的关键概念",{"type":7,"value":8,"toc":15653},"minimark",[9,31,34,52,55,60,75,101,104,113,121,132,135,158,162,248,251,262,335,338,353,356,359,397,580,614,779,782,789,857,1354,1358,1369,1447,1493,1527,1532,1535,1541,1555,1558,1681,1684,2120,4154,4157,4192,4479,4628,4779,5096,5318,5321,5551,5554,5563,5566,5600,5876,6230,6278,6285,6481,6570,6803,6816,6825,6829,6836,6839,6871,7440,7489,7787,7790,7966,8033,8035,8046,8049,10041,10050,10528,10532,10767,10963,11189,11279,11282,11289,11295,11298,12388,13018,13021,14025,14090,14095,14098,14109,14375,14621,14626,14630,14774,15640,15649],[10,11,13],"note",{"title":12},"译者说明",[14,15,16,17,24,25,30],"p",{},"本文是 OpenAI ",[18,19,23],"a",{"href":20,"rel":21},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002F",[22],"nofollow","Spinning Up in Deep RL"," 教程第一部分 ",[18,26,29],{"href":27,"rel":28},"https:\u002F\u002Fspinningup.openai.com\u002Fen\u002Flatest\u002Fspinningup\u002Frl_intro.html",[22],"Part 1: Key Concepts in RL"," 的全文翻译，原仓库以 MIT 协议发布。原文的提示框、外链与代码示例全部保留，视频以链接代替嵌入。术语首次出现时附英文原词，之后沿用同一译名。",[14,32,33],{},"欢迎来到强化学习导论！这一部分的目标是让你熟悉：",[35,36,37,41,49],"ul",{},[38,39,40],"li",{},"讨论这一主题所用的语言与记号；",[38,42,43,44,48],{},"RL 算法在做什么的高层解释（不过我们大多回避它们",[45,46,47],"em",{},"如何","做到这一问题）；",[38,50,51],{},"以及支撑这些算法的一点点核心数学。",[14,53,54],{},"概括地说，RL 研究的是智能体（agent）以及它们如何通过试错（trial and error）学习。它把这个想法形式化：对智能体的行为给予奖励或惩罚，会让它未来更倾向于重复或放弃该行为。",[56,57,59],"h2",{"id":58},"rl-能做什么","RL 能做什么",[14,61,62,63,68,69,74],{},"近年来，RL 方法收获了大范围的成功。例如，它被用来教计算机在模拟中控制机器人：",[18,64,67],{"href":65,"rel":66},"https:\u002F\u002Fd4mucfpksywv.cloudfront.net\u002Fopenai-baselines-ppo\u002Fknocked-over-stand-up.mp4",[22],"模拟环境里，机器人被撞倒后重新站起来（视频）","；也在真实世界中这样做：",[18,70,73],{"href":71,"rel":72},"https:\u002F\u002Fwww.youtube.com\u002Fwatch?v=jwSbzNHGflM",[22],"真实机器人控制（YouTube）","。",[14,76,77,78,83,84,89,90,95,96,74],{},"它还因在复杂策略游戏中造出突破性 AI 而闻名，最有名的是 ",[18,79,82],{"href":80,"rel":81},"https:\u002F\u002Fdeepmind.com\u002Fresearch\u002Falphago\u002F",[22],"Go（围棋）","和 ",[18,85,88],{"href":86,"rel":87},"https:\u002F\u002Fblog.openai.com\u002Fopenai-five\u002F",[22],"Dota","；它教会计算机从原始像素",[18,91,94],{"href":92,"rel":93},"https:\u002F\u002Fdeepmind.com\u002Fresearch\u002Fdqn\u002F",[22],"玩 Atari 游戏","，也训练过模拟机器人",[18,97,100],{"href":98,"rel":99},"https:\u002F\u002Fblog.openai.com\u002Fdeep-reinforcement-learning-from-human-preferences\u002F",[22],"跟随人类指令",[56,102,103],{"id":103},"关键概念与术语",[14,105,106],{},[107,108],"img",{"alt":109,"className":110,"src":112},"智能体与环境的交互循环",[111],"dark-ink","\u002Fimages\u002Fspinning-up-rl-loop.png",[14,114,115,116,120],{},"RL 的两个主角是",[117,118,119],"strong",{},"智能体（agent）和环境（environment）","。环境是智能体身处其中并与之交互的世界。交互的每一步，智能体都会看到世界状态的一个（可能不完整的）观测，然后决定要采取的动作。智能体作用于环境时，环境会随之改变，但它也可能自行变化。",[14,122,123,124,127,128,131],{},"智能体还会从环境感知到一个",[117,125,126],{},"奖励（reward）"," 信号：一个数字，告诉它当前的世界状态有多好或多坏。智能体的目标是最大化自己的累积奖励，这个累积量称为",[117,129,130],{},"回报（return）","。强化学习方法，就是智能体学习行为以达成这一目标的各种途径。",[14,133,134],{},"要把 RL 做的事说得更具体，还需要引入一批术语。我们要讨论：",[35,136,137,140,143,146,149,152,155],{},[38,138,139],{},"状态与观测；",[38,141,142],{},"动作空间；",[38,144,145],{},"策略；",[38,147,148],{},"轨迹；",[38,150,151],{},"回报的不同形式；",[38,153,154],{},"RL 优化问题；",[38,156,157],{},"以及价值函数。",[159,160,161],"h3",{"id":161},"状态与观测",[14,163,164,167,168,214,215,167,218,247],{},[117,165,166],{},"状态（state）"," ",[169,170,173,195],"span",{"className":171},[172],"katex",[169,174,177],{"className":175},[176],"katex-mathml",[178,179,181],"math",{"xmlns":180},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[182,183,184,191],"semantics",{},[185,186,187],"mrow",{},[188,189,190],"mi",{},"s",[192,193,190],"annotation",{"encoding":194},"application\u002Fx-tex",[169,196,200],{"className":197,"ariaHidden":199},[198],"katex-html","true",[169,201,204,209],{"className":202},[203],"base",[169,205],{"className":206,"style":208},[207],"strut","height:0.4306em;",[169,210,190],{"className":211},[212,213],"mord","mathnormal"," 是对世界的完整描述，没有任何关于世界的信息被它遗漏。",[117,216,217],{},"观测（observation）",[169,219,221,235],{"className":220},[172],[169,222,224],{"className":223},[176],[178,225,226],{"xmlns":180},[182,227,228,233],{},[185,229,230],{},[188,231,232],{},"o",[192,234,232],{"encoding":194},[169,236,238],{"className":237,"ariaHidden":199},[198],[169,239,241,244],{"className":240},[203],[169,242],{"className":243,"style":208},[207],[169,245,232],{"className":246},[212,213]," 则是对状态的部分描述，可能省略一些信息。",[14,249,250],{},"在深度 RL 中，我们几乎总是用实值向量、矩阵或更高阶张量来表示状态和观测。例如，一个视觉观测可以用其像素值的 RGB（红绿蓝）矩阵表示；机器人的状态可以用它的关节角度和速度表示。",[14,252,253,254,257,258,261],{},"当智能体能够观测到环境的完整状态时，我们说环境是",[117,255,256],{},"完全可观测（fully observed）"," 的；当它只能看到部分观测时，我们说环境是",[117,259,260],{},"部分可观测（partially observed）"," 的。",[10,263,265,326],{"title":264},"你应该知道",[14,266,267,268,296,297,325],{},"RL 的记号有时会在严格来说更适合写观测符号 ",[169,269,271,284],{"className":270},[172],[169,272,274],{"className":273},[176],[178,275,276],{"xmlns":180},[182,277,278,282],{},[185,279,280],{},[188,281,232],{},[192,283,232],{"encoding":194},[169,285,287],{"className":286,"ariaHidden":199},[198],[169,288,290,293],{"className":289},[203],[169,291],{"className":292,"style":208},[207],[169,294,232],{"className":295},[212,213]," 的地方写状态符号 ",[169,298,300,313],{"className":299},[172],[169,301,303],{"className":302},[176],[178,304,305],{"xmlns":180},[182,306,307,311],{},[185,308,309],{},[188,310,190],{},[192,312,190],{"encoding":194},[169,314,316],{"className":315,"ariaHidden":199},[198],[169,317,319,322],{"className":318},[203],[169,320],{"className":321,"style":208},[207],[169,323,190],{"className":324},[212,213],"。描述智能体如何决定动作时就常见这种情况：记号上通常写作“动作以状态为条件”，但实践中动作取决于观测，因为智能体拿不到状态。",[14,327,328,329,334],{},"本指南遵循标准记号约定，具体指哪个应当能从上下文分辨出来。如果有不清楚的地方，欢迎到",[18,330,333],{"href":331,"rel":332},"https:\u002F\u002Fgithub.com\u002Fopenai\u002Fspinningup\u002Fissues",[22],"原仓库","提 issue！我们的目标是把人教会，而不是把人绕晕。",[159,336,337],{"id":337},"动作空间",[14,339,340,341,344,345,348,349,352],{},"不同的环境允许不同种类的动作。一个给定环境中所有合法动作的集合，通常叫作",[117,342,343],{},"动作空间（action space）","。有些环境（比如 Atari 和围棋）具有",[117,346,347],{},"离散动作空间（discrete action space）","：智能体可用的走法只有有限多个。另一些环境（比如智能体在物理世界中控制机器人）则具有",[117,350,351],{},"连续动作空间（continuous action space）","：动作是实值向量。",[14,354,355],{},"这个区分对深度 RL 方法的影响相当深远：有些算法家族只能直接用于其中一种情形，要用到另一种情形上就得动大手术。",[159,357,358],{"id":358},"策略",[14,360,361,364,365,396],{},[117,362,363],{},"策略（policy）"," 是智能体用来决定采取什么动作的规则。它可以是确定型（deterministic）的，通常记作 ",[169,366,368,383],{"className":367},[172],[169,369,371],{"className":370},[176],[178,372,373],{"xmlns":180},[182,374,375,380],{},[185,376,377],{},[188,378,379],{},"μ",[192,381,382],{"encoding":194},"\\mu",[169,384,386],{"className":385,"ariaHidden":199},[198],[169,387,389,393],{"className":388},[203],[169,390],{"className":391,"style":392},[207],"height:0.625em;vertical-align:-0.1944em;",[169,394,379],{"className":395},[212,213],"：",[169,398,401],{"className":399},[400],"katex-display",[169,402,404,444],{"className":403},[172],[169,405,407],{"className":406},[176],[178,408,410],{"xmlns":180,"display":409},"block",[182,411,412,441],{},[185,413,414,422,426,428,432,438],{},[415,416,417,419],"msub",{},[188,418,18],{},[188,420,421],{},"t",[423,424,425],"mo",{},"=",[188,427,379],{},[423,429,431],{"stretchy":430},"false","(",[415,433,434,436],{},[188,435,190],{},[188,437,421],{},[423,439,440],{"stretchy":430},")",[192,442,443],{"encoding":194},"a_t = \\mu(s_t)",[169,445,447,522],{"className":446,"ariaHidden":199},[198],[169,448,450,454,510,515,519],{"className":449},[203],[169,451],{"className":452,"style":453},[207],"height:0.5806em;vertical-align:-0.15em;",[169,455,457,460],{"className":456},[212],[169,458,18],{"className":459},[212,213],[169,461,464],{"className":462},[463],"msupsub",[169,465,469,501],{"className":466},[467,468],"vlist-t","vlist-t2",[169,470,473,496],{"className":471},[472],"vlist-r",[169,474,478],{"className":475,"style":477},[476],"vlist","height:0.2806em;",[169,479,481,486],{"style":480},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[169,482],{"className":483,"style":485},[484],"pstrut","height:2.7em;",[169,487,493],{"className":488},[489,490,491,492],"sizing","reset-size6","size3","mtight",[169,494,421],{"className":495},[212,213,492],[169,497,500],{"className":498},[499],"vlist-s","​",[169,502,504],{"className":503},[472],[169,505,508],{"className":506,"style":507},[476],"height:0.15em;",[169,509],{},[169,511],{"className":512,"style":514},[513],"mspace","margin-right:0.2778em;",[169,516,425],{"className":517},[518],"mrel",[169,520],{"className":521,"style":514},[513],[169,523,525,529,532,536,576],{"className":524},[203],[169,526],{"className":527,"style":528},[207],"height:1em;vertical-align:-0.25em;",[169,530,379],{"className":531},[212,213],[169,533,431],{"className":534},[535],"mopen",[169,537,539,542],{"className":538},[212],[169,540,190],{"className":541},[212,213],[169,543,545],{"className":544},[463],[169,546,548,568],{"className":547},[467,468],[169,549,551,565],{"className":550},[472],[169,552,554],{"className":553,"style":477},[476],[169,555,556,559],{"style":480},[169,557],{"className":558,"style":485},[484],[169,560,562],{"className":561},[489,490,491,492],[169,563,421],{"className":564},[212,213,492],[169,566,500],{"className":567},[499],[169,569,571],{"className":570},[472],[169,572,574],{"className":573,"style":507},[476],[169,575],{},[169,577,440],{"className":578},[579],"mclose",[14,581,582,583,396],{},"也可以是随机型（stochastic）的，通常记作 ",[169,584,586,601],{"className":585},[172],[169,587,589],{"className":588},[176],[178,590,591],{"xmlns":180},[182,592,593,598],{},[185,594,595],{},[188,596,597],{},"π",[192,599,600],{"encoding":194},"\\pi",[169,602,604],{"className":603,"ariaHidden":199},[198],[169,605,607,610],{"className":606},[203],[169,608],{"className":609,"style":208},[207],[169,611,597],{"className":612,"style":613},[212,213],"margin-right:0.0359em;",[169,615,617],{"className":616},[400],[169,618,620,660],{"className":619},[172],[169,621,623],{"className":622},[176],[178,624,625],{"xmlns":180,"display":409},[182,626,627,657],{},[185,628,629,635,638,640,642,645,649,655],{},[415,630,631,633],{},[188,632,18],{},[188,634,421],{},[423,636,637],{},"∼",[188,639,597],{},[423,641,431],{"stretchy":430},[423,643,644],{},"⋅",[188,646,648],{"mathvariant":647},"normal","∣",[415,650,651,653],{},[188,652,190],{},[188,654,421],{},[423,656,440],{"stretchy":430},[192,658,659],{"encoding":194},"a_t \\sim \\pi(\\cdot | s_t)",[169,661,663,718],{"className":662,"ariaHidden":199},[198],[169,664,666,669,709,712,715],{"className":665},[203],[169,667],{"className":668,"style":453},[207],[169,670,672,675],{"className":671},[212],[169,673,18],{"className":674},[212,213],[169,676,678],{"className":677},[463],[169,679,681,701],{"className":680},[467,468],[169,682,684,698],{"className":683},[472],[169,685,687],{"className":686,"style":477},[476],[169,688,689,692],{"style":480},[169,690],{"className":691,"style":485},[484],[169,693,695],{"className":694},[489,490,491,492],[169,696,421],{"className":697},[212,213,492],[169,699,500],{"className":700},[499],[169,702,704],{"className":703},[472],[169,705,707],{"className":706,"style":507},[476],[169,708],{},[169,710],{"className":711,"style":514},[513],[169,713,637],{"className":714},[518],[169,716],{"className":717,"style":514},[513],[169,719,721,724,727,730,733,736,776],{"className":720},[203],[169,722],{"className":723,"style":528},[207],[169,725,597],{"className":726,"style":613},[212,213],[169,728,431],{"className":729},[535],[169,731,644],{"className":732},[212],[169,734,648],{"className":735},[212],[169,737,739,742],{"className":738},[212],[169,740,190],{"className":741},[212,213],[169,743,745],{"className":744},[463],[169,746,748,768],{"className":747},[467,468],[169,749,751,765],{"className":750},[472],[169,752,754],{"className":753,"style":477},[476],[169,755,756,759],{"style":480},[169,757],{"className":758,"style":485},[484],[169,760,762],{"className":761},[489,490,491,492],[169,763,421],{"className":764},[212,213,492],[169,766,500],{"className":767},[499],[169,769,771],{"className":770},[472],[169,772,774],{"className":773,"style":507},[476],[169,775],{},[169,777,440],{"className":778},[579],[14,780,781],{},"策略是智能体的大脑，所以拿「policy」一词替换「agent」来说话并不少见，比如『策略正在试图最大化奖励』。",[14,783,784,785,788],{},"在深度 RL 中，我们处理的是",[117,786,787],{},"参数化策略（parameterized policy）","：这类策略的输出是可计算函数，函数依赖一组参数（比如神经网络的权重和偏置）；通过某种优化算法调整这些参数，就能改变策略的行为。",[14,790,791,792,824,825,856],{},"这类策略的参数常记作 ",[169,793,795,810],{"className":794},[172],[169,796,798],{"className":797},[176],[178,799,800],{"xmlns":180},[182,801,802,807],{},[185,803,804],{},[188,805,806],{},"θ",[192,808,809],{"encoding":194},"\\theta",[169,811,813],{"className":812,"ariaHidden":199},[198],[169,814,816,820],{"className":815},[203],[169,817],{"className":818,"style":819},[207],"height:0.6944em;",[169,821,806],{"className":822,"style":823},[212,213],"margin-right:0.0278em;"," 或 ",[169,826,828,843],{"className":827},[172],[169,829,831],{"className":830},[176],[178,832,833],{"xmlns":180},[182,834,835,840],{},[185,836,837],{},[188,838,839],{},"ϕ",[192,841,842],{"encoding":194},"\\phi",[169,844,846],{"className":845,"ariaHidden":199},[198],[169,847,849,853],{"className":848},[203],[169,850],{"className":851,"style":852},[207],"height:0.8889em;vertical-align:-0.1944em;",[169,854,839],{"className":855},[212,213],"，并写成策略符号的下标来点明这层关系：",[169,858,860],{"className":859},[400],[169,861,863,963],{"className":862},[172],[169,864,866],{"className":865},[176],[178,867,868],{"xmlns":180,"display":409},[182,869,870,960],{},[871,872,876,918],"mtable",{"rowspacing":873,"columnalign":874,"columnspacing":875},"0.25em","right left","0em",[877,878,879,892],"mtr",{},[880,881,882],"mtd",{},[883,884,886],"mstyle",{"scriptlevel":885,"displaystyle":199},"0",[415,887,888,890],{},[188,889,18],{},[188,891,421],{},[880,893,894],{},[883,895,896],{"scriptlevel":885,"displaystyle":199},[185,897,898,900,902,908,910,916],{},[185,899],{},[423,901,425],{},[415,903,904,906],{},[188,905,379],{},[188,907,806],{},[423,909,431],{"stretchy":430},[415,911,912,914],{},[188,913,190],{},[188,915,421],{},[423,917,440],{"stretchy":430},[877,919,920,930],{},[880,921,922],{},[883,923,924],{"scriptlevel":885,"displaystyle":199},[415,925,926,928],{},[188,927,18],{},[188,929,421],{},[880,931,932],{},[883,933,934],{"scriptlevel":885,"displaystyle":199},[185,935,936,938,940,946,948,950,952,958],{},[185,937],{},[423,939,637],{},[415,941,942,944],{},[188,943,597],{},[188,945,806],{},[423,947,431],{"stretchy":430},[423,949,644],{},[188,951,648],{"mathvariant":647},[415,953,954,956],{},[188,955,190],{},[188,957,421],{},[423,959,440],{"stretchy":430},[192,961,962],{"encoding":194},"\\begin{aligned}\na_t &= \\mu_{\\theta}(s_t) \\\\\na_t &\\sim \\pi_{\\theta}(\\cdot | s_t)\n\\end{aligned}",[169,964,966],{"className":965,"ariaHidden":199},[198],[169,967,969,973],{"className":968},[203],[169,970],{"className":971,"style":972},[207],"height:2.7em;vertical-align:-1.1em;",[169,974,976],{"className":975},[212],[169,977,979,1104],{"className":978},[871],[169,980,983],{"className":981},[982],"col-align-r",[169,984,986,1095],{"className":985},[467,468],[169,987,989,1092],{"className":988},[472],[169,990,993,1043],{"className":991,"style":992},[476],"height:1.6em;",[169,994,996,1000],{"style":995},"top:-3.76em;",[169,997],{"className":998,"style":999},[484],"height:3em;",[169,1001,1003],{"className":1002},[212],[169,1004,1006,1009],{"className":1005},[212],[169,1007,18],{"className":1008},[212,213],[169,1010,1012],{"className":1011},[463],[169,1013,1015,1035],{"className":1014},[467,468],[169,1016,1018,1032],{"className":1017},[472],[169,1019,1021],{"className":1020,"style":477},[476],[169,1022,1023,1026],{"style":480},[169,1024],{"className":1025,"style":485},[484],[169,1027,1029],{"className":1028},[489,490,491,492],[169,1030,421],{"className":1031},[212,213,492],[169,1033,500],{"className":1034},[499],[169,1036,1038],{"className":1037},[472],[169,1039,1041],{"className":1040,"style":507},[476],[169,1042],{},[169,1044,1046,1049],{"style":1045},"top:-2.26em;",[169,1047],{"className":1048,"style":999},[484],[169,1050,1052],{"className":1051},[212],[169,1053,1055,1058],{"className":1054},[212],[169,1056,18],{"className":1057},[212,213],[169,1059,1061],{"className":1060},[463],[169,1062,1064,1084],{"className":1063},[467,468],[169,1065,1067,1081],{"className":1066},[472],[169,1068,1070],{"className":1069,"style":477},[476],[169,1071,1072,1075],{"style":480},[169,1073],{"className":1074,"style":485},[484],[169,1076,1078],{"className":1077},[489,490,491,492],[169,1079,421],{"className":1080},[212,213,492],[169,1082,500],{"className":1083},[499],[169,1085,1087],{"className":1086},[472],[169,1088,1090],{"className":1089,"style":507},[476],[169,1091],{},[169,1093,500],{"className":1094},[499],[169,1096,1098],{"className":1097},[472],[169,1099,1102],{"className":1100,"style":1101},[476],"height:1.1em;",[169,1103],{},[169,1105,1108],{"className":1106},[1107],"col-align-l",[169,1109,1111,1346],{"className":1110},[467,468],[169,1112,1114,1343],{"className":1113},[472],[169,1115,1117,1227],{"className":1116,"style":992},[476],[169,1118,1119,1122],{"style":995},[169,1120],{"className":1121,"style":999},[484],[169,1123,1125,1128,1131,1134,1137,1181,1184,1224],{"className":1124},[212],[169,1126],{"className":1127},[212],[169,1129],{"className":1130,"style":514},[513],[169,1132,425],{"className":1133},[518],[169,1135],{"className":1136,"style":514},[513],[169,1138,1140,1143],{"className":1139},[212],[169,1141,379],{"className":1142},[212,213],[169,1144,1146],{"className":1145},[463],[169,1147,1149,1173],{"className":1148},[467,468],[169,1150,1152,1170],{"className":1151},[472],[169,1153,1156],{"className":1154,"style":1155},[476],"height:0.3361em;",[169,1157,1158,1161],{"style":480},[169,1159],{"className":1160,"style":485},[484],[169,1162,1164],{"className":1163},[489,490,491,492],[169,1165,1167],{"className":1166},[212,492],[169,1168,806],{"className":1169,"style":823},[212,213,492],[169,1171,500],{"className":1172},[499],[169,1174,1176],{"className":1175},[472],[169,1177,1179],{"className":1178,"style":507},[476],[169,1180],{},[169,1182,431],{"className":1183},[535],[169,1185,1187,1190],{"className":1186},[212],[169,1188,190],{"className":1189},[212,213],[169,1191,1193],{"className":1192},[463],[169,1194,1196,1216],{"className":1195},[467,468],[169,1197,1199,1213],{"className":1198},[472],[169,1200,1202],{"className":1201,"style":477},[476],[169,1203,1204,1207],{"style":480},[169,1205],{"className":1206,"style":485},[484],[169,1208,1210],{"className":1209},[489,490,491,492],[169,1211,421],{"className":1212},[212,213,492],[169,1214,500],{"className":1215},[499],[169,1217,1219],{"className":1218},[472],[169,1220,1222],{"className":1221,"style":507},[476],[169,1223],{},[169,1225,440],{"className":1226},[579],[169,1228,1229,1232],{"style":1045},[169,1230],{"className":1231,"style":999},[484],[169,1233,1235,1238,1241,1244,1247,1291,1294,1297,1300,1340],{"className":1234},[212],[169,1236],{"className":1237},[212],[169,1239],{"className":1240,"style":514},[513],[169,1242,637],{"className":1243},[518],[169,1245],{"className":1246,"style":514},[513],[169,1248,1250,1253],{"className":1249},[212],[169,1251,597],{"className":1252,"style":613},[212,213],[169,1254,1256],{"className":1255},[463],[169,1257,1259,1283],{"className":1258},[467,468],[169,1260,1262,1280],{"className":1261},[472],[169,1263,1265],{"className":1264,"style":1155},[476],[169,1266,1268,1271],{"style":1267},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[169,1269],{"className":1270,"style":485},[484],[169,1272,1274],{"className":1273},[489,490,491,492],[169,1275,1277],{"className":1276},[212,492],[169,1278,806],{"className":1279,"style":823},[212,213,492],[169,1281,500],{"className":1282},[499],[169,1284,1286],{"className":1285},[472],[169,1287,1289],{"className":1288,"style":507},[476],[169,1290],{},[169,1292,431],{"className":1293},[535],[169,1295,644],{"className":1296},[212],[169,1298,648],{"className":1299},[212],[169,1301,1303,1306],{"className":1302},[212],[169,1304,190],{"className":1305},[212,213],[169,1307,1309],{"className":1308},[463],[169,1310,1312,1332],{"className":1311},[467,468],[169,1313,1315,1329],{"className":1314},[472],[169,1316,1318],{"className":1317,"style":477},[476],[169,1319,1320,1323],{"style":480},[169,1321],{"className":1322,"style":485},[484],[169,1324,1326],{"className":1325},[489,490,491,492],[169,1327,421],{"className":1328},[212,213,492],[169,1330,500],{"className":1331},[499],[169,1333,1335],{"className":1334},[472],[169,1336,1338],{"className":1337,"style":507},[476],[169,1339],{},[169,1341,440],{"className":1342},[579],[169,1344,500],{"className":1345},[499],[169,1347,1349],{"className":1348},[472],[169,1350,1352],{"className":1351,"style":1101},[476],[169,1353],{},[1355,1356,1357],"h4",{"id":1357},"确定型策略",[14,1359,1360,1363,1364,1368],{},[117,1361,1362],{},"例子：确定型策略","。下面这段代码用 PyTorch 的 ",[1365,1366,1367],"code",{},"torch.nn"," 包，为连续动作空间构建一个简单的确定型策略：",[1370,1371,1376],"pre",{"className":1372,"code":1373,"language":1374,"meta":1375,"style":1375},"language-python3 shiki shiki-themes github-light github-dark","pi_net = nn.Sequential(\n              nn.Linear(obs_dim, 64),\n              nn.Tanh(),\n              nn.Linear(64, 64),\n              nn.Tanh(),\n              nn.Linear(64, act_dim)\n            )\n","python3","",[1365,1377,1378,1392,1405,1411,1426,1431,1441],{"__ignoreMap":1375},[169,1379,1382,1386,1389],{"class":1380,"line":1381},"line",1,[169,1383,1385],{"class":1384},"sVt8B","pi_net ",[169,1387,425],{"class":1388},"szBVR",[169,1390,1391],{"class":1384}," nn.Sequential(\n",[169,1393,1395,1398,1402],{"class":1380,"line":1394},2,[169,1396,1397],{"class":1384},"              nn.Linear(obs_dim, ",[169,1399,1401],{"class":1400},"sj4cs","64",[169,1403,1404],{"class":1384},"),\n",[169,1406,1408],{"class":1380,"line":1407},3,[169,1409,1410],{"class":1384},"              nn.Tanh(),\n",[169,1412,1414,1417,1419,1422,1424],{"class":1380,"line":1413},4,[169,1415,1416],{"class":1384},"              nn.Linear(",[169,1418,1401],{"class":1400},[169,1420,1421],{"class":1384},", ",[169,1423,1401],{"class":1400},[169,1425,1404],{"class":1384},[169,1427,1429],{"class":1380,"line":1428},5,[169,1430,1410],{"class":1384},[169,1432,1434,1436,1438],{"class":1380,"line":1433},6,[169,1435,1416],{"class":1384},[169,1437,1401],{"class":1400},[169,1439,1440],{"class":1384},", act_dim)\n",[169,1442,1444],{"class":1380,"line":1443},7,[169,1445,1446],{"class":1384},"            )\n",[14,1448,1449,1450,1484,1485,1488,1489,1492],{},"它构建了一个 MLP（multi-layer perceptron，多层感知机）网络：两个隐藏层各 64 个单元，激活函数是 ",[169,1451,1453,1471],{"className":1452},[172],[169,1454,1456],{"className":1455},[176],[178,1457,1458],{"xmlns":180},[182,1459,1460,1468],{},[185,1461,1462,1465],{},[188,1463,1464],{},"tanh",[423,1466,1467],{},"⁡",[192,1469,1470],{"encoding":194},"\\tanh",[169,1472,1474],{"className":1473,"ariaHidden":199},[198],[169,1475,1477,1480],{"className":1476},[203],[169,1478],{"className":1479,"style":819},[207],[169,1481,1464],{"className":1482},[1483],"mop","。如果 ",[1365,1486,1487],{},"obs"," 是装着一批观测的 Numpy 数组，可以这样用 ",[1365,1490,1491],{},"pi_net"," 求出这批观测对应的动作：",[1370,1494,1496],{"className":1372,"code":1495,"language":1374,"meta":1375,"style":1375},"obs_tensor = torch.as_tensor(obs, dtype=torch.float32)\nactions = pi_net(obs_tensor)\n",[1365,1497,1498,1517],{"__ignoreMap":1375},[169,1499,1500,1503,1505,1508,1512,1514],{"class":1380,"line":1381},[169,1501,1502],{"class":1384},"obs_tensor ",[169,1504,425],{"class":1388},[169,1506,1507],{"class":1384}," torch.as_tensor(obs, ",[169,1509,1511],{"class":1510},"s4XuR","dtype",[169,1513,425],{"class":1388},[169,1515,1516],{"class":1384},"torch.float32)\n",[169,1518,1519,1522,1524],{"class":1380,"line":1394},[169,1520,1521],{"class":1384},"actions ",[169,1523,425],{"class":1388},[169,1525,1526],{"class":1384}," pi_net(obs_tensor)\n",[10,1528,1529],{"title":264},[14,1530,1531],{},"神经网络的内容如果不熟悉，不必担心：本教程聚焦 RL，不展开神经网络那一侧。可以先跳过这个例子，之后再回来看。不过我们猜想，如果你已经了解，它会有所帮助。",[1355,1533,1534],{"id":1534},"随机型策略",[14,1536,1537,1538,74],{},"深度 RL 中最常见的两种随机型策略是",[117,1539,1540],{},"分类策略（categorical policy）和对角高斯策略（diagonal Gaussian policy）",[14,1542,1543,1548,1549,1554],{},[18,1544,1547],{"href":1545,"rel":1546},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FCategorical_distribution",[22],"分类","策略可用于离散动作空间，对角",[18,1550,1553],{"href":1551,"rel":1552},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FMultivariate_normal_distribution",[22],"高斯","策略则用于连续动作空间。",[14,1556,1557],{},"使用和训练随机型策略时，有两项关键计算居于中心地位：",[35,1559,1560,1563],{},[38,1561,1562],{},"从策略中采样动作；",[38,1564,1565,1566,74],{},"计算特定动作的对数似然（log likelihood）",[169,1567,1569,1602],{"className":1568},[172],[169,1570,1572],{"className":1571},[176],[178,1573,1574],{"xmlns":180},[182,1575,1576,1599],{},[185,1577,1578,1581,1583,1589,1591,1593,1595,1597],{},[188,1579,1580],{},"log",[423,1582,1467],{},[415,1584,1585,1587],{},[188,1586,597],{},[188,1588,806],{},[423,1590,431],{"stretchy":430},[188,1592,18],{},[188,1594,648],{"mathvariant":647},[188,1596,190],{},[423,1598,440],{"stretchy":430},[192,1600,1601],{"encoding":194},"\\log \\pi_{\\theta}(a|s)",[169,1603,1605],{"className":1604,"ariaHidden":199},[198],[169,1606,1608,1611,1619,1623,1666,1669,1672,1675,1678],{"className":1607},[203],[169,1609],{"className":1610,"style":528},[207],[169,1612,1614,1615],{"className":1613},[1483],"lo",[169,1616,1618],{"style":1617},"margin-right:0.0139em;","g",[169,1620],{"className":1621,"style":1622},[513],"margin-right:0.1667em;",[169,1624,1626,1629],{"className":1625},[212],[169,1627,597],{"className":1628,"style":613},[212,213],[169,1630,1632],{"className":1631},[463],[169,1633,1635,1658],{"className":1634},[467,468],[169,1636,1638,1655],{"className":1637},[472],[169,1639,1641],{"className":1640,"style":1155},[476],[169,1642,1643,1646],{"style":1267},[169,1644],{"className":1645,"style":485},[484],[169,1647,1649],{"className":1648},[489,490,491,492],[169,1650,1652],{"className":1651},[212,492],[169,1653,806],{"className":1654,"style":823},[212,213,492],[169,1656,500],{"className":1657},[499],[169,1659,1661],{"className":1660},[472],[169,1662,1664],{"className":1663,"style":507},[476],[169,1665],{},[169,1667,431],{"className":1668},[535],[169,1670,18],{"className":1671},[212,213],[169,1673,648],{"className":1674},[212],[169,1676,190],{"className":1677},[212,213],[169,1679,440],{"className":1680},[579],[14,1682,1683],{},"接下来分别说明这两件事在分类策略和对角高斯策略上怎么做。",[10,1685,1687,1696,1722,1849],{"title":1686},"分类策略",[14,1688,1689,1690,1695],{},"分类策略就像作用在离散动作上的分类器。构建分类策略的神经网络和构建分类器是同一套做法：输入观测，接若干层（视输入类型可以是卷积层或全连接层），最后一个线性层给出每个动作的 logits，再接一个 ",[18,1691,1694],{"href":1692,"rel":1693},"https:\u002F\u002Fdevelopers.google.com\u002Fmachine-learning\u002Fcrash-course\u002Fmulti-class-neural-networks\u002Fsoftmax",[22],"softmax"," 把 logits 转成概率。",[14,1697,1698,1701,1702,1707,1708,1707,1715,824,1718,1721],{},[117,1699,1700],{},"采样","。拿到每个动作的概率后，PyTorch 和 TensorFlow 这类框架都内置了采样工具，例如 ",[18,1703,1706],{"href":1704,"rel":1705},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Fdistributions.html#categorical",[22],"PyTorch 的 Categorical 分布","、",[18,1709,1712],{"href":1710,"rel":1711},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Ftorch.html#torch.multinomial",[22],[1365,1713,1714],{},"torch.multinomial",[1365,1716,1717],{},"tf.distributions.Categorical",[1365,1719,1720],{},"tf.multinomial"," 的文档。",[14,1723,1724,1727,1728,1819,1820,1848],{},[117,1725,1726],{},"对数似然","。记最后一层输出的概率为 ",[169,1729,1731,1756],{"className":1730},[172],[169,1732,1734],{"className":1733},[176],[178,1735,1736],{"xmlns":180},[182,1737,1738,1753],{},[185,1739,1740,1747,1749,1751],{},[415,1741,1742,1745],{},[188,1743,1744],{},"P",[188,1746,806],{},[423,1748,431],{"stretchy":430},[188,1750,190],{},[423,1752,440],{"stretchy":430},[192,1754,1755],{"encoding":194},"P_{\\theta}(s)",[169,1757,1759],{"className":1758,"ariaHidden":199},[198],[169,1760,1762,1765,1810,1813,1816],{"className":1761},[203],[169,1763],{"className":1764,"style":528},[207],[169,1766,1768,1772],{"className":1767},[212],[169,1769,1744],{"className":1770,"style":1771},[212,213],"margin-right:0.1389em;",[169,1773,1775],{"className":1774},[463],[169,1776,1778,1802],{"className":1777},[467,468],[169,1779,1781,1799],{"className":1780},[472],[169,1782,1784],{"className":1783,"style":1155},[476],[169,1785,1787,1790],{"style":1786},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[169,1788],{"className":1789,"style":485},[484],[169,1791,1793],{"className":1792},[489,490,491,492],[169,1794,1796],{"className":1795},[212,492],[169,1797,806],{"className":1798,"style":823},[212,213,492],[169,1800,500],{"className":1801},[499],[169,1803,1805],{"className":1804},[472],[169,1806,1808],{"className":1807,"style":507},[476],[169,1809],{},[169,1811,431],{"className":1812},[535],[169,1814,190],{"className":1815},[212,213],[169,1817,440],{"className":1818},[579],"。它是一个向量，条目数与动作数相同，因此可以把动作当作向量的索引。动作 ",[169,1821,1823,1836],{"className":1822},[172],[169,1824,1826],{"className":1825},[176],[178,1827,1828],{"xmlns":180},[182,1829,1830,1834],{},[185,1831,1832],{},[188,1833,18],{},[192,1835,18],{"encoding":194},[169,1837,1839],{"className":1838,"ariaHidden":199},[198],[169,1840,1842,1845],{"className":1841},[203],[169,1843],{"className":1844,"style":208},[207],[169,1846,18],{"className":1847},[212,213]," 的对数似然通过索引得到：",[169,1850,1852],{"className":1851},[400],[169,1853,1855,1917],{"className":1854},[172],[169,1856,1858],{"className":1857},[176],[178,1859,1860],{"xmlns":180,"display":409},[182,1861,1862,1914],{},[185,1863,1864,1866,1868,1874,1876,1878,1880,1882,1884,1886,1888,1890],{},[188,1865,1580],{},[423,1867,1467],{},[415,1869,1870,1872],{},[188,1871,597],{},[188,1873,806],{},[423,1875,431],{"stretchy":430},[188,1877,18],{},[188,1879,648],{"mathvariant":647},[188,1881,190],{},[423,1883,440],{"stretchy":430},[423,1885,425],{},[188,1887,1580],{},[423,1889,1467],{},[415,1891,1892,1912],{},[185,1893,1894,1897,1903,1905,1907,1909],{},[423,1895,1896],{"fence":199},"[",[415,1898,1899,1901],{},[188,1900,1744],{},[188,1902,806],{},[423,1904,431],{"stretchy":430},[188,1906,190],{},[423,1908,440],{"stretchy":430},[423,1910,1911],{"fence":199},"]",[188,1913,18],{},[192,1915,1916],{"encoding":194},"\\log \\pi_{\\theta}(a|s) = \\log \\left[P_{\\theta}(s)\\right]_a",[169,1918,1920,2001],{"className":1919,"ariaHidden":199},[198],[169,1921,1923,1926,1931,1934,1977,1980,1983,1986,1989,1992,1995,1998],{"className":1922},[203],[169,1924],{"className":1925,"style":528},[207],[169,1927,1614,1929],{"className":1928},[1483],[169,1930,1618],{"style":1617},[169,1932],{"className":1933,"style":1622},[513],[169,1935,1937,1940],{"className":1936},[212],[169,1938,597],{"className":1939,"style":613},[212,213],[169,1941,1943],{"className":1942},[463],[169,1944,1946,1969],{"className":1945},[467,468],[169,1947,1949,1966],{"className":1948},[472],[169,1950,1952],{"className":1951,"style":1155},[476],[169,1953,1954,1957],{"style":1267},[169,1955],{"className":1956,"style":485},[484],[169,1958,1960],{"className":1959},[489,490,491,492],[169,1961,1963],{"className":1962},[212,492],[169,1964,806],{"className":1965,"style":823},[212,213,492],[169,1967,500],{"className":1968},[499],[169,1970,1972],{"className":1971},[472],[169,1973,1975],{"className":1974,"style":507},[476],[169,1976],{},[169,1978,431],{"className":1979},[535],[169,1981,18],{"className":1982},[212,213],[169,1984,648],{"className":1985},[212],[169,1987,190],{"className":1988},[212,213],[169,1990,440],{"className":1991},[579],[169,1993],{"className":1994,"style":514},[513],[169,1996,425],{"className":1997},[518],[169,1999],{"className":2000,"style":514},[513],[169,2002,2004,2008,2013,2016],{"className":2003},[203],[169,2005],{"className":2006,"style":2007},[207],"height:1.0497em;vertical-align:-0.2997em;",[169,2009,1614,2011],{"className":2010},[1483],[169,2012,1618],{"style":1617},[169,2014],{"className":2015,"style":1622},[513],[169,2017,2020,2083],{"className":2018},[2019],"minner",[169,2021,2023,2028,2071,2074,2077,2080],{"className":2022},[2019],[169,2024,1896],{"className":2025,"style":2027},[535,2026],"delimcenter","top:0em;",[169,2029,2031,2034],{"className":2030},[212],[169,2032,1744],{"className":2033,"style":1771},[212,213],[169,2035,2037],{"className":2036},[463],[169,2038,2040,2063],{"className":2039},[467,468],[169,2041,2043,2060],{"className":2042},[472],[169,2044,2046],{"className":2045,"style":1155},[476],[169,2047,2048,2051],{"style":1786},[169,2049],{"className":2050,"style":485},[484],[169,2052,2054],{"className":2053},[489,490,491,492],[169,2055,2057],{"className":2056},[212,492],[169,2058,806],{"className":2059,"style":823},[212,213,492],[169,2061,500],{"className":2062},[499],[169,2064,2066],{"className":2065},[472],[169,2067,2069],{"className":2068,"style":507},[476],[169,2070],{},[169,2072,431],{"className":2073},[535],[169,2075,190],{"className":2076},[212,213],[169,2078,440],{"className":2079},[579],[169,2081,1911],{"className":2082,"style":2027},[579,2026],[169,2084,2086],{"className":2085},[463],[169,2087,2089,2111],{"className":2088},[467,468],[169,2090,2092,2108],{"className":2091},[472],[169,2093,2096],{"className":2094,"style":2095},[476],"height:0.0017em;",[169,2097,2099,2102],{"style":2098},"top:-2.4003em;margin-right:0.05em;",[169,2100],{"className":2101,"style":485},[484],[169,2103,2105],{"className":2104},[489,490,491,492],[169,2106,18],{"className":2107},[212,213,492],[169,2109,500],{"className":2110},[499],[169,2112,2114],{"className":2113},[472],[169,2115,2118],{"className":2116,"style":2117},[476],"height:0.2997em;",[169,2119],{},[10,2121,2123,2187,2279,2373,2480,2545,2845,3065,3121,3408],{"title":2122},"对角高斯策略",[14,2124,2125,2126,2154,2155,2186],{},"多元高斯分布（multivariate Gaussian distribution，也叫多元正态分布）由均值向量 ",[169,2127,2129,2142],{"className":2128},[172],[169,2130,2132],{"className":2131},[176],[178,2133,2134],{"xmlns":180},[182,2135,2136,2140],{},[185,2137,2138],{},[188,2139,379],{},[192,2141,382],{"encoding":194},[169,2143,2145],{"className":2144,"ariaHidden":199},[198],[169,2146,2148,2151],{"className":2147},[203],[169,2149],{"className":2150,"style":392},[207],[169,2152,379],{"className":2153},[212,213]," 和协方差矩阵 ",[169,2156,2158,2173],{"className":2157},[172],[169,2159,2161],{"className":2160},[176],[178,2162,2163],{"xmlns":180},[182,2164,2165,2170],{},[185,2166,2167],{},[188,2168,2169],{"mathvariant":647},"Σ",[192,2171,2172],{"encoding":194},"\\Sigma",[169,2174,2176],{"className":2175,"ariaHidden":199},[198],[169,2177,2179,2183],{"className":2178},[203],[169,2180],{"className":2181,"style":2182},[207],"height:0.6833em;",[169,2184,2169],{"className":2185},[212]," 描述。对角高斯分布是协方差矩阵只有对角线元素的特例，因此可以用一个向量来表示它。",[14,2188,2189,2190,2278],{},"对角高斯策略总有一个从观测映射到均值动作 ",[169,2191,2193,2217],{"className":2192},[172],[169,2194,2196],{"className":2195},[176],[178,2197,2198],{"xmlns":180},[182,2199,2200,2214],{},[185,2201,2202,2208,2210,2212],{},[415,2203,2204,2206],{},[188,2205,379],{},[188,2207,806],{},[423,2209,431],{"stretchy":430},[188,2211,190],{},[423,2213,440],{"stretchy":430},[192,2215,2216],{"encoding":194},"\\mu_{\\theta}(s)",[169,2218,2220],{"className":2219,"ariaHidden":199},[198],[169,2221,2223,2226,2269,2272,2275],{"className":2222},[203],[169,2224],{"className":2225,"style":528},[207],[169,2227,2229,2232],{"className":2228},[212],[169,2230,379],{"className":2231},[212,213],[169,2233,2235],{"className":2234},[463],[169,2236,2238,2261],{"className":2237},[467,468],[169,2239,2241,2258],{"className":2240},[472],[169,2242,2244],{"className":2243,"style":1155},[476],[169,2245,2246,2249],{"style":480},[169,2247],{"className":2248,"style":485},[484],[169,2250,2252],{"className":2251},[489,490,491,492],[169,2253,2255],{"className":2254},[212,492],[169,2256,806],{"className":2257,"style":823},[212,213,492],[169,2259,500],{"className":2260},[499],[169,2262,2264],{"className":2263},[472],[169,2265,2267],{"className":2266,"style":507},[476],[169,2268],{},[169,2270,431],{"className":2271},[535],[169,2273,190],{"className":2274},[212,213],[169,2276,440],{"className":2277},[579]," 的神经网络。协方差矩阵通常有两种表示方式。",[14,2280,2281,2284,2285,2327,2328,2331,2332,2372],{},[117,2282,2283],{},"第一种","：只有一个对数标准差向量 ",[169,2286,2288,2307],{"className":2287},[172],[169,2289,2291],{"className":2290},[176],[178,2292,2293],{"xmlns":180},[182,2294,2295,2304],{},[185,2296,2297,2299,2301],{},[188,2298,1580],{},[423,2300,1467],{},[188,2302,2303],{},"σ",[192,2305,2306],{"encoding":194},"\\log \\sigma",[169,2308,2310],{"className":2309,"ariaHidden":199},[198],[169,2311,2313,2316,2321,2324],{"className":2312},[203],[169,2314],{"className":2315,"style":852},[207],[169,2317,1614,2319],{"className":2318},[1483],[169,2320,1618],{"style":1617},[169,2322],{"className":2323,"style":1622},[513],[169,2325,2303],{"className":2326,"style":613},[212,213],"，它",[117,2329,2330],{},"不是","状态的函数：这些 ",[169,2333,2335,2352],{"className":2334},[172],[169,2336,2338],{"className":2337},[176],[178,2339,2340],{"xmlns":180},[182,2341,2342,2350],{},[185,2343,2344,2346,2348],{},[188,2345,1580],{},[423,2347,1467],{},[188,2349,2303],{},[192,2351,2306],{"encoding":194},[169,2353,2355],{"className":2354,"ariaHidden":199},[198],[169,2356,2358,2361,2366,2369],{"className":2357},[203],[169,2359],{"className":2360,"style":852},[207],[169,2362,1614,2364],{"className":2363},[1483],[169,2365,1618],{"style":1617},[169,2367],{"className":2368,"style":1622},[513],[169,2370,2303],{"className":2371,"style":613},[212,213]," 是独立的参数。（你应该知道：我们的 VPG（Vanilla Policy Gradient）、TRPO（Trust Region Policy Optimization）和 PPO 的实现用的就是这一种。）",[14,2374,2375,2378,2379,2479],{},[117,2376,2377],{},"第二种","：有一个从状态映射到对数标准差 ",[169,2380,2382,2410],{"className":2381},[172],[169,2383,2385],{"className":2384},[176],[178,2386,2387],{"xmlns":180},[182,2388,2389,2407],{},[185,2390,2391,2393,2395,2401,2403,2405],{},[188,2392,1580],{},[423,2394,1467],{},[415,2396,2397,2399],{},[188,2398,2303],{},[188,2400,806],{},[423,2402,431],{"stretchy":430},[188,2404,190],{},[423,2406,440],{"stretchy":430},[192,2408,2409],{"encoding":194},"\\log \\sigma_{\\theta}(s)",[169,2411,2413],{"className":2412,"ariaHidden":199},[198],[169,2414,2416,2419,2424,2427,2470,2473,2476],{"className":2415},[203],[169,2417],{"className":2418,"style":528},[207],[169,2420,1614,2422],{"className":2421},[1483],[169,2423,1618],{"style":1617},[169,2425],{"className":2426,"style":1622},[513],[169,2428,2430,2433],{"className":2429},[212],[169,2431,2303],{"className":2432,"style":613},[212,213],[169,2434,2436],{"className":2435},[463],[169,2437,2439,2462],{"className":2438},[467,468],[169,2440,2442,2459],{"className":2441},[472],[169,2443,2445],{"className":2444,"style":1155},[476],[169,2446,2447,2450],{"style":1267},[169,2448],{"className":2449,"style":485},[484],[169,2451,2453],{"className":2452},[489,490,491,492],[169,2454,2456],{"className":2455},[212,492],[169,2457,806],{"className":2458,"style":823},[212,213,492],[169,2460,500],{"className":2461},[499],[169,2463,2465],{"className":2464},[472],[169,2466,2468],{"className":2467,"style":507},[476],[169,2469],{},[169,2471,431],{"className":2472},[535],[169,2474,190],{"className":2475},[212,213],[169,2477,440],{"className":2478},[579]," 的神经网络，它可以选择性地与均值网络共享一些层。",[14,2481,2482,2483,2544],{},"注意，两种情况下输出的都是对数标准差，并非直接输出标准差。原因是对数标准差可以在 ",[169,2484,2486,2513],{"className":2485},[172],[169,2487,2489],{"className":2488},[176],[178,2490,2491],{"xmlns":180},[182,2492,2493,2510],{},[185,2494,2495,2497,2500,2503,2506,2508],{},[423,2496,431],{"stretchy":430},[423,2498,2499],{},"−",[188,2501,2502],{"mathvariant":647},"∞",[423,2504,2505],{"separator":199},",",[188,2507,2502],{"mathvariant":647},[423,2509,440],{"stretchy":430},[192,2511,2512],{"encoding":194},"(-\\infty, \\infty)",[169,2514,2516],{"className":2515,"ariaHidden":199},[198],[169,2517,2519,2522,2525,2528,2531,2535,2538,2541],{"className":2518},[203],[169,2520],{"className":2521,"style":528},[207],[169,2523,431],{"className":2524},[535],[169,2526,2499],{"className":2527},[212],[169,2529,2502],{"className":2530},[212],[169,2532,2505],{"className":2533},[2534],"mpunct",[169,2536],{"className":2537,"style":1622},[513],[169,2539,2502],{"className":2540},[212],[169,2542,440],{"className":2543},[579]," 上自由取值，而标准差必须非负；不必强制这类约束，参数训练起来更容易。对对数标准差取指数即可立即得到标准差，所以这种表示不损失任何东西。",[14,2546,2547,2549,2550,2637,2638,2726,2727,2757,2758,2844],{},[117,2548,1700],{},"。给定均值动作 ",[169,2551,2553,2576],{"className":2552},[172],[169,2554,2556],{"className":2555},[176],[178,2557,2558],{"xmlns":180},[182,2559,2560,2574],{},[185,2561,2562,2568,2570,2572],{},[415,2563,2564,2566],{},[188,2565,379],{},[188,2567,806],{},[423,2569,431],{"stretchy":430},[188,2571,190],{},[423,2573,440],{"stretchy":430},[192,2575,2216],{"encoding":194},[169,2577,2579],{"className":2578,"ariaHidden":199},[198],[169,2580,2582,2585,2628,2631,2634],{"className":2581},[203],[169,2583],{"className":2584,"style":528},[207],[169,2586,2588,2591],{"className":2587},[212],[169,2589,379],{"className":2590},[212,213],[169,2592,2594],{"className":2593},[463],[169,2595,2597,2620],{"className":2596},[467,468],[169,2598,2600,2617],{"className":2599},[472],[169,2601,2603],{"className":2602,"style":1155},[476],[169,2604,2605,2608],{"style":480},[169,2606],{"className":2607,"style":485},[484],[169,2609,2611],{"className":2610},[489,490,491,492],[169,2612,2614],{"className":2613},[212,492],[169,2615,806],{"className":2616,"style":823},[212,213,492],[169,2618,500],{"className":2619},[499],[169,2621,2623],{"className":2622},[472],[169,2624,2626],{"className":2625,"style":507},[476],[169,2627],{},[169,2629,431],{"className":2630},[535],[169,2632,190],{"className":2633},[212,213],[169,2635,440],{"className":2636},[579],"、标准差 ",[169,2639,2641,2665],{"className":2640},[172],[169,2642,2644],{"className":2643},[176],[178,2645,2646],{"xmlns":180},[182,2647,2648,2662],{},[185,2649,2650,2656,2658,2660],{},[415,2651,2652,2654],{},[188,2653,2303],{},[188,2655,806],{},[423,2657,431],{"stretchy":430},[188,2659,190],{},[423,2661,440],{"stretchy":430},[192,2663,2664],{"encoding":194},"\\sigma_{\\theta}(s)",[169,2666,2668],{"className":2667,"ariaHidden":199},[198],[169,2669,2671,2674,2717,2720,2723],{"className":2670},[203],[169,2672],{"className":2673,"style":528},[207],[169,2675,2677,2680],{"className":2676},[212],[169,2678,2303],{"className":2679,"style":613},[212,213],[169,2681,2683],{"className":2682},[463],[169,2684,2686,2709],{"className":2685},[467,468],[169,2687,2689,2706],{"className":2688},[472],[169,2690,2692],{"className":2691,"style":1155},[476],[169,2693,2694,2697],{"style":1267},[169,2695],{"className":2696,"style":485},[484],[169,2698,2700],{"className":2699},[489,490,491,492],[169,2701,2703],{"className":2702},[212,492],[169,2704,806],{"className":2705,"style":823},[212,213,492],[169,2707,500],{"className":2708},[499],[169,2710,2712],{"className":2711},[472],[169,2713,2715],{"className":2714,"style":507},[476],[169,2716],{},[169,2718,431],{"className":2719},[535],[169,2721,190],{"className":2722},[212,213],[169,2724,440],{"className":2725},[579],"，以及一个来自球形高斯的噪声向量 ",[169,2728,2730,2744],{"className":2729},[172],[169,2731,2733],{"className":2732},[176],[178,2734,2735],{"xmlns":180},[182,2736,2737,2742],{},[185,2738,2739],{},[188,2740,2741],{},"z",[192,2743,2741],{"encoding":194},[169,2745,2747],{"className":2746,"ariaHidden":199},[198],[169,2748,2750,2753],{"className":2749},[203],[169,2751],{"className":2752,"style":208},[207],[169,2754,2741],{"className":2755,"style":2756},[212,213],"margin-right:0.044em;","（",[169,2759,2761,2793],{"className":2760},[172],[169,2762,2764],{"className":2763},[176],[178,2765,2766],{"xmlns":180},[182,2767,2768,2790],{},[185,2769,2770,2772,2774,2778,2780,2783,2785,2788],{},[188,2771,2741],{},[423,2773,637],{},[188,2775,2777],{"mathvariant":2776},"script","N",[423,2779,431],{"stretchy":430},[2781,2782,885],"mn",{},[423,2784,2505],{"separator":199},[188,2786,2787],{},"I",[423,2789,440],{"stretchy":430},[192,2791,2792],{"encoding":194},"z \\sim \\mathcal{N}(0, I)",[169,2794,2796,2814],{"className":2795,"ariaHidden":199},[198],[169,2797,2799,2802,2805,2808,2811],{"className":2798},[203],[169,2800],{"className":2801,"style":208},[207],[169,2803,2741],{"className":2804,"style":2756},[212,213],[169,2806],{"className":2807,"style":514},[513],[169,2809,637],{"className":2810},[518],[169,2812],{"className":2813,"style":514},[513],[169,2815,2817,2820,2825,2828,2831,2834,2837,2841],{"className":2816},[203],[169,2818],{"className":2819,"style":528},[207],[169,2821,2777],{"className":2822,"style":2824},[212,2823],"mathcal","margin-right:0.1474em;",[169,2826,431],{"className":2827},[535],[169,2829,885],{"className":2830},[212],[169,2832,2505],{"className":2833},[2534],[169,2835],{"className":2836,"style":1622},[513],[169,2838,2787],{"className":2839,"style":2840},[212,213],"margin-right:0.0785em;",[169,2842,440],{"className":2843},[579],"），动作样本可以这样计算：",[169,2846,2848],{"className":2847},[400],[169,2849,2851,2899],{"className":2850},[172],[169,2852,2854],{"className":2853},[176],[178,2855,2856],{"xmlns":180,"display":409},[182,2857,2858,2896],{},[185,2859,2860,2862,2864,2870,2872,2874,2876,2879,2885,2887,2889,2891,2894],{},[188,2861,18],{},[423,2863,425],{},[415,2865,2866,2868],{},[188,2867,379],{},[188,2869,806],{},[423,2871,431],{"stretchy":430},[188,2873,190],{},[423,2875,440],{"stretchy":430},[423,2877,2878],{},"+",[415,2880,2881,2883],{},[188,2882,2303],{},[188,2884,806],{},[423,2886,431],{"stretchy":430},[188,2888,190],{},[423,2890,440],{"stretchy":430},[423,2892,2893],{},"⊙",[188,2895,2741],{},[192,2897,2898],{"encoding":194},"a = \\mu_{\\theta}(s) + \\sigma_{\\theta}(s) \\odot z",[169,2900,2902,2920,2989,3056],{"className":2901,"ariaHidden":199},[198],[169,2903,2905,2908,2911,2914,2917],{"className":2904},[203],[169,2906],{"className":2907,"style":208},[207],[169,2909,18],{"className":2910},[212,213],[169,2912],{"className":2913,"style":514},[513],[169,2915,425],{"className":2916},[518],[169,2918],{"className":2919,"style":514},[513],[169,2921,2923,2926,2969,2972,2975,2978,2982,2986],{"className":2922},[203],[169,2924],{"className":2925,"style":528},[207],[169,2927,2929,2932],{"className":2928},[212],[169,2930,379],{"className":2931},[212,213],[169,2933,2935],{"className":2934},[463],[169,2936,2938,2961],{"className":2937},[467,468],[169,2939,2941,2958],{"className":2940},[472],[169,2942,2944],{"className":2943,"style":1155},[476],[169,2945,2946,2949],{"style":480},[169,2947],{"className":2948,"style":485},[484],[169,2950,2952],{"className":2951},[489,490,491,492],[169,2953,2955],{"className":2954},[212,492],[169,2956,806],{"className":2957,"style":823},[212,213,492],[169,2959,500],{"className":2960},[499],[169,2962,2964],{"className":2963},[472],[169,2965,2967],{"className":2966,"style":507},[476],[169,2968],{},[169,2970,431],{"className":2971},[535],[169,2973,190],{"className":2974},[212,213],[169,2976,440],{"className":2977},[579],[169,2979],{"className":2980,"style":2981},[513],"margin-right:0.2222em;",[169,2983,2878],{"className":2984},[2985],"mbin",[169,2987],{"className":2988,"style":2981},[513],[169,2990,2992,2995,3038,3041,3044,3047,3050,3053],{"className":2991},[203],[169,2993],{"className":2994,"style":528},[207],[169,2996,2998,3001],{"className":2997},[212],[169,2999,2303],{"className":3000,"style":613},[212,213],[169,3002,3004],{"className":3003},[463],[169,3005,3007,3030],{"className":3006},[467,468],[169,3008,3010,3027],{"className":3009},[472],[169,3011,3013],{"className":3012,"style":1155},[476],[169,3014,3015,3018],{"style":1267},[169,3016],{"className":3017,"style":485},[484],[169,3019,3021],{"className":3020},[489,490,491,492],[169,3022,3024],{"className":3023},[212,492],[169,3025,806],{"className":3026,"style":823},[212,213,492],[169,3028,500],{"className":3029},[499],[169,3031,3033],{"className":3032},[472],[169,3034,3036],{"className":3035,"style":507},[476],[169,3037],{},[169,3039,431],{"className":3040},[535],[169,3042,190],{"className":3043},[212,213],[169,3045,440],{"className":3046},[579],[169,3048],{"className":3049,"style":2981},[513],[169,3051,2893],{"className":3052},[2985],[169,3054],{"className":3055,"style":2981},[513],[169,3057,3059,3062],{"className":3058},[203],[169,3060],{"className":3061,"style":208},[207],[169,3063,2741],{"className":3064,"style":2756},[212,213],[14,3066,3067,3068,3098,3099,824,3106,3109,3110,824,3117,3120],{},"其中 ",[169,3069,3071,3085],{"className":3070},[172],[169,3072,3074],{"className":3073},[176],[178,3075,3076],{"xmlns":180},[182,3077,3078,3082],{},[185,3079,3080],{},[423,3081,2893],{},[192,3083,3084],{"encoding":194},"\\odot",[169,3086,3088],{"className":3087,"ariaHidden":199},[198],[169,3089,3091,3095],{"className":3090},[203],[169,3092],{"className":3093,"style":3094},[207],"height:0.6667em;vertical-align:-0.0833em;",[169,3096,2893],{"className":3097},[212]," 表示两个向量的逐元素乘积。主流框架都有内置的方式生成噪声向量，比如 ",[18,3100,3103],{"href":3101,"rel":3102},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Ftorch.html#torch.normal",[22],[1365,3104,3105],{},"torch.normal",[1365,3107,3108],{},"tf.random_normal","；也可以构建分布对象，比如 ",[18,3111,3114],{"href":3112,"rel":3113},"https:\u002F\u002Fpytorch.org\u002Fdocs\u002Fstable\u002Fdistributions.html#normal",[22],[1365,3115,3116],{},"torch.distributions.Normal",[1365,3118,3119],{},"tf.distributions.Normal","，再用它们生成样本。（后一种方式的好处是，这些对象还能替你算对数似然。）",[14,3122,3123,3125,3126,3236,3237,3347,3348,3378,3379,3407],{},[117,3124,1726],{},"。对均值为 ",[169,3127,3129,3157],{"className":3128},[172],[169,3130,3132],{"className":3131},[176],[178,3133,3134],{"xmlns":180},[182,3135,3136,3154],{},[185,3137,3138,3140,3142,3148,3150,3152],{},[188,3139,379],{},[423,3141,425],{},[415,3143,3144,3146],{},[188,3145,379],{},[188,3147,806],{},[423,3149,431],{"stretchy":430},[188,3151,190],{},[423,3153,440],{"stretchy":430},[192,3155,3156],{"encoding":194},"\\mu = \\mu_{\\theta}(s)",[169,3158,3160,3178],{"className":3159,"ariaHidden":199},[198],[169,3161,3163,3166,3169,3172,3175],{"className":3162},[203],[169,3164],{"className":3165,"style":392},[207],[169,3167,379],{"className":3168},[212,213],[169,3170],{"className":3171,"style":514},[513],[169,3173,425],{"className":3174},[518],[169,3176],{"className":3177,"style":514},[513],[169,3179,3181,3184,3227,3230,3233],{"className":3180},[203],[169,3182],{"className":3183,"style":528},[207],[169,3185,3187,3190],{"className":3186},[212],[169,3188,379],{"className":3189},[212,213],[169,3191,3193],{"className":3192},[463],[169,3194,3196,3219],{"className":3195},[467,468],[169,3197,3199,3216],{"className":3198},[472],[169,3200,3202],{"className":3201,"style":1155},[476],[169,3203,3204,3207],{"style":480},[169,3205],{"className":3206,"style":485},[484],[169,3208,3210],{"className":3209},[489,490,491,492],[169,3211,3213],{"className":3212},[212,492],[169,3214,806],{"className":3215,"style":823},[212,213,492],[169,3217,500],{"className":3218},[499],[169,3220,3222],{"className":3221},[472],[169,3223,3225],{"className":3224,"style":507},[476],[169,3226],{},[169,3228,431],{"className":3229},[535],[169,3231,190],{"className":3232},[212,213],[169,3234,440],{"className":3235},[579],"、标准差为 ",[169,3238,3240,3268],{"className":3239},[172],[169,3241,3243],{"className":3242},[176],[178,3244,3245],{"xmlns":180},[182,3246,3247,3265],{},[185,3248,3249,3251,3253,3259,3261,3263],{},[188,3250,2303],{},[423,3252,425],{},[415,3254,3255,3257],{},[188,3256,2303],{},[188,3258,806],{},[423,3260,431],{"stretchy":430},[188,3262,190],{},[423,3264,440],{"stretchy":430},[192,3266,3267],{"encoding":194},"\\sigma = \\sigma_{\\theta}(s)",[169,3269,3271,3289],{"className":3270,"ariaHidden":199},[198],[169,3272,3274,3277,3280,3283,3286],{"className":3273},[203],[169,3275],{"className":3276,"style":208},[207],[169,3278,2303],{"className":3279,"style":613},[212,213],[169,3281],{"className":3282,"style":514},[513],[169,3284,425],{"className":3285},[518],[169,3287],{"className":3288,"style":514},[513],[169,3290,3292,3295,3338,3341,3344],{"className":3291},[203],[169,3293],{"className":3294,"style":528},[207],[169,3296,3298,3301],{"className":3297},[212],[169,3299,2303],{"className":3300,"style":613},[212,213],[169,3302,3304],{"className":3303},[463],[169,3305,3307,3330],{"className":3306},[467,468],[169,3308,3310,3327],{"className":3309},[472],[169,3311,3313],{"className":3312,"style":1155},[476],[169,3314,3315,3318],{"style":1267},[169,3316],{"className":3317,"style":485},[484],[169,3319,3321],{"className":3320},[489,490,491,492],[169,3322,3324],{"className":3323},[212,492],[169,3325,806],{"className":3326,"style":823},[212,213,492],[169,3328,500],{"className":3329},[499],[169,3331,3333],{"className":3332},[472],[169,3334,3336],{"className":3335,"style":507},[476],[169,3337],{},[169,3339,431],{"className":3340},[535],[169,3342,190],{"className":3343},[212,213],[169,3345,440],{"className":3346},[579]," 的对角高斯，",[169,3349,3351,3365],{"className":3350},[172],[169,3352,3354],{"className":3353},[176],[178,3355,3356],{"xmlns":180},[182,3357,3358,3363],{},[185,3359,3360],{},[188,3361,3362],{},"k",[192,3364,3362],{"encoding":194},[169,3366,3368],{"className":3367,"ariaHidden":199},[198],[169,3369,3371,3374],{"className":3370},[203],[169,3372],{"className":3373,"style":819},[207],[169,3375,3362],{"className":3376,"style":3377},[212,213],"margin-right:0.0315em;"," 维动作 ",[169,3380,3382,3395],{"className":3381},[172],[169,3383,3385],{"className":3384},[176],[178,3386,3387],{"xmlns":180},[182,3388,3389,3393],{},[185,3390,3391],{},[188,3392,18],{},[192,3394,18],{"encoding":194},[169,3396,3398],{"className":3397,"ariaHidden":199},[198],[169,3399,3401,3404],{"className":3400},[203],[169,3402],{"className":3403,"style":208},[207],[169,3405,18],{"className":3406},[212,213]," 的对数似然为",[169,3409,3411],{"className":3410},[400],[169,3412,3414,3550],{"className":3413},[172],[169,3415,3417],{"className":3416},[176],[178,3418,3419],{"xmlns":180,"display":409},[182,3420,3421,3547],{},[185,3422,3423,3425,3427,3433,3435,3437,3439,3441,3443,3445,3447,3456],{},[188,3424,1580],{},[423,3426,1467],{},[415,3428,3429,3431],{},[188,3430,597],{},[188,3432,806],{},[423,3434,431],{"stretchy":430},[188,3436,18],{},[188,3438,648],{"mathvariant":647},[188,3440,190],{},[423,3442,440],{"stretchy":430},[423,3444,425],{},[423,3446,2499],{},[3448,3449,3450,3453],"mfrac",{},[2781,3451,3452],{},"1",[2781,3454,3455],{},"2",[185,3457,3458,3460,3477,3533,3535,3537,3539,3541,3543,3545],{},[423,3459,431],{"fence":199},[3461,3462,3463,3466,3475],"munderover",{},[423,3464,3465],{},"∑",[185,3467,3468,3471,3473],{},[188,3469,3470],{},"i",[423,3472,425],{},[2781,3474,3452],{},[188,3476,3362],{},[185,3478,3479,3481,3517,3519,3521,3523,3525,3531],{},[423,3480,431],{"fence":199},[3448,3482,3483,3508],{},[185,3484,3485,3487,3493,3495,3501],{},[423,3486,431],{"stretchy":430},[415,3488,3489,3491],{},[188,3490,18],{},[188,3492,3470],{},[423,3494,2499],{},[415,3496,3497,3499],{},[188,3498,379],{},[188,3500,3470],{},[3502,3503,3504,3506],"msup",{},[423,3505,440],{"stretchy":430},[2781,3507,3455],{},[3509,3510,3511,3513,3515],"msubsup",{},[188,3512,2303],{},[188,3514,3470],{},[2781,3516,3455],{},[423,3518,2878],{},[2781,3520,3455],{},[188,3522,1580],{},[423,3524,1467],{},[415,3526,3527,3529],{},[188,3528,2303],{},[188,3530,3470],{},[423,3532,440],{"fence":199},[423,3534,2878],{},[188,3536,3362],{},[188,3538,1580],{},[423,3540,1467],{},[2781,3542,3455],{},[188,3544,597],{},[423,3546,440],{"fence":199},[192,3548,3549],{"encoding":194},"\\log \\pi_{\\theta}(a|s) = -\\frac{1}{2}\\left(\\sum_{i=1}^k \\left(\\frac{(a_i - \\mu_i)^2}{\\sigma_i^2} + 2 \\log \\sigma_i \\right) + k \\log 2\\pi \\right)",[169,3551,3553,3634],{"className":3552,"ariaHidden":199},[198],[169,3554,3556,3559,3564,3567,3610,3613,3616,3619,3622,3625,3628,3631],{"className":3555},[203],[169,3557],{"className":3558,"style":528},[207],[169,3560,1614,3562],{"className":3561},[1483],[169,3563,1618],{"style":1617},[169,3565],{"className":3566,"style":1622},[513],[169,3568,3570,3573],{"className":3569},[212],[169,3571,597],{"className":3572,"style":613},[212,213],[169,3574,3576],{"className":3575},[463],[169,3577,3579,3602],{"className":3578},[467,468],[169,3580,3582,3599],{"className":3581},[472],[169,3583,3585],{"className":3584,"style":1155},[476],[169,3586,3587,3590],{"style":1267},[169,3588],{"className":3589,"style":485},[484],[169,3591,3593],{"className":3592},[489,490,491,492],[169,3594,3596],{"className":3595},[212,492],[169,3597,806],{"className":3598,"style":823},[212,213,492],[169,3600,500],{"className":3601},[499],[169,3603,3605],{"className":3604},[472],[169,3606,3608],{"className":3607,"style":507},[476],[169,3609],{},[169,3611,431],{"className":3612},[535],[169,3614,18],{"className":3615},[212,213],[169,3617,648],{"className":3618},[212],[169,3620,190],{"className":3621},[212,213],[169,3623,440],{"className":3624},[579],[169,3626],{"className":3627,"style":514},[513],[169,3629,425],{"className":3630},[518],[169,3632],{"className":3633,"style":514},[513],[169,3635,3637,3641,3644,3714,3717],{"className":3636},[203],[169,3638],{"className":3639,"style":3640},[207],"height:3.1138em;vertical-align:-1.2777em;",[169,3642,2499],{"className":3643},[212],[169,3645,3647,3651,3711],{"className":3646},[212],[169,3648],{"className":3649},[535,3650],"nulldelimiter",[169,3652,3654],{"className":3653},[3448],[169,3655,3657,3702],{"className":3656},[467,468],[169,3658,3660,3699],{"className":3659},[472],[169,3661,3664,3676,3687],{"className":3662,"style":3663},[476],"height:1.3214em;",[169,3665,3667,3670],{"style":3666},"top:-2.314em;",[169,3668],{"className":3669,"style":999},[484],[169,3671,3673],{"className":3672},[212],[169,3674,3455],{"className":3675},[212],[169,3677,3679,3682],{"style":3678},"top:-3.23em;",[169,3680],{"className":3681,"style":999},[484],[169,3683],{"className":3684,"style":3686},[3685],"frac-line","border-bottom-width:0.04em;",[169,3688,3690,3693],{"style":3689},"top:-3.677em;",[169,3691],{"className":3692,"style":999},[484],[169,3694,3696],{"className":3695},[212],[169,3697,3452],{"className":3698},[212],[169,3700,500],{"className":3701},[499],[169,3703,3705],{"className":3704},[472],[169,3706,3709],{"className":3707,"style":3708},[476],"height:0.686em;",[169,3710],{},[169,3712],{"className":3713},[579,3650],[169,3715],{"className":3716,"style":1622},[513],[169,3718,3720,3728,3801,3804,4119,4122,4125,4128,4131,4134,4139,4142,4145,4148],{"className":3719},[2019],[169,3721,3723],{"className":3722,"style":2027},[535,2026],[169,3724,431],{"className":3725},[3726,3727],"delimsizing","size4",[169,3729,3732],{"className":3730},[1483,3731],"op-limits",[169,3733,3735,3792],{"className":3734},[467,468],[169,3736,3738,3789],{"className":3737},[472],[169,3739,3742,3764,3777],{"className":3740,"style":3741},[476],"height:1.8361em;",[169,3743,3745,3749],{"style":3744},"top:-1.8723em;margin-left:0em;",[169,3746],{"className":3747,"style":3748},[484],"height:3.05em;",[169,3750,3752],{"className":3751},[489,490,491,492],[169,3753,3755,3758,3761],{"className":3754},[212,492],[169,3756,3470],{"className":3757},[212,213,492],[169,3759,425],{"className":3760},[518,492],[169,3762,3452],{"className":3763},[212,492],[169,3765,3767,3770],{"style":3766},"top:-3.05em;",[169,3768],{"className":3769,"style":3748},[484],[169,3771,3772],{},[169,3773,3465],{"className":3774},[1483,3775,3776],"op-symbol","large-op",[169,3778,3780,3783],{"style":3779},"top:-4.3em;margin-left:0em;",[169,3781],{"className":3782,"style":3748},[484],[169,3784,3786],{"className":3785},[489,490,491,492],[169,3787,3362],{"className":3788,"style":3377},[212,213,492],[169,3790,500],{"className":3791},[499],[169,3793,3795],{"className":3794},[472],[169,3796,3799],{"className":3797,"style":3798},[476],"height:1.2777em;",[169,3800],{},[169,3802],{"className":3803,"style":1622},[513],[169,3805,3807,3813,4050,4053,4056,4059,4062,4065,4070,4073,4113],{"className":3806},[2019],[169,3808,3810],{"className":3809,"style":2027},[535,2026],[169,3811,431],{"className":3812},[3726,491],[169,3814,3816,3819,4047],{"className":3815},[212],[169,3817],{"className":3818},[535,3650],[169,3820,3822],{"className":3821},[3448],[169,3823,3825,4038],{"className":3824},[467,468],[169,3826,3828,4035],{"className":3827},[472],[169,3829,3832,3895,3903],{"className":3830,"style":3831},[476],"height:1.4911em;",[169,3833,3834,3837],{"style":3666},[169,3835],{"className":3836,"style":999},[484],[169,3838,3840],{"className":3839},[212],[169,3841,3843,3846],{"className":3842},[212],[169,3844,2303],{"className":3845,"style":613},[212,213],[169,3847,3849],{"className":3848},[463],[169,3850,3852,3886],{"className":3851},[467,468],[169,3853,3855,3883],{"className":3854},[472],[169,3856,3859,3871],{"className":3857,"style":3858},[476],"height:0.7959em;",[169,3860,3862,3865],{"style":3861},"top:-2.4231em;margin-left:-0.0359em;margin-right:0.05em;",[169,3863],{"className":3864,"style":485},[484],[169,3866,3868],{"className":3867},[489,490,491,492],[169,3869,3470],{"className":3870},[212,213,492],[169,3872,3874,3877],{"style":3873},"top:-3.0448em;margin-right:0.05em;",[169,3875],{"className":3876,"style":485},[484],[169,3878,3880],{"className":3879},[489,490,491,492],[169,3881,3455],{"className":3882},[212,492],[169,3884,500],{"className":3885},[499],[169,3887,3889],{"className":3888},[472],[169,3890,3893],{"className":3891,"style":3892},[476],"height:0.2769em;",[169,3894],{},[169,3896,3897,3900],{"style":3678},[169,3898],{"className":3899,"style":999},[484],[169,3901],{"className":3902,"style":3686},[3685],[169,3904,3905,3908],{"style":3689},[169,3906],{"className":3907,"style":999},[484],[169,3909,3911,3914,3955,3958,3961,3964,4004],{"className":3910},[212],[169,3912,431],{"className":3913},[535],[169,3915,3917,3920],{"className":3916},[212],[169,3918,18],{"className":3919},[212,213],[169,3921,3923],{"className":3922},[463],[169,3924,3926,3947],{"className":3925},[467,468],[169,3927,3929,3944],{"className":3928},[472],[169,3930,3933],{"className":3931,"style":3932},[476],"height:0.3117em;",[169,3934,3935,3938],{"style":480},[169,3936],{"className":3937,"style":485},[484],[169,3939,3941],{"className":3940},[489,490,491,492],[169,3942,3470],{"className":3943},[212,213,492],[169,3945,500],{"className":3946},[499],[169,3948,3950],{"className":3949},[472],[169,3951,3953],{"className":3952,"style":507},[476],[169,3954],{},[169,3956],{"className":3957,"style":2981},[513],[169,3959,2499],{"className":3960},[2985],[169,3962],{"className":3963,"style":2981},[513],[169,3965,3967,3970],{"className":3966},[212],[169,3968,379],{"className":3969},[212,213],[169,3971,3973],{"className":3972},[463],[169,3974,3976,3996],{"className":3975},[467,468],[169,3977,3979,3993],{"className":3978},[472],[169,3980,3982],{"className":3981,"style":3932},[476],[169,3983,3984,3987],{"style":480},[169,3985],{"className":3986,"style":485},[484],[169,3988,3990],{"className":3989},[489,490,491,492],[169,3991,3470],{"className":3992},[212,213,492],[169,3994,500],{"className":3995},[499],[169,3997,3999],{"className":3998},[472],[169,4000,4002],{"className":4001,"style":507},[476],[169,4003],{},[169,4005,4007,4010],{"className":4006},[579],[169,4008,440],{"className":4009},[579],[169,4011,4013],{"className":4012},[463],[169,4014,4016],{"className":4015},[467],[169,4017,4019],{"className":4018},[472],[169,4020,4023],{"className":4021,"style":4022},[476],"height:0.8141em;",[169,4024,4026,4029],{"style":4025},"top:-3.063em;margin-right:0.05em;",[169,4027],{"className":4028,"style":485},[484],[169,4030,4032],{"className":4031},[489,490,491,492],[169,4033,3455],{"className":4034},[212,492],[169,4036,500],{"className":4037},[499],[169,4039,4041],{"className":4040},[472],[169,4042,4045],{"className":4043,"style":4044},[476],"height:0.9629em;",[169,4046],{},[169,4048],{"className":4049},[579,3650],[169,4051],{"className":4052,"style":2981},[513],[169,4054,2878],{"className":4055},[2985],[169,4057],{"className":4058,"style":2981},[513],[169,4060,3455],{"className":4061},[212],[169,4063],{"className":4064,"style":1622},[513],[169,4066,1614,4068],{"className":4067},[1483],[169,4069,1618],{"style":1617},[169,4071],{"className":4072,"style":1622},[513],[169,4074,4076,4079],{"className":4075},[212],[169,4077,2303],{"className":4078,"style":613},[212,213],[169,4080,4082],{"className":4081},[463],[169,4083,4085,4105],{"className":4084},[467,468],[169,4086,4088,4102],{"className":4087},[472],[169,4089,4091],{"className":4090,"style":3932},[476],[169,4092,4093,4096],{"style":1267},[169,4094],{"className":4095,"style":485},[484],[169,4097,4099],{"className":4098},[489,490,491,492],[169,4100,3470],{"className":4101},[212,213,492],[169,4103,500],{"className":4104},[499],[169,4106,4108],{"className":4107},[472],[169,4109,4111],{"className":4110,"style":507},[476],[169,4112],{},[169,4114,4116],{"className":4115,"style":2027},[579,2026],[169,4117,440],{"className":4118},[3726,491],[169,4120],{"className":4121,"style":2981},[513],[169,4123,2878],{"className":4124},[2985],[169,4126],{"className":4127,"style":2981},[513],[169,4129,3362],{"className":4130,"style":3377},[212,213],[169,4132],{"className":4133,"style":1622},[513],[169,4135,1614,4137],{"className":4136},[1483],[169,4138,1618],{"style":1617},[169,4140],{"className":4141,"style":1622},[513],[169,4143,3455],{"className":4144},[212],[169,4146,597],{"className":4147,"style":613},[212,213],[169,4149,4151],{"className":4150,"style":2027},[579,2026],[169,4152,440],{"className":4153},[3726,3727],[159,4155,4156],{"id":4156},"轨迹",[14,4158,4159,4160,4191],{},"轨迹（trajectory）",[169,4161,4163,4178],{"className":4162},[172],[169,4164,4166],{"className":4165},[176],[178,4167,4168],{"xmlns":180},[182,4169,4170,4175],{},[185,4171,4172],{},[188,4173,4174],{},"τ",[192,4176,4177],{"encoding":194},"\\tau",[169,4179,4181],{"className":4180,"ariaHidden":199},[198],[169,4182,4184,4187],{"className":4183},[203],[169,4185],{"className":4186,"style":208},[207],[169,4188,4174],{"className":4189,"style":4190},[212,213],"margin-right:0.1132em;"," 是世界中状态与动作构成的序列：",[169,4193,4195],{"className":4194},[400],[169,4196,4198,4257],{"className":4197},[172],[169,4199,4201],{"className":4200},[176],[178,4202,4203],{"xmlns":180,"display":409},[182,4204,4205,4254],{},[185,4206,4207,4209,4211,4213,4219,4221,4227,4229,4235,4237,4243,4245,4248,4250,4252],{},[188,4208,4174],{},[423,4210,425],{},[423,4212,431],{"stretchy":430},[415,4214,4215,4217],{},[188,4216,190],{},[2781,4218,885],{},[423,4220,2505],{"separator":199},[415,4222,4223,4225],{},[188,4224,18],{},[2781,4226,885],{},[423,4228,2505],{"separator":199},[415,4230,4231,4233],{},[188,4232,190],{},[2781,4234,3452],{},[423,4236,2505],{"separator":199},[415,4238,4239,4241],{},[188,4240,18],{},[2781,4242,3452],{},[423,4244,2505],{"separator":199},[188,4246,4247],{"mathvariant":647},".",[188,4249,4247],{"mathvariant":647},[188,4251,4247],{"mathvariant":647},[423,4253,440],{"stretchy":430},[192,4255,4256],{"encoding":194},"\\tau = (s_0, a_0, s_1, a_1, ...)",[169,4258,4260,4278],{"className":4259,"ariaHidden":199},[198],[169,4261,4263,4266,4269,4272,4275],{"className":4262},[203],[169,4264],{"className":4265,"style":208},[207],[169,4267,4174],{"className":4268,"style":4190},[212,213],[169,4270],{"className":4271,"style":514},[513],[169,4273,425],{"className":4274},[518],[169,4276],{"className":4277,"style":514},[513],[169,4279,4281,4284,4287,4328,4331,4334,4374,4377,4380,4420,4423,4426,4466,4469,4472,4476],{"className":4280},[203],[169,4282],{"className":4283,"style":528},[207],[169,4285,431],{"className":4286},[535],[169,4288,4290,4293],{"className":4289},[212],[169,4291,190],{"className":4292},[212,213],[169,4294,4296],{"className":4295},[463],[169,4297,4299,4320],{"className":4298},[467,468],[169,4300,4302,4317],{"className":4301},[472],[169,4303,4306],{"className":4304,"style":4305},[476],"height:0.3011em;",[169,4307,4308,4311],{"style":480},[169,4309],{"className":4310,"style":485},[484],[169,4312,4314],{"className":4313},[489,490,491,492],[169,4315,885],{"className":4316},[212,492],[169,4318,500],{"className":4319},[499],[169,4321,4323],{"className":4322},[472],[169,4324,4326],{"className":4325,"style":507},[476],[169,4327],{},[169,4329,2505],{"className":4330},[2534],[169,4332],{"className":4333,"style":1622},[513],[169,4335,4337,4340],{"className":4336},[212],[169,4338,18],{"className":4339},[212,213],[169,4341,4343],{"className":4342},[463],[169,4344,4346,4366],{"className":4345},[467,468],[169,4347,4349,4363],{"className":4348},[472],[169,4350,4352],{"className":4351,"style":4305},[476],[169,4353,4354,4357],{"style":480},[169,4355],{"className":4356,"style":485},[484],[169,4358,4360],{"className":4359},[489,490,491,492],[169,4361,885],{"className":4362},[212,492],[169,4364,500],{"className":4365},[499],[169,4367,4369],{"className":4368},[472],[169,4370,4372],{"className":4371,"style":507},[476],[169,4373],{},[169,4375,2505],{"className":4376},[2534],[169,4378],{"className":4379,"style":1622},[513],[169,4381,4383,4386],{"className":4382},[212],[169,4384,190],{"className":4385},[212,213],[169,4387,4389],{"className":4388},[463],[169,4390,4392,4412],{"className":4391},[467,468],[169,4393,4395,4409],{"className":4394},[472],[169,4396,4398],{"className":4397,"style":4305},[476],[169,4399,4400,4403],{"style":480},[169,4401],{"className":4402,"style":485},[484],[169,4404,4406],{"className":4405},[489,490,491,492],[169,4407,3452],{"className":4408},[212,492],[169,4410,500],{"className":4411},[499],[169,4413,4415],{"className":4414},[472],[169,4416,4418],{"className":4417,"style":507},[476],[169,4419],{},[169,4421,2505],{"className":4422},[2534],[169,4424],{"className":4425,"style":1622},[513],[169,4427,4429,4432],{"className":4428},[212],[169,4430,18],{"className":4431},[212,213],[169,4433,4435],{"className":4434},[463],[169,4436,4438,4458],{"className":4437},[467,468],[169,4439,4441,4455],{"className":4440},[472],[169,4442,4444],{"className":4443,"style":4305},[476],[169,4445,4446,4449],{"style":480},[169,4447],{"className":4448,"style":485},[484],[169,4450,4452],{"className":4451},[489,490,491,492],[169,4453,3452],{"className":4454},[212,492],[169,4456,500],{"className":4457},[499],[169,4459,4461],{"className":4460},[472],[169,4462,4464],{"className":4463,"style":507},[476],[169,4465],{},[169,4467,2505],{"className":4468},[2534],[169,4470],{"className":4471,"style":1622},[513],[169,4473,4475],{"className":4474},[212],"...",[169,4477,440],{"className":4478},[579],[14,4480,4481,4482,4552,4553,4556,4557,396],{},"世界的第一个状态 ",[169,4483,4485,4503],{"className":4484},[172],[169,4486,4488],{"className":4487},[176],[178,4489,4490],{"xmlns":180},[182,4491,4492,4500],{},[185,4493,4494],{},[415,4495,4496,4498],{},[188,4497,190],{},[2781,4499,885],{},[192,4501,4502],{"encoding":194},"s_0",[169,4504,4506],{"className":4505,"ariaHidden":199},[198],[169,4507,4509,4512],{"className":4508},[203],[169,4510],{"className":4511,"style":453},[207],[169,4513,4515,4518],{"className":4514},[212],[169,4516,190],{"className":4517},[212,213],[169,4519,4521],{"className":4520},[463],[169,4522,4524,4544],{"className":4523},[467,468],[169,4525,4527,4541],{"className":4526},[472],[169,4528,4530],{"className":4529,"style":4305},[476],[169,4531,4532,4535],{"style":480},[169,4533],{"className":4534,"style":485},[484],[169,4536,4538],{"className":4537},[489,490,491,492],[169,4539,885],{"className":4540},[212,492],[169,4542,500],{"className":4543},[499],[169,4545,4547],{"className":4546},[472],[169,4548,4550],{"className":4549,"style":507},[476],[169,4551],{}," 从",[117,4554,4555],{},"初始状态分布（start-state distribution）"," 中随机采样，后者有时记作 ",[169,4558,4560,4579],{"className":4559},[172],[169,4561,4563],{"className":4562},[176],[178,4564,4565],{"xmlns":180},[182,4566,4567,4576],{},[185,4568,4569],{},[415,4570,4571,4574],{},[188,4572,4573],{},"ρ",[2781,4575,885],{},[192,4577,4578],{"encoding":194},"\\rho_0",[169,4580,4582],{"className":4581,"ariaHidden":199},[198],[169,4583,4585,4588],{"className":4584},[203],[169,4586],{"className":4587,"style":392},[207],[169,4589,4591,4594],{"className":4590},[212],[169,4592,4573],{"className":4593},[212,213],[169,4595,4597],{"className":4596},[463],[169,4598,4600,4620],{"className":4599},[467,468],[169,4601,4603,4617],{"className":4602},[472],[169,4604,4606],{"className":4605,"style":4305},[476],[169,4607,4608,4611],{"style":480},[169,4609],{"className":4610,"style":485},[484],[169,4612,4614],{"className":4613},[489,490,491,492],[169,4615,885],{"className":4616},[212,492],[169,4618,500],{"className":4619},[499],[169,4621,4623],{"className":4622},[472],[169,4624,4626],{"className":4625,"style":507},[476],[169,4627],{},[169,4629,4631],{"className":4630},[400],[169,4632,4634,4666],{"className":4633},[172],[169,4635,4637],{"className":4636},[176],[178,4638,4639],{"xmlns":180,"display":409},[182,4640,4641,4663],{},[185,4642,4643,4649,4651,4657,4659,4661],{},[415,4644,4645,4647],{},[188,4646,190],{},[2781,4648,885],{},[423,4650,637],{},[415,4652,4653,4655],{},[188,4654,4573],{},[2781,4656,885],{},[423,4658,431],{"stretchy":430},[423,4660,644],{},[423,4662,440],{"stretchy":430},[192,4664,4665],{"encoding":194},"s_0 \\sim \\rho_0(\\cdot)",[169,4667,4669,4724],{"className":4668,"ariaHidden":199},[198],[169,4670,4672,4675,4715,4718,4721],{"className":4671},[203],[169,4673],{"className":4674,"style":453},[207],[169,4676,4678,4681],{"className":4677},[212],[169,4679,190],{"className":4680},[212,213],[169,4682,4684],{"className":4683},[463],[169,4685,4687,4707],{"className":4686},[467,468],[169,4688,4690,4704],{"className":4689},[472],[169,4691,4693],{"className":4692,"style":4305},[476],[169,4694,4695,4698],{"style":480},[169,4696],{"className":4697,"style":485},[484],[169,4699,4701],{"className":4700},[489,490,491,492],[169,4702,885],{"className":4703},[212,492],[169,4705,500],{"className":4706},[499],[169,4708,4710],{"className":4709},[472],[169,4711,4713],{"className":4712,"style":507},[476],[169,4714],{},[169,4716],{"className":4717,"style":514},[513],[169,4719,637],{"className":4720},[518],[169,4722],{"className":4723,"style":514},[513],[169,4725,4727,4730,4770,4773,4776],{"className":4726},[203],[169,4728],{"className":4729,"style":528},[207],[169,4731,4733,4736],{"className":4732},[212],[169,4734,4573],{"className":4735},[212,213],[169,4737,4739],{"className":4738},[463],[169,4740,4742,4762],{"className":4741},[467,468],[169,4743,4745,4759],{"className":4744},[472],[169,4746,4748],{"className":4747,"style":4305},[476],[169,4749,4750,4753],{"style":480},[169,4751],{"className":4752,"style":485},[484],[169,4754,4756],{"className":4755},[489,490,491,492],[169,4757,885],{"className":4758},[212,492],[169,4760,500],{"className":4761},[499],[169,4763,4765],{"className":4764},[472],[169,4766,4768],{"className":4767,"style":507},[476],[169,4769],{},[169,4771,431],{"className":4772},[535],[169,4774,644],{"className":4775},[212],[169,4777,440],{"className":4778},[579],[14,4780,4781,4782,4811,4812,4882,4883,4936,4937,5024,5025,5095],{},"状态转移（从时刻 ",[169,4783,4785,4798],{"className":4784},[172],[169,4786,4788],{"className":4787},[176],[178,4789,4790],{"xmlns":180},[182,4791,4792,4796],{},[185,4793,4794],{},[188,4795,421],{},[192,4797,421],{"encoding":194},[169,4799,4801],{"className":4800,"ariaHidden":199},[198],[169,4802,4804,4808],{"className":4803},[203],[169,4805],{"className":4806,"style":4807},[207],"height:0.6151em;",[169,4809,421],{"className":4810},[212,213]," 的世界状态 ",[169,4813,4815,4833],{"className":4814},[172],[169,4816,4818],{"className":4817},[176],[178,4819,4820],{"xmlns":180},[182,4821,4822,4830],{},[185,4823,4824],{},[415,4825,4826,4828],{},[188,4827,190],{},[188,4829,421],{},[192,4831,4832],{"encoding":194},"s_t",[169,4834,4836],{"className":4835,"ariaHidden":199},[198],[169,4837,4839,4842],{"className":4838},[203],[169,4840],{"className":4841,"style":453},[207],[169,4843,4845,4848],{"className":4844},[212],[169,4846,190],{"className":4847},[212,213],[169,4849,4851],{"className":4850},[463],[169,4852,4854,4874],{"className":4853},[467,468],[169,4855,4857,4871],{"className":4856},[472],[169,4858,4860],{"className":4859,"style":477},[476],[169,4861,4862,4865],{"style":480},[169,4863],{"className":4864,"style":485},[484],[169,4866,4868],{"className":4867},[489,490,491,492],[169,4869,421],{"className":4870},[212,213,492],[169,4872,500],{"className":4873},[499],[169,4875,4877],{"className":4876},[472],[169,4878,4880],{"className":4879,"style":507},[476],[169,4881],{}," 到时刻 ",[169,4884,4886,4904],{"className":4885},[172],[169,4887,4889],{"className":4888},[176],[178,4890,4891],{"xmlns":180},[182,4892,4893,4901],{},[185,4894,4895,4897,4899],{},[188,4896,421],{},[423,4898,2878],{},[2781,4900,3452],{},[192,4902,4903],{"encoding":194},"t+1",[169,4905,4907,4926],{"className":4906,"ariaHidden":199},[198],[169,4908,4910,4914,4917,4920,4923],{"className":4909},[203],[169,4911],{"className":4912,"style":4913},[207],"height:0.6984em;vertical-align:-0.0833em;",[169,4915,421],{"className":4916},[212,213],[169,4918],{"className":4919,"style":2981},[513],[169,4921,2878],{"className":4922},[2985],[169,4924],{"className":4925,"style":2981},[513],[169,4927,4929,4933],{"className":4928},[203],[169,4930],{"className":4931,"style":4932},[207],"height:0.6444em;",[169,4934,3452],{"className":4935},[212]," 的状态 ",[169,4938,4940,4964],{"className":4939},[172],[169,4941,4943],{"className":4942},[176],[178,4944,4945],{"xmlns":180},[182,4946,4947,4961],{},[185,4948,4949],{},[415,4950,4951,4953],{},[188,4952,190],{},[185,4954,4955,4957,4959],{},[188,4956,421],{},[423,4958,2878],{},[2781,4960,3452],{},[192,4962,4963],{"encoding":194},"s_{t+1}",[169,4965,4967],{"className":4966,"ariaHidden":199},[198],[169,4968,4970,4974],{"className":4969},[203],[169,4971],{"className":4972,"style":4973},[207],"height:0.6389em;vertical-align:-0.2083em;",[169,4975,4977,4980],{"className":4976},[212],[169,4978,190],{"className":4979},[212,213],[169,4981,4983],{"className":4982},[463],[169,4984,4986,5015],{"className":4985},[467,468],[169,4987,4989,5012],{"className":4988},[472],[169,4990,4992],{"className":4991,"style":4305},[476],[169,4993,4994,4997],{"style":480},[169,4995],{"className":4996,"style":485},[484],[169,4998,5000],{"className":4999},[489,490,491,492],[169,5001,5003,5006,5009],{"className":5002},[212,492],[169,5004,421],{"className":5005},[212,213,492],[169,5007,2878],{"className":5008},[2985,492],[169,5010,3452],{"className":5011},[212,492],[169,5013,500],{"className":5014},[499],[169,5016,5018],{"className":5017},[472],[169,5019,5022],{"className":5020,"style":5021},[476],"height:0.2083em;",[169,5023],{}," 之间世界发生了什么）由环境的自然法则支配，且只依赖最近的动作 ",[169,5026,5028,5046],{"className":5027},[172],[169,5029,5031],{"className":5030},[176],[178,5032,5033],{"xmlns":180},[182,5034,5035,5043],{},[185,5036,5037],{},[415,5038,5039,5041],{},[188,5040,18],{},[188,5042,421],{},[192,5044,5045],{"encoding":194},"a_t",[169,5047,5049],{"className":5048,"ariaHidden":199},[198],[169,5050,5052,5055],{"className":5051},[203],[169,5053],{"className":5054,"style":453},[207],[169,5056,5058,5061],{"className":5057},[212],[169,5059,18],{"className":5060},[212,213],[169,5062,5064],{"className":5063},[463],[169,5065,5067,5087],{"className":5066},[467,468],[169,5068,5070,5084],{"className":5069},[472],[169,5071,5073],{"className":5072,"style":477},[476],[169,5074,5075,5078],{"style":480},[169,5076],{"className":5077,"style":485},[484],[169,5079,5081],{"className":5080},[489,490,491,492],[169,5082,421],{"className":5083},[212,213,492],[169,5085,500],{"className":5086},[499],[169,5088,5090],{"className":5089},[472],[169,5091,5093],{"className":5092,"style":507},[476],[169,5094],{},"。它可以是确定型的：",[169,5097,5099],{"className":5098},[400],[169,5100,5102,5149],{"className":5101},[172],[169,5103,5105],{"className":5104},[176],[178,5106,5107],{"xmlns":180,"display":409},[182,5108,5109,5146],{},[185,5110,5111,5123,5125,5128,5130,5136,5138,5144],{},[415,5112,5113,5115],{},[188,5114,190],{},[185,5116,5117,5119,5121],{},[188,5118,421],{},[423,5120,2878],{},[2781,5122,3452],{},[423,5124,425],{},[188,5126,5127],{},"f",[423,5129,431],{"stretchy":430},[415,5131,5132,5134],{},[188,5133,190],{},[188,5135,421],{},[423,5137,2505],{"separator":199},[415,5139,5140,5142],{},[188,5141,18],{},[188,5143,421],{},[423,5145,440],{"stretchy":430},[192,5147,5148],{"encoding":194},"s_{t+1} = f(s_t, a_t)",[169,5150,5152,5216],{"className":5151,"ariaHidden":199},[198],[169,5153,5155,5158,5207,5210,5213],{"className":5154},[203],[169,5156],{"className":5157,"style":4973},[207],[169,5159,5161,5164],{"className":5160},[212],[169,5162,190],{"className":5163},[212,213],[169,5165,5167],{"className":5166},[463],[169,5168,5170,5199],{"className":5169},[467,468],[169,5171,5173,5196],{"className":5172},[472],[169,5174,5176],{"className":5175,"style":4305},[476],[169,5177,5178,5181],{"style":480},[169,5179],{"className":5180,"style":485},[484],[169,5182,5184],{"className":5183},[489,490,491,492],[169,5185,5187,5190,5193],{"className":5186},[212,492],[169,5188,421],{"className":5189},[212,213,492],[169,5191,2878],{"className":5192},[2985,492],[169,5194,3452],{"className":5195},[212,492],[169,5197,500],{"className":5198},[499],[169,5200,5202],{"className":5201},[472],[169,5203,5205],{"className":5204,"style":5021},[476],[169,5206],{},[169,5208],{"className":5209,"style":514},[513],[169,5211,425],{"className":5212},[518],[169,5214],{"className":5215,"style":514},[513],[169,5217,5219,5222,5226,5229,5269,5272,5275,5315],{"className":5218},[203],[169,5220],{"className":5221,"style":528},[207],[169,5223,5127],{"className":5224,"style":5225},[212,213],"margin-right:0.1076em;",[169,5227,431],{"className":5228},[535],[169,5230,5232,5235],{"className":5231},[212],[169,5233,190],{"className":5234},[212,213],[169,5236,5238],{"className":5237},[463],[169,5239,5241,5261],{"className":5240},[467,468],[169,5242,5244,5258],{"className":5243},[472],[169,5245,5247],{"className":5246,"style":477},[476],[169,5248,5249,5252],{"style":480},[169,5250],{"className":5251,"style":485},[484],[169,5253,5255],{"className":5254},[489,490,491,492],[169,5256,421],{"className":5257},[212,213,492],[169,5259,500],{"className":5260},[499],[169,5262,5264],{"className":5263},[472],[169,5265,5267],{"className":5266,"style":507},[476],[169,5268],{},[169,5270,2505],{"className":5271},[2534],[169,5273],{"className":5274,"style":1622},[513],[169,5276,5278,5281],{"className":5277},[212],[169,5279,18],{"className":5280},[212,213],[169,5282,5284],{"className":5283},[463],[169,5285,5287,5307],{"className":5286},[467,468],[169,5288,5290,5304],{"className":5289},[472],[169,5291,5293],{"className":5292,"style":477},[476],[169,5294,5295,5298],{"style":480},[169,5296],{"className":5297,"style":485},[484],[169,5299,5301],{"className":5300},[489,490,491,492],[169,5302,421],{"className":5303},[212,213,492],[169,5305,500],{"className":5306},[499],[169,5308,5310],{"className":5309},[472],[169,5311,5313],{"className":5312,"style":507},[476],[169,5314],{},[169,5316,440],{"className":5317},[579],[14,5319,5320],{},"也可以是随机型的：",[169,5322,5324],{"className":5323},[400],[169,5325,5327,5377],{"className":5326},[172],[169,5328,5330],{"className":5329},[176],[178,5331,5332],{"xmlns":180,"display":409},[182,5333,5334,5374],{},[185,5335,5336,5348,5350,5352,5354,5356,5358,5364,5366,5372],{},[415,5337,5338,5340],{},[188,5339,190],{},[185,5341,5342,5344,5346],{},[188,5343,421],{},[423,5345,2878],{},[2781,5347,3452],{},[423,5349,637],{},[188,5351,1744],{},[423,5353,431],{"stretchy":430},[423,5355,644],{},[188,5357,648],{"mathvariant":647},[415,5359,5360,5362],{},[188,5361,190],{},[188,5363,421],{},[423,5365,2505],{"separator":199},[415,5367,5368,5370],{},[188,5369,18],{},[188,5371,421],{},[423,5373,440],{"stretchy":430},[192,5375,5376],{"encoding":194},"s_{t+1} \\sim P(\\cdot|s_t, a_t)",[169,5378,5380,5444],{"className":5379,"ariaHidden":199},[198],[169,5381,5383,5386,5435,5438,5441],{"className":5382},[203],[169,5384],{"className":5385,"style":4973},[207],[169,5387,5389,5392],{"className":5388},[212],[169,5390,190],{"className":5391},[212,213],[169,5393,5395],{"className":5394},[463],[169,5396,5398,5427],{"className":5397},[467,468],[169,5399,5401,5424],{"className":5400},[472],[169,5402,5404],{"className":5403,"style":4305},[476],[169,5405,5406,5409],{"style":480},[169,5407],{"className":5408,"style":485},[484],[169,5410,5412],{"className":5411},[489,490,491,492],[169,5413,5415,5418,5421],{"className":5414},[212,492],[169,5416,421],{"className":5417},[212,213,492],[169,5419,2878],{"className":5420},[2985,492],[169,5422,3452],{"className":5423},[212,492],[169,5425,500],{"className":5426},[499],[169,5428,5430],{"className":5429},[472],[169,5431,5433],{"className":5432,"style":5021},[476],[169,5434],{},[169,5436],{"className":5437,"style":514},[513],[169,5439,637],{"className":5440},[518],[169,5442],{"className":5443,"style":514},[513],[169,5445,5447,5450,5453,5456,5459,5462,5502,5505,5508,5548],{"className":5446},[203],[169,5448],{"className":5449,"style":528},[207],[169,5451,1744],{"className":5452,"style":1771},[212,213],[169,5454,431],{"className":5455},[535],[169,5457,644],{"className":5458},[212],[169,5460,648],{"className":5461},[212],[169,5463,5465,5468],{"className":5464},[212],[169,5466,190],{"className":5467},[212,213],[169,5469,5471],{"className":5470},[463],[169,5472,5474,5494],{"className":5473},[467,468],[169,5475,5477,5491],{"className":5476},[472],[169,5478,5480],{"className":5479,"style":477},[476],[169,5481,5482,5485],{"style":480},[169,5483],{"className":5484,"style":485},[484],[169,5486,5488],{"className":5487},[489,490,491,492],[169,5489,421],{"className":5490},[212,213,492],[169,5492,500],{"className":5493},[499],[169,5495,5497],{"className":5496},[472],[169,5498,5500],{"className":5499,"style":507},[476],[169,5501],{},[169,5503,2505],{"className":5504},[2534],[169,5506],{"className":5507,"style":1622},[513],[169,5509,5511,5514],{"className":5510},[212],[169,5512,18],{"className":5513},[212,213],[169,5515,5517],{"className":5516},[463],[169,5518,5520,5540],{"className":5519},[467,468],[169,5521,5523,5537],{"className":5522},[472],[169,5524,5526],{"className":5525,"style":477},[476],[169,5527,5528,5531],{"style":480},[169,5529],{"className":5530,"style":485},[484],[169,5532,5534],{"className":5533},[489,490,491,492],[169,5535,421],{"className":5536},[212,213,492],[169,5538,500],{"className":5539},[499],[169,5541,5543],{"className":5542},[472],[169,5544,5546],{"className":5545,"style":507},[476],[169,5547],{},[169,5549,440],{"className":5550},[579],[14,5552,5553],{},"动作则由智能体依照其策略产生。",[10,5555,5556],{"title":264},[14,5557,5558,5559,5562],{},"轨迹也常被称为",[117,5560,5561],{},"回合（episode）"," 或 rollout。",[159,5564,5565],{"id":5565},"奖励与回报",[14,5567,5568,5569,5599],{},"奖励函数 ",[169,5570,5572,5586],{"className":5571},[172],[169,5573,5575],{"className":5574},[176],[178,5576,5577],{"xmlns":180},[182,5578,5579,5584],{},[185,5580,5581],{},[188,5582,5583],{},"R",[192,5585,5583],{"encoding":194},[169,5587,5589],{"className":5588,"ariaHidden":199},[198],[169,5590,5592,5595],{"className":5591},[203],[169,5593],{"className":5594,"style":2182},[207],[169,5596,5583],{"className":5597,"style":5598},[212,213],"margin-right:0.0077em;"," 在强化学习中至关重要。它依赖当前的世界状态、刚采取的动作和下一个世界状态：",[169,5601,5603],{"className":5602},[400],[169,5604,5606,5661],{"className":5605},[172],[169,5607,5609],{"className":5608},[176],[178,5610,5611],{"xmlns":180,"display":409},[182,5612,5613,5658],{},[185,5614,5615,5622,5624,5626,5628,5634,5636,5642,5644,5656],{},[415,5616,5617,5620],{},[188,5618,5619],{},"r",[188,5621,421],{},[423,5623,425],{},[188,5625,5583],{},[423,5627,431],{"stretchy":430},[415,5629,5630,5632],{},[188,5631,190],{},[188,5633,421],{},[423,5635,2505],{"separator":199},[415,5637,5638,5640],{},[188,5639,18],{},[188,5641,421],{},[423,5643,2505],{"separator":199},[415,5645,5646,5648],{},[188,5647,190],{},[185,5649,5650,5652,5654],{},[188,5651,421],{},[423,5653,2878],{},[2781,5655,3452],{},[423,5657,440],{"stretchy":430},[192,5659,5660],{"encoding":194},"r_t = R(s_t, a_t, s_{t+1})",[169,5662,5664,5720],{"className":5663,"ariaHidden":199},[198],[169,5665,5667,5670,5711,5714,5717],{"className":5666},[203],[169,5668],{"className":5669,"style":453},[207],[169,5671,5673,5676],{"className":5672},[212],[169,5674,5619],{"className":5675,"style":823},[212,213],[169,5677,5679],{"className":5678},[463],[169,5680,5682,5703],{"className":5681},[467,468],[169,5683,5685,5700],{"className":5684},[472],[169,5686,5688],{"className":5687,"style":477},[476],[169,5689,5691,5694],{"style":5690},"top:-2.55em;margin-left:-0.0278em;margin-right:0.05em;",[169,5692],{"className":5693,"style":485},[484],[169,5695,5697],{"className":5696},[489,490,491,492],[169,5698,421],{"className":5699},[212,213,492],[169,5701,500],{"className":5702},[499],[169,5704,5706],{"className":5705},[472],[169,5707,5709],{"className":5708,"style":507},[476],[169,5710],{},[169,5712],{"className":5713,"style":514},[513],[169,5715,425],{"className":5716},[518],[169,5718],{"className":5719,"style":514},[513],[169,5721,5723,5726,5729,5732,5772,5775,5778,5818,5821,5824,5873],{"className":5722},[203],[169,5724],{"className":5725,"style":528},[207],[169,5727,5583],{"className":5728,"style":5598},[212,213],[169,5730,431],{"className":5731},[535],[169,5733,5735,5738],{"className":5734},[212],[169,5736,190],{"className":5737},[212,213],[169,5739,5741],{"className":5740},[463],[169,5742,5744,5764],{"className":5743},[467,468],[169,5745,5747,5761],{"className":5746},[472],[169,5748,5750],{"className":5749,"style":477},[476],[169,5751,5752,5755],{"style":480},[169,5753],{"className":5754,"style":485},[484],[169,5756,5758],{"className":5757},[489,490,491,492],[169,5759,421],{"className":5760},[212,213,492],[169,5762,500],{"className":5763},[499],[169,5765,5767],{"className":5766},[472],[169,5768,5770],{"className":5769,"style":507},[476],[169,5771],{},[169,5773,2505],{"className":5774},[2534],[169,5776],{"className":5777,"style":1622},[513],[169,5779,5781,5784],{"className":5780},[212],[169,5782,18],{"className":5783},[212,213],[169,5785,5787],{"className":5786},[463],[169,5788,5790,5810],{"className":5789},[467,468],[169,5791,5793,5807],{"className":5792},[472],[169,5794,5796],{"className":5795,"style":477},[476],[169,5797,5798,5801],{"style":480},[169,5799],{"className":5800,"style":485},[484],[169,5802,5804],{"className":5803},[489,490,491,492],[169,5805,421],{"className":5806},[212,213,492],[169,5808,500],{"className":5809},[499],[169,5811,5813],{"className":5812},[472],[169,5814,5816],{"className":5815,"style":507},[476],[169,5817],{},[169,5819,2505],{"className":5820},[2534],[169,5822],{"className":5823,"style":1622},[513],[169,5825,5827,5830],{"className":5826},[212],[169,5828,190],{"className":5829},[212,213],[169,5831,5833],{"className":5832},[463],[169,5834,5836,5865],{"className":5835},[467,468],[169,5837,5839,5862],{"className":5838},[472],[169,5840,5842],{"className":5841,"style":4305},[476],[169,5843,5844,5847],{"style":480},[169,5845],{"className":5846,"style":485},[484],[169,5848,5850],{"className":5849},[489,490,491,492],[169,5851,5853,5856,5859],{"className":5852},[212,492],[169,5854,421],{"className":5855},[212,213,492],[169,5857,2878],{"className":5858},[2985,492],[169,5860,3452],{"className":5861},[212,492],[169,5863,500],{"className":5864},[499],[169,5866,5868],{"className":5867},[472],[169,5869,5871],{"className":5870,"style":5021},[476],[169,5872],{},[169,5874,440],{"className":5875},[579],[14,5877,5878,5879,6027,6028,74],{},"不过它经常被简化成只依赖当前状态 ",[169,5880,5882,5914],{"className":5881},[172],[169,5883,5885],{"className":5884},[176],[178,5886,5887],{"xmlns":180},[182,5888,5889,5911],{},[185,5890,5891,5897,5899,5901,5903,5909],{},[415,5892,5893,5895],{},[188,5894,5619],{},[188,5896,421],{},[423,5898,425],{},[188,5900,5583],{},[423,5902,431],{"stretchy":430},[415,5904,5905,5907],{},[188,5906,190],{},[188,5908,421],{},[423,5910,440],{"stretchy":430},[192,5912,5913],{"encoding":194},"r_t = R(s_t)",[169,5915,5917,5972],{"className":5916,"ariaHidden":199},[198],[169,5918,5920,5923,5963,5966,5969],{"className":5919},[203],[169,5921],{"className":5922,"style":453},[207],[169,5924,5926,5929],{"className":5925},[212],[169,5927,5619],{"className":5928,"style":823},[212,213],[169,5930,5932],{"className":5931},[463],[169,5933,5935,5955],{"className":5934},[467,468],[169,5936,5938,5952],{"className":5937},[472],[169,5939,5941],{"className":5940,"style":477},[476],[169,5942,5943,5946],{"style":5690},[169,5944],{"className":5945,"style":485},[484],[169,5947,5949],{"className":5948},[489,490,491,492],[169,5950,421],{"className":5951},[212,213,492],[169,5953,500],{"className":5954},[499],[169,5956,5958],{"className":5957},[472],[169,5959,5961],{"className":5960,"style":507},[476],[169,5962],{},[169,5964],{"className":5965,"style":514},[513],[169,5967,425],{"className":5968},[518],[169,5970],{"className":5971,"style":514},[513],[169,5973,5975,5978,5981,5984,6024],{"className":5974},[203],[169,5976],{"className":5977,"style":528},[207],[169,5979,5583],{"className":5980,"style":5598},[212,213],[169,5982,431],{"className":5983},[535],[169,5985,5987,5990],{"className":5986},[212],[169,5988,190],{"className":5989},[212,213],[169,5991,5993],{"className":5992},[463],[169,5994,5996,6016],{"className":5995},[467,468],[169,5997,5999,6013],{"className":5998},[472],[169,6000,6002],{"className":6001,"style":477},[476],[169,6003,6004,6007],{"style":480},[169,6005],{"className":6006,"style":485},[484],[169,6008,6010],{"className":6009},[489,490,491,492],[169,6011,421],{"className":6012},[212,213,492],[169,6014,500],{"className":6015},[499],[169,6017,6019],{"className":6018},[472],[169,6020,6022],{"className":6021,"style":507},[476],[169,6023],{},[169,6025,440],{"className":6026},[579],"，或依赖状态-动作对 ",[169,6029,6031,6071],{"className":6030},[172],[169,6032,6034],{"className":6033},[176],[178,6035,6036],{"xmlns":180},[182,6037,6038,6068],{},[185,6039,6040,6046,6048,6050,6052,6058,6060,6066],{},[415,6041,6042,6044],{},[188,6043,5619],{},[188,6045,421],{},[423,6047,425],{},[188,6049,5583],{},[423,6051,431],{"stretchy":430},[415,6053,6054,6056],{},[188,6055,190],{},[188,6057,421],{},[423,6059,2505],{"separator":199},[415,6061,6062,6064],{},[188,6063,18],{},[188,6065,421],{},[423,6067,440],{"stretchy":430},[192,6069,6070],{"encoding":194},"r_t = R(s_t, a_t)",[169,6072,6074,6129],{"className":6073,"ariaHidden":199},[198],[169,6075,6077,6080,6120,6123,6126],{"className":6076},[203],[169,6078],{"className":6079,"style":453},[207],[169,6081,6083,6086],{"className":6082},[212],[169,6084,5619],{"className":6085,"style":823},[212,213],[169,6087,6089],{"className":6088},[463],[169,6090,6092,6112],{"className":6091},[467,468],[169,6093,6095,6109],{"className":6094},[472],[169,6096,6098],{"className":6097,"style":477},[476],[169,6099,6100,6103],{"style":5690},[169,6101],{"className":6102,"style":485},[484],[169,6104,6106],{"className":6105},[489,490,491,492],[169,6107,421],{"className":6108},[212,213,492],[169,6110,500],{"className":6111},[499],[169,6113,6115],{"className":6114},[472],[169,6116,6118],{"className":6117,"style":507},[476],[169,6119],{},[169,6121],{"className":6122,"style":514},[513],[169,6124,425],{"className":6125},[518],[169,6127],{"className":6128,"style":514},[513],[169,6130,6132,6135,6138,6141,6181,6184,6187,6227],{"className":6131},[203],[169,6133],{"className":6134,"style":528},[207],[169,6136,5583],{"className":6137,"style":5598},[212,213],[169,6139,431],{"className":6140},[535],[169,6142,6144,6147],{"className":6143},[212],[169,6145,190],{"className":6146},[212,213],[169,6148,6150],{"className":6149},[463],[169,6151,6153,6173],{"className":6152},[467,468],[169,6154,6156,6170],{"className":6155},[472],[169,6157,6159],{"className":6158,"style":477},[476],[169,6160,6161,6164],{"style":480},[169,6162],{"className":6163,"style":485},[484],[169,6165,6167],{"className":6166},[489,490,491,492],[169,6168,421],{"className":6169},[212,213,492],[169,6171,500],{"className":6172},[499],[169,6174,6176],{"className":6175},[472],[169,6177,6179],{"className":6178,"style":507},[476],[169,6180],{},[169,6182,2505],{"className":6183},[2534],[169,6185],{"className":6186,"style":1622},[513],[169,6188,6190,6193],{"className":6189},[212],[169,6191,18],{"className":6192},[212,213],[169,6194,6196],{"className":6195},[463],[169,6197,6199,6219],{"className":6198},[467,468],[169,6200,6202,6216],{"className":6201},[472],[169,6203,6205],{"className":6204,"style":477},[476],[169,6206,6207,6210],{"style":480},[169,6208],{"className":6209,"style":485},[484],[169,6211,6213],{"className":6212},[489,490,491,492],[169,6214,421],{"className":6215},[212,213,492],[169,6217,500],{"className":6218},[499],[169,6220,6222],{"className":6221},[472],[169,6223,6225],{"className":6224,"style":507},[476],[169,6226],{},[169,6228,440],{"className":6229},[579],[14,6231,6232,6233,6277],{},"智能体的目标是最大化某种「沿轨迹累积的奖励」，但这话实际可以有几种含义。我们对所有这些情形统一记作 ",[169,6234,6236,6256],{"className":6235},[172],[169,6237,6239],{"className":6238},[176],[178,6240,6241],{"xmlns":180},[182,6242,6243,6253],{},[185,6244,6245,6247,6249,6251],{},[188,6246,5583],{},[423,6248,431],{"stretchy":430},[188,6250,4174],{},[423,6252,440],{"stretchy":430},[192,6254,6255],{"encoding":194},"R(\\tau)",[169,6257,6259],{"className":6258,"ariaHidden":199},[198],[169,6260,6262,6265,6268,6271,6274],{"className":6261},[203],[169,6263],{"className":6264,"style":528},[207],[169,6266,5583],{"className":6267,"style":5598},[212,213],[169,6269,431],{"className":6270},[535],[169,6272,4174],{"className":6273,"style":4190},[212,213],[169,6275,440],{"className":6276},[579],"：具体指哪一种，要么能从上下文分辨，要么无关紧要（因为同样的方程适用于所有情形）。",[14,6279,6280,6281,6284],{},"一种回报是",[117,6282,6283],{},"有限视界不折扣回报（finite-horizon undiscounted return）","：固定步数窗口内获得的奖励之和：",[169,6286,6288],{"className":6287},[400],[169,6289,6291,6334],{"className":6290},[172],[169,6292,6294],{"className":6293},[176],[178,6295,6296],{"xmlns":180,"display":409},[182,6297,6298,6331],{},[185,6299,6300,6302,6304,6306,6308,6310,6325],{},[188,6301,5583],{},[423,6303,431],{"stretchy":430},[188,6305,4174],{},[423,6307,440],{"stretchy":430},[423,6309,425],{},[3461,6311,6312,6314,6322],{},[423,6313,3465],{},[185,6315,6316,6318,6320],{},[188,6317,421],{},[423,6319,425],{},[2781,6321,885],{},[188,6323,6324],{},"T",[415,6326,6327,6329],{},[188,6328,5619],{},[188,6330,421],{},[192,6332,6333],{"encoding":194},"R(\\tau) = \\sum_{t=0}^T r_t",[169,6335,6337,6364],{"className":6336,"ariaHidden":199},[198],[169,6338,6340,6343,6346,6349,6352,6355,6358,6361],{"className":6339},[203],[169,6341],{"className":6342,"style":528},[207],[169,6344,5583],{"className":6345,"style":5598},[212,213],[169,6347,431],{"className":6348},[535],[169,6350,4174],{"className":6351,"style":4190},[212,213],[169,6353,440],{"className":6354},[579],[169,6356],{"className":6357,"style":514},[513],[169,6359,425],{"className":6360},[518],[169,6362],{"className":6363,"style":514},[513],[169,6365,6367,6371,6438,6441],{"className":6366},[203],[169,6368],{"className":6369,"style":6370},[207],"height:3.0954em;vertical-align:-1.2671em;",[169,6372,6374],{"className":6373},[1483,3731],[169,6375,6377,6429],{"className":6376},[467,468],[169,6378,6380,6426],{"className":6379},[472],[169,6381,6384,6405,6415],{"className":6382,"style":6383},[476],"height:1.8283em;",[169,6385,6387,6390],{"style":6386},"top:-1.8829em;margin-left:0em;",[169,6388],{"className":6389,"style":3748},[484],[169,6391,6393],{"className":6392},[489,490,491,492],[169,6394,6396,6399,6402],{"className":6395},[212,492],[169,6397,421],{"className":6398},[212,213,492],[169,6400,425],{"className":6401},[518,492],[169,6403,885],{"className":6404},[212,492],[169,6406,6407,6410],{"style":3766},[169,6408],{"className":6409,"style":3748},[484],[169,6411,6412],{},[169,6413,3465],{"className":6414},[1483,3775,3776],[169,6416,6417,6420],{"style":3779},[169,6418],{"className":6419,"style":3748},[484],[169,6421,6423],{"className":6422},[489,490,491,492],[169,6424,6324],{"className":6425,"style":1771},[212,213,492],[169,6427,500],{"className":6428},[499],[169,6430,6432],{"className":6431},[472],[169,6433,6436],{"className":6434,"style":6435},[476],"height:1.2671em;",[169,6437],{},[169,6439],{"className":6440,"style":1622},[513],[169,6442,6444,6447],{"className":6443},[212],[169,6445,5619],{"className":6446,"style":823},[212,213],[169,6448,6450],{"className":6449},[463],[169,6451,6453,6473],{"className":6452},[467,468],[169,6454,6456,6470],{"className":6455},[472],[169,6457,6459],{"className":6458,"style":477},[476],[169,6460,6461,6464],{"style":5690},[169,6462],{"className":6463,"style":485},[484],[169,6465,6467],{"className":6466},[489,490,491,492],[169,6468,421],{"className":6469},[212,213,492],[169,6471,500],{"className":6472},[499],[169,6474,6476],{"className":6475},[472],[169,6477,6479],{"className":6478,"style":507},[476],[169,6480],{},[14,6482,6483,6484,6487,6488,6491,6492,396],{},"另一种是",[117,6485,6486],{},"无限视界折扣回报（infinite-horizon discounted return）","：智能体",[45,6489,6490],{},"曾经","获得的全部奖励之和，但按奖励到来时刻的远近打折。这种回报形式引入了折扣因子 ",[169,6493,6495,6523],{"className":6494},[172],[169,6496,6498],{"className":6497},[176],[178,6499,6500],{"xmlns":180},[182,6501,6502,6520],{},[185,6503,6504,6507,6510,6512,6514,6516,6518],{},[188,6505,6506],{},"γ",[423,6508,6509],{},"∈",[423,6511,431],{"stretchy":430},[2781,6513,885],{},[423,6515,2505],{"separator":199},[2781,6517,3452],{},[423,6519,440],{"stretchy":430},[192,6521,6522],{"encoding":194},"\\gamma \\in (0,1)",[169,6524,6526,6546],{"className":6525,"ariaHidden":199},[198],[169,6527,6529,6533,6537,6540,6543],{"className":6528},[203],[169,6530],{"className":6531,"style":6532},[207],"height:0.7335em;vertical-align:-0.1944em;",[169,6534,6506],{"className":6535,"style":6536},[212,213],"margin-right:0.0556em;",[169,6538],{"className":6539,"style":514},[513],[169,6541,6509],{"className":6542},[518],[169,6544],{"className":6545,"style":514},[513],[169,6547,6549,6552,6555,6558,6561,6564,6567],{"className":6548},[203],[169,6550],{"className":6551,"style":528},[207],[169,6553,431],{"className":6554},[535],[169,6556,885],{"className":6557},[212],[169,6559,2505],{"className":6560},[2534],[169,6562],{"className":6563,"style":1622},[513],[169,6565,3452],{"className":6566},[212],[169,6568,440],{"className":6569},[579],[169,6571,6573],{"className":6572},[400],[169,6574,6576,6624],{"className":6575},[172],[169,6577,6579],{"className":6578},[176],[178,6580,6581],{"xmlns":180,"display":409},[182,6582,6583,6621],{},[185,6584,6585,6587,6589,6591,6593,6595,6609,6615],{},[188,6586,5583],{},[423,6588,431],{"stretchy":430},[188,6590,4174],{},[423,6592,440],{"stretchy":430},[423,6594,425],{},[3461,6596,6597,6599,6607],{},[423,6598,3465],{},[185,6600,6601,6603,6605],{},[188,6602,421],{},[423,6604,425],{},[2781,6606,885],{},[188,6608,2502],{"mathvariant":647},[3502,6610,6611,6613],{},[188,6612,6506],{},[188,6614,421],{},[415,6616,6617,6619],{},[188,6618,5619],{},[188,6620,421],{},[192,6622,6623],{"encoding":194},"R(\\tau) = \\sum_{t=0}^{\\infty} \\gamma^t r_t",[169,6625,6627,6654],{"className":6626,"ariaHidden":199},[198],[169,6628,6630,6633,6636,6639,6642,6645,6648,6651],{"className":6629},[203],[169,6631],{"className":6632,"style":528},[207],[169,6634,5583],{"className":6635,"style":5598},[212,213],[169,6637,431],{"className":6638},[535],[169,6640,4174],{"className":6641,"style":4190},[212,213],[169,6643,440],{"className":6644},[579],[169,6646],{"className":6647,"style":514},[513],[169,6649,425],{"className":6650},[518],[169,6652],{"className":6653,"style":514},[513],[169,6655,6657,6661,6729,6732,6763],{"className":6656},[203],[169,6658],{"className":6659,"style":6660},[207],"height:2.9185em;vertical-align:-1.2671em;",[169,6662,6664],{"className":6663},[1483,3731],[169,6665,6667,6721],{"className":6666},[467,468],[169,6668,6670,6718],{"className":6669},[472],[169,6671,6674,6694,6704],{"className":6672,"style":6673},[476],"height:1.6514em;",[169,6675,6676,6679],{"style":6386},[169,6677],{"className":6678,"style":3748},[484],[169,6680,6682],{"className":6681},[489,490,491,492],[169,6683,6685,6688,6691],{"className":6684},[212,492],[169,6686,421],{"className":6687},[212,213,492],[169,6689,425],{"className":6690},[518,492],[169,6692,885],{"className":6693},[212,492],[169,6695,6696,6699],{"style":3766},[169,6697],{"className":6698,"style":3748},[484],[169,6700,6701],{},[169,6702,3465],{"className":6703},[1483,3775,3776],[169,6705,6706,6709],{"style":3779},[169,6707],{"className":6708,"style":3748},[484],[169,6710,6712],{"className":6711},[489,490,491,492],[169,6713,6715],{"className":6714},[212,492],[169,6716,2502],{"className":6717},[212,492],[169,6719,500],{"className":6720},[499],[169,6722,6724],{"className":6723},[472],[169,6725,6727],{"className":6726,"style":6435},[476],[169,6728],{},[169,6730],{"className":6731,"style":1622},[513],[169,6733,6735,6738],{"className":6734},[212],[169,6736,6506],{"className":6737,"style":6536},[212,213],[169,6739,6741],{"className":6740},[463],[169,6742,6744],{"className":6743},[467],[169,6745,6747],{"className":6746},[472],[169,6748,6751],{"className":6749,"style":6750},[476],"height:0.8436em;",[169,6752,6754,6757],{"style":6753},"top:-3.113em;margin-right:0.05em;",[169,6755],{"className":6756,"style":485},[484],[169,6758,6760],{"className":6759},[489,490,491,492],[169,6761,421],{"className":6762},[212,213,492],[169,6764,6766,6769],{"className":6765},[212],[169,6767,5619],{"className":6768,"style":823},[212,213],[169,6770,6772],{"className":6771},[463],[169,6773,6775,6795],{"className":6774},[467,468],[169,6776,6778,6792],{"className":6777},[472],[169,6779,6781],{"className":6780,"style":477},[476],[169,6782,6783,6786],{"style":5690},[169,6784],{"className":6785,"style":485},[484],[169,6787,6789],{"className":6788},[489,490,491,492],[169,6790,421],{"className":6791},[212,213,492],[169,6793,500],{"className":6794},[499],[169,6796,6798],{"className":6797},[472],[169,6799,6801],{"className":6800,"style":507},[476],[169,6802],{},[14,6804,6805,6806,6809,6810,6815],{},"可我们为什么会想要折扣因子？难道不就是把",[45,6807,6808],{},"所有","奖励都拿到手吗？确实想，但折扣因子既直观又便于数学处理。直观层面：现在的现金好过以后的现金。数学层面：无限视界的奖励和",[18,6811,6814],{"href":6812,"rel":6813},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FConvergent_series",[22],"可能不收敛","到有限值，在方程里难以处理；而有了折扣因子，再加上合理条件，无穷和就收敛了。",[10,6817,6818],{"title":264},[14,6819,6820,6821,6824],{},"这两种回报形式在 RL 形式化里界限分明，但深度 RL 的实践常把界线弄得相当模糊：比如我们经常把算法设定为优化不折扣的回报，却在估计",[117,6822,6823],{},"价值函数","时使用折扣因子。",[159,6826,6828],{"id":6827},"rl-问题","RL 问题",[14,6830,6831,6832,6835],{},"无论选哪种回报度量（无限视界折扣，还是有限视界不折扣），也无论策略怎么选，RL 的目标都是：挑出一个策略，使智能体按它行动时的",[117,6833,6834],{},"期望回报（expected return）"," 最大。",[14,6837,6838],{},"要谈期望回报，得先谈轨迹上的概率分布。",[14,6840,6841,6842,6870],{},"假设环境转移和策略都是随机型的。这时，一条 ",[169,6843,6845,6858],{"className":6844},[172],[169,6846,6848],{"className":6847},[176],[178,6849,6850],{"xmlns":180},[182,6851,6852,6856],{},[185,6853,6854],{},[188,6855,6324],{},[192,6857,6324],{"encoding":194},[169,6859,6861],{"className":6860,"ariaHidden":199},[198],[169,6862,6864,6867],{"className":6863},[203],[169,6865],{"className":6866,"style":2182},[207],[169,6868,6324],{"className":6869,"style":1771},[212,213]," 步轨迹的概率是：",[169,6872,6874],{"className":6873},[400],[169,6875,6877,6994],{"className":6876},[172],[169,6878,6880],{"className":6879},[176],[178,6881,6882],{"xmlns":180,"display":409},[182,6883,6884,6991],{},[185,6885,6886,6888,6890,6892,6894,6896,6898,6900,6906,6908,6914,6916,6937,6939,6941,6953,6955,6961,6963,6969,6971,6973,6975,6981,6983,6989],{},[188,6887,1744],{},[423,6889,431],{"stretchy":430},[188,6891,4174],{},[188,6893,648],{"mathvariant":647},[188,6895,597],{},[423,6897,440],{"stretchy":430},[423,6899,425],{},[415,6901,6902,6904],{},[188,6903,4573],{},[2781,6905,885],{},[423,6907,431],{"stretchy":430},[415,6909,6910,6912],{},[188,6911,190],{},[2781,6913,885],{},[423,6915,440],{"stretchy":430},[3461,6917,6918,6921,6929],{},[423,6919,6920],{},"∏",[185,6922,6923,6925,6927],{},[188,6924,421],{},[423,6926,425],{},[2781,6928,885],{},[185,6930,6931,6933,6935],{},[188,6932,6324],{},[423,6934,2499],{},[2781,6936,3452],{},[188,6938,1744],{},[423,6940,431],{"stretchy":430},[415,6942,6943,6945],{},[188,6944,190],{},[185,6946,6947,6949,6951],{},[188,6948,421],{},[423,6950,2878],{},[2781,6952,3452],{},[188,6954,648],{"mathvariant":647},[415,6956,6957,6959],{},[188,6958,190],{},[188,6960,421],{},[423,6962,2505],{"separator":199},[415,6964,6965,6967],{},[188,6966,18],{},[188,6968,421],{},[423,6970,440],{"stretchy":430},[188,6972,597],{},[423,6974,431],{"stretchy":430},[415,6976,6977,6979],{},[188,6978,18],{},[188,6980,421],{},[188,6982,648],{"mathvariant":647},[415,6984,6985,6987],{},[188,6986,190],{},[188,6988,421],{},[423,6990,440],{"stretchy":430},[192,6992,6993],{"encoding":194},"P(\\tau|\\pi) = \\rho_0 (s_0) \\prod_{t=0}^{T-1} P(s_{t+1} | s_t, a_t) \\pi(a_t | s_t)",[169,6995,6997,7030],{"className":6996,"ariaHidden":199},[198],[169,6998,7000,7003,7006,7009,7012,7015,7018,7021,7024,7027],{"className":6999},[203],[169,7001],{"className":7002,"style":528},[207],[169,7004,1744],{"className":7005,"style":1771},[212,213],[169,7007,431],{"className":7008},[535],[169,7010,4174],{"className":7011,"style":4190},[212,213],[169,7013,648],{"className":7014},[212],[169,7016,597],{"className":7017,"style":613},[212,213],[169,7019,440],{"className":7020},[579],[169,7022],{"className":7023,"style":514},[513],[169,7025,425],{"className":7026},[518],[169,7028],{"className":7029,"style":514},[513],[169,7031,7033,7036,7076,7079,7119,7122,7125,7198,7201,7204,7207,7256,7259,7299,7302,7305,7345,7348,7351,7354,7394,7397,7437],{"className":7032},[203],[169,7034],{"className":7035,"style":6370},[207],[169,7037,7039,7042],{"className":7038},[212],[169,7040,4573],{"className":7041},[212,213],[169,7043,7045],{"className":7044},[463],[169,7046,7048,7068],{"className":7047},[467,468],[169,7049,7051,7065],{"className":7050},[472],[169,7052,7054],{"className":7053,"style":4305},[476],[169,7055,7056,7059],{"style":480},[169,7057],{"className":7058,"style":485},[484],[169,7060,7062],{"className":7061},[489,490,491,492],[169,7063,885],{"className":7064},[212,492],[169,7066,500],{"className":7067},[499],[169,7069,7071],{"className":7070},[472],[169,7072,7074],{"className":7073,"style":507},[476],[169,7075],{},[169,7077,431],{"className":7078},[535],[169,7080,7082,7085],{"className":7081},[212],[169,7083,190],{"className":7084},[212,213],[169,7086,7088],{"className":7087},[463],[169,7089,7091,7111],{"className":7090},[467,468],[169,7092,7094,7108],{"className":7093},[472],[169,7095,7097],{"className":7096,"style":4305},[476],[169,7098,7099,7102],{"style":480},[169,7100],{"className":7101,"style":485},[484],[169,7103,7105],{"className":7104},[489,490,491,492],[169,7106,885],{"className":7107},[212,492],[169,7109,500],{"className":7110},[499],[169,7112,7114],{"className":7113},[472],[169,7115,7117],{"className":7116,"style":507},[476],[169,7118],{},[169,7120,440],{"className":7121},[579],[169,7123],{"className":7124,"style":1622},[513],[169,7126,7128],{"className":7127},[1483,3731],[169,7129,7131,7190],{"className":7130},[467,468],[169,7132,7134,7187],{"className":7133},[472],[169,7135,7137,7157,7167],{"className":7136,"style":6383},[476],[169,7138,7139,7142],{"style":6386},[169,7140],{"className":7141,"style":3748},[484],[169,7143,7145],{"className":7144},[489,490,491,492],[169,7146,7148,7151,7154],{"className":7147},[212,492],[169,7149,421],{"className":7150},[212,213,492],[169,7152,425],{"className":7153},[518,492],[169,7155,885],{"className":7156},[212,492],[169,7158,7159,7162],{"style":3766},[169,7160],{"className":7161,"style":3748},[484],[169,7163,7164],{},[169,7165,6920],{"className":7166},[1483,3775,3776],[169,7168,7169,7172],{"style":3779},[169,7170],{"className":7171,"style":3748},[484],[169,7173,7175],{"className":7174},[489,490,491,492],[169,7176,7178,7181,7184],{"className":7177},[212,492],[169,7179,6324],{"className":7180,"style":1771},[212,213,492],[169,7182,2499],{"className":7183},[2985,492],[169,7185,3452],{"className":7186},[212,492],[169,7188,500],{"className":7189},[499],[169,7191,7193],{"className":7192},[472],[169,7194,7196],{"className":7195,"style":6435},[476],[169,7197],{},[169,7199],{"className":7200,"style":1622},[513],[169,7202,1744],{"className":7203,"style":1771},[212,213],[169,7205,431],{"className":7206},[535],[169,7208,7210,7213],{"className":7209},[212],[169,7211,190],{"className":7212},[212,213],[169,7214,7216],{"className":7215},[463],[169,7217,7219,7248],{"className":7218},[467,468],[169,7220,7222,7245],{"className":7221},[472],[169,7223,7225],{"className":7224,"style":4305},[476],[169,7226,7227,7230],{"style":480},[169,7228],{"className":7229,"style":485},[484],[169,7231,7233],{"className":7232},[489,490,491,492],[169,7234,7236,7239,7242],{"className":7235},[212,492],[169,7237,421],{"className":7238},[212,213,492],[169,7240,2878],{"className":7241},[2985,492],[169,7243,3452],{"className":7244},[212,492],[169,7246,500],{"className":7247},[499],[169,7249,7251],{"className":7250},[472],[169,7252,7254],{"className":7253,"style":5021},[476],[169,7255],{},[169,7257,648],{"className":7258},[212],[169,7260,7262,7265],{"className":7261},[212],[169,7263,190],{"className":7264},[212,213],[169,7266,7268],{"className":7267},[463],[169,7269,7271,7291],{"className":7270},[467,468],[169,7272,7274,7288],{"className":7273},[472],[169,7275,7277],{"className":7276,"style":477},[476],[169,7278,7279,7282],{"style":480},[169,7280],{"className":7281,"style":485},[484],[169,7283,7285],{"className":7284},[489,490,491,492],[169,7286,421],{"className":7287},[212,213,492],[169,7289,500],{"className":7290},[499],[169,7292,7294],{"className":7293},[472],[169,7295,7297],{"className":7296,"style":507},[476],[169,7298],{},[169,7300,2505],{"className":7301},[2534],[169,7303],{"className":7304,"style":1622},[513],[169,7306,7308,7311],{"className":7307},[212],[169,7309,18],{"className":7310},[212,213],[169,7312,7314],{"className":7313},[463],[169,7315,7317,7337],{"className":7316},[467,468],[169,7318,7320,7334],{"className":7319},[472],[169,7321,7323],{"className":7322,"style":477},[476],[169,7324,7325,7328],{"style":480},[169,7326],{"className":7327,"style":485},[484],[169,7329,7331],{"className":7330},[489,490,491,492],[169,7332,421],{"className":7333},[212,213,492],[169,7335,500],{"className":7336},[499],[169,7338,7340],{"className":7339},[472],[169,7341,7343],{"className":7342,"style":507},[476],[169,7344],{},[169,7346,440],{"className":7347},[579],[169,7349,597],{"className":7350,"style":613},[212,213],[169,7352,431],{"className":7353},[535],[169,7355,7357,7360],{"className":7356},[212],[169,7358,18],{"className":7359},[212,213],[169,7361,7363],{"className":7362},[463],[169,7364,7366,7386],{"className":7365},[467,468],[169,7367,7369,7383],{"className":7368},[472],[169,7370,7372],{"className":7371,"style":477},[476],[169,7373,7374,7377],{"style":480},[169,7375],{"className":7376,"style":485},[484],[169,7378,7380],{"className":7379},[489,490,491,492],[169,7381,421],{"className":7382},[212,213,492],[169,7384,500],{"className":7385},[499],[169,7387,7389],{"className":7388},[472],[169,7390,7392],{"className":7391,"style":507},[476],[169,7393],{},[169,7395,648],{"className":7396},[212],[169,7398,7400,7403],{"className":7399},[212],[169,7401,190],{"className":7402},[212,213],[169,7404,7406],{"className":7405},[463],[169,7407,7409,7429],{"className":7408},[467,468],[169,7410,7412,7426],{"className":7411},[472],[169,7413,7415],{"className":7414,"style":477},[476],[169,7416,7417,7420],{"style":480},[169,7418],{"className":7419,"style":485},[484],[169,7421,7423],{"className":7422},[489,490,491,492],[169,7424,421],{"className":7425},[212,213,492],[169,7427,500],{"className":7428},[499],[169,7430,7432],{"className":7431},[472],[169,7433,7435],{"className":7434,"style":507},[476],[169,7436],{},[169,7438,440],{"className":7439},[579],[14,7441,7442,7443,396],{},"期望回报（无论按哪种度量）记作 ",[169,7444,7446,7467],{"className":7445},[172],[169,7447,7449],{"className":7448},[176],[178,7450,7451],{"xmlns":180},[182,7452,7453,7464],{},[185,7454,7455,7458,7460,7462],{},[188,7456,7457],{},"J",[423,7459,431],{"stretchy":430},[188,7461,597],{},[423,7463,440],{"stretchy":430},[192,7465,7466],{"encoding":194},"J(\\pi)",[169,7468,7470],{"className":7469,"ariaHidden":199},[198],[169,7471,7473,7476,7480,7483,7486],{"className":7472},[203],[169,7474],{"className":7475,"style":528},[207],[169,7477,7457],{"className":7478,"style":7479},[212,213],"margin-right:0.0962em;",[169,7481,431],{"className":7482},[535],[169,7484,597],{"className":7485,"style":613},[212,213],[169,7487,440],{"className":7488},[579],[169,7490,7492],{"className":7491},[400],[169,7493,7495,7577],{"className":7494},[172],[169,7496,7498],{"className":7497},[176],[178,7499,7500],{"xmlns":180,"display":409},[182,7501,7502,7574],{},[185,7503,7504,7506,7508,7510,7512,7514,7521,7523,7525,7527,7529,7531,7533,7535,7537,7539,7541,7543,7562,7564,7566,7568,7570,7572],{},[188,7505,7457],{},[423,7507,431],{"stretchy":430},[188,7509,597],{},[423,7511,440],{"stretchy":430},[423,7513,425],{},[415,7515,7516,7519],{},[423,7517,7518],{},"∫",[188,7520,4174],{},[188,7522,1744],{},[423,7524,431],{"stretchy":430},[188,7526,4174],{},[188,7528,648],{"mathvariant":647},[188,7530,597],{},[423,7532,440],{"stretchy":430},[188,7534,5583],{},[423,7536,431],{"stretchy":430},[188,7538,4174],{},[423,7540,440],{"stretchy":430},[423,7542,425],{},[188,7544,7545],{},[7546,7547,7548,7554],"munder",{},[423,7549,7550],{},[188,7551,7553],{"mathvariant":7552},"double-struck","E",[185,7555,7556,7558,7560],{},[188,7557,4174],{},[423,7559,637],{},[188,7561,597],{},[423,7563,1896],{"stretchy":430},[188,7565,5583],{},[423,7567,431],{"stretchy":430},[188,7569,4174],{},[423,7571,440],{"stretchy":430},[423,7573,1911],{"stretchy":430},[192,7575,7576],{"encoding":194},"J(\\pi) = \\int_{\\tau} P(\\tau|\\pi) R(\\tau) = \\underset{\\tau\\sim \\pi}{\\mathbb{E}}[R(\\tau)]",[169,7578,7580,7607,7703],{"className":7579,"ariaHidden":199},[198],[169,7581,7583,7586,7589,7592,7595,7598,7601,7604],{"className":7582},[203],[169,7584],{"className":7585,"style":528},[207],[169,7587,7457],{"className":7588,"style":7479},[212,213],[169,7590,431],{"className":7591},[535],[169,7593,597],{"className":7594,"style":613},[212,213],[169,7596,440],{"className":7597},[579],[169,7599],{"className":7600,"style":514},[513],[169,7602,425],{"className":7603},[518],[169,7605],{"className":7606,"style":514},[513],[169,7608,7610,7614,7661,7664,7667,7670,7673,7676,7679,7682,7685,7688,7691,7694,7697,7700],{"className":7609},[203],[169,7611],{"className":7612,"style":7613},[207],"height:2.2719em;vertical-align:-0.9119em;",[169,7615,7617,7621],{"className":7616},[1483],[169,7618,7518],{"className":7619,"style":7620},[1483,3775,3776],"margin-right:0.4445em;position:relative;top:-0.0011em;",[169,7622,7624],{"className":7623},[463],[169,7625,7627,7652],{"className":7626},[467,468],[169,7628,7630,7649],{"className":7629},[472],[169,7631,7634],{"className":7632,"style":7633},[476],"height:-0.6106em;",[169,7635,7637,7640],{"style":7636},"top:-1.7881em;margin-left:-0.4445em;margin-right:0.05em;",[169,7638],{"className":7639,"style":485},[484],[169,7641,7643],{"className":7642},[489,490,491,492],[169,7644,7646],{"className":7645},[212,492],[169,7647,4174],{"className":7648,"style":4190},[212,213,492],[169,7650,500],{"className":7651},[499],[169,7653,7655],{"className":7654},[472],[169,7656,7659],{"className":7657,"style":7658},[476],"height:0.9119em;",[169,7660],{},[169,7662],{"className":7663,"style":1622},[513],[169,7665,1744],{"className":7666,"style":1771},[212,213],[169,7668,431],{"className":7669},[535],[169,7671,4174],{"className":7672,"style":4190},[212,213],[169,7674,648],{"className":7675},[212],[169,7677,597],{"className":7678,"style":613},[212,213],[169,7680,440],{"className":7681},[579],[169,7683,5583],{"className":7684,"style":5598},[212,213],[169,7686,431],{"className":7687},[535],[169,7689,4174],{"className":7690,"style":4190},[212,213],[169,7692,440],{"className":7693},[579],[169,7695],{"className":7696,"style":514},[513],[169,7698,425],{"className":7699},[518],[169,7701],{"className":7702,"style":514},[513],[169,7704,7706,7710,7771,7774,7777,7780,7783],{"className":7705},[203],[169,7707],{"className":7708,"style":7709},[207],"height:1.45em;vertical-align:-0.7em;",[169,7711,7713],{"className":7712},[212],[169,7714,7716],{"className":7715},[1483,3731],[169,7717,7719,7762],{"className":7718},[467,468],[169,7720,7722,7759],{"className":7721},[472],[169,7723,7726,7747],{"className":7724,"style":7725},[476],"height:0.6889em;",[169,7727,7729,7732],{"style":7728},"top:-2.4em;margin-left:0em;",[169,7730],{"className":7731,"style":999},[484],[169,7733,7735],{"className":7734},[489,490,491,492],[169,7736,7738,7741,7744],{"className":7737},[212,492],[169,7739,4174],{"className":7740,"style":4190},[212,213,492],[169,7742,637],{"className":7743},[518,492],[169,7745,597],{"className":7746,"style":613},[212,213,492],[169,7748,7750,7753],{"style":7749},"top:-3em;",[169,7751],{"className":7752,"style":999},[484],[169,7754,7755],{},[169,7756,7553],{"className":7757},[1483,7758],"mathbb",[169,7760,500],{"className":7761},[499],[169,7763,7765],{"className":7764},[472],[169,7766,7769],{"className":7767,"style":7768},[476],"height:0.7em;",[169,7770],{},[169,7772,1896],{"className":7773},[535],[169,7775,5583],{"className":7776,"style":5598},[212,213],[169,7778,431],{"className":7779},[535],[169,7781,4174],{"className":7782,"style":4190},[212,213],[169,7784,7786],{"className":7785},[579],")]",[14,7788,7789],{},"RL 的中心优化问题于是可以写成：",[169,7791,7793],{"className":7792},[400],[169,7794,7796,7841],{"className":7795},[172],[169,7797,7799],{"className":7798},[176],[178,7800,7801],{"xmlns":180,"display":409},[182,7802,7803,7838],{},[185,7804,7805,7812,7814,7817,7819,7830,7832,7834,7836],{},[3502,7806,7807,7809],{},[188,7808,597],{},[423,7810,7811],{},"∗",[423,7813,425],{},[188,7815,7816],{},"arg",[423,7818,1467],{},[7546,7820,7821,7828],{},[185,7822,7823,7826],{},[188,7824,7825],{},"max",[423,7827,1467],{},[188,7829,597],{},[188,7831,7457],{},[423,7833,431],{"stretchy":430},[188,7835,597],{},[423,7837,440],{"stretchy":430},[192,7839,7840],{"encoding":194},"\\pi^* = \\arg \\max_{\\pi} J(\\pi)",[169,7842,7844,7889],{"className":7843,"ariaHidden":199},[198],[169,7845,7847,7851,7880,7883,7886],{"className":7846},[203],[169,7848],{"className":7849,"style":7850},[207],"height:0.7387em;",[169,7852,7854,7857],{"className":7853},[212],[169,7855,597],{"className":7856,"style":613},[212,213],[169,7858,7860],{"className":7859},[463],[169,7861,7863],{"className":7862},[467],[169,7864,7866],{"className":7865},[472],[169,7867,7869],{"className":7868,"style":7850},[476],[169,7870,7871,7874],{"style":6753},[169,7872],{"className":7873,"style":485},[484],[169,7875,7877],{"className":7876},[489,490,491,492],[169,7878,7811],{"className":7879},[2985,492],[169,7881],{"className":7882,"style":514},[513],[169,7884,425],{"className":7885},[518],[169,7887],{"className":7888,"style":514},[513],[169,7890,7892,7895,7901,7904,7951,7954,7957,7960,7963],{"className":7891},[203],[169,7893],{"className":7894,"style":7709},[207],[169,7896,7898,7899],{"className":7897},[1483],"ar",[169,7900,1618],{"style":1617},[169,7902],{"className":7903,"style":1622},[513],[169,7905,7907],{"className":7906},[1483,3731],[169,7908,7910,7943],{"className":7909},[467,468],[169,7911,7913,7940],{"className":7912},[472],[169,7914,7916,7930],{"className":7915,"style":208},[476],[169,7917,7918,7921],{"style":7728},[169,7919],{"className":7920,"style":999},[484],[169,7922,7924],{"className":7923},[489,490,491,492],[169,7925,7927],{"className":7926},[212,492],[169,7928,597],{"className":7929,"style":613},[212,213,492],[169,7931,7932,7935],{"style":7749},[169,7933],{"className":7934,"style":999},[484],[169,7936,7937],{},[169,7938,7825],{"className":7939},[1483],[169,7941,500],{"className":7942},[499],[169,7944,7946],{"className":7945},[472],[169,7947,7949],{"className":7948,"style":7768},[476],[169,7950],{},[169,7952],{"className":7953,"style":1622},[513],[169,7955,7457],{"className":7956,"style":7479},[212,213],[169,7958,431],{"className":7959},[535],[169,7961,597],{"className":7962,"style":613},[212,213],[169,7964,440],{"className":7965},[579],[14,7967,7968,7969,8029,8030,74],{},"其中的 ",[169,7970,7972,7990],{"className":7971},[172],[169,7973,7975],{"className":7974},[176],[178,7976,7977],{"xmlns":180},[182,7978,7979,7987],{},[185,7980,7981],{},[3502,7982,7983,7985],{},[188,7984,597],{},[423,7986,7811],{},[192,7988,7989],{"encoding":194},"\\pi^*",[169,7991,7993],{"className":7992,"ariaHidden":199},[198],[169,7994,7996,8000],{"className":7995},[203],[169,7997],{"className":7998,"style":7999},[207],"height:0.6887em;",[169,8001,8003,8006],{"className":8002},[212],[169,8004,597],{"className":8005,"style":613},[212,213],[169,8007,8009],{"className":8008},[463],[169,8010,8012],{"className":8011},[467],[169,8013,8015],{"className":8014},[472],[169,8016,8018],{"className":8017,"style":7999},[476],[169,8019,8020,8023],{"style":4025},[169,8021],{"className":8022,"style":485},[484],[169,8024,8026],{"className":8025},[489,490,491,492],[169,8027,7811],{"className":8028},[2985,492]," 就是",[117,8031,8032],{},"最优策略（optimal policy）",[159,8034,6823],{"id":6823},[14,8036,8037,8038,8041,8042,8045],{},"知道一个状态或状态-动作对的",[117,8039,8040],{},"价值（value）"," 常常很有用。所谓价值，意思是：如果从该状态或状态-动作对出发，之后一直按某个特定策略行动，所能得到的期望回报。",[117,8043,8044],{},"价值函数（value function）"," 以这样或那样的方式出现在几乎每一个 RL 算法里。",[14,8047,8048],{},"这里有四个值得注意的主要函数。",[8050,8051,8052,8476,9021,9470],"ol",{},[38,8053,8054,167,8057,8136,8137,8165,8166,8194,8195],{},[117,8055,8056],{},"同策略价值函数（on-policy value function）",[169,8058,8060,8085],{"className":8059},[172],[169,8061,8063],{"className":8062},[176],[178,8064,8065],{"xmlns":180},[182,8066,8067,8082],{},[185,8068,8069,8076,8078,8080],{},[3502,8070,8071,8074],{},[188,8072,8073],{},"V",[188,8075,597],{},[423,8077,431],{"stretchy":430},[188,8079,190],{},[423,8081,440],{"stretchy":430},[192,8083,8084],{"encoding":194},"V^{\\pi}(s)",[169,8086,8088],{"className":8087,"ariaHidden":199},[198],[169,8089,8091,8094,8127,8130,8133],{"className":8090},[203],[169,8092],{"className":8093,"style":528},[207],[169,8095,8097,8100],{"className":8096},[212],[169,8098,8073],{"className":8099,"style":2981},[212,213],[169,8101,8103],{"className":8102},[463],[169,8104,8106],{"className":8105},[467],[169,8107,8109],{"className":8108},[472],[169,8110,8113],{"className":8111,"style":8112},[476],"height:0.6644em;",[169,8114,8115,8118],{"style":4025},[169,8116],{"className":8117,"style":485},[484],[169,8119,8121],{"className":8120},[489,490,491,492],[169,8122,8124],{"className":8123},[212,492],[169,8125,597],{"className":8126,"style":613},[212,213,492],[169,8128,431],{"className":8129},[535],[169,8131,190],{"className":8132},[212,213],[169,8134,440],{"className":8135},[579],"：从状态 ",[169,8138,8140,8153],{"className":8139},[172],[169,8141,8143],{"className":8142},[176],[178,8144,8145],{"xmlns":180},[182,8146,8147,8151],{},[185,8148,8149],{},[188,8150,190],{},[192,8152,190],{"encoding":194},[169,8154,8156],{"className":8155,"ariaHidden":199},[198],[169,8157,8159,8162],{"className":8158},[203],[169,8160],{"className":8161,"style":208},[207],[169,8163,190],{"className":8164},[212,213]," 出发、之后一直按策略 ",[169,8167,8169,8182],{"className":8168},[172],[169,8170,8172],{"className":8171},[176],[178,8173,8174],{"xmlns":180},[182,8175,8176,8180],{},[185,8177,8178],{},[188,8179,597],{},[192,8181,600],{"encoding":194},[169,8183,8185],{"className":8184,"ariaHidden":199},[198],[169,8186,8188,8191],{"className":8187},[203],[169,8189],{"className":8190,"style":208},[207],[169,8192,597],{"className":8193,"style":613},[212,213]," 行动，能得到的期望回报：",[169,8196,8198],{"className":8197},[400],[169,8199,8201,8269],{"className":8200},[172],[169,8202,8204],{"className":8203},[176],[178,8205,8206],{"xmlns":180,"display":409},[182,8207,8208,8266],{},[185,8209,8210,8216,8218,8220,8222,8224,8240],{},[3502,8211,8212,8214],{},[188,8213,8073],{},[188,8215,597],{},[423,8217,431],{"stretchy":430},[188,8219,190],{},[423,8221,440],{"stretchy":430},[423,8223,425],{},[188,8225,8226],{},[7546,8227,8228,8232],{},[423,8229,8230],{},[188,8231,7553],{"mathvariant":7552},[185,8233,8234,8236,8238],{},[188,8235,4174],{},[423,8237,637],{},[188,8239,597],{},[185,8241,8242,8244,8246,8248,8250,8252,8254,8260,8262,8264],{},[423,8243,1896],{"fence":199},[188,8245,5583],{},[423,8247,431],{"stretchy":430},[188,8249,4174],{},[423,8251,440],{"stretchy":430},[423,8253,648],{},[415,8255,8256,8258],{},[188,8257,190],{},[2781,8259,885],{},[423,8261,425],{},[188,8263,190],{},[423,8265,1911],{"fence":199},[192,8267,8268],{"encoding":194},"V^{\\pi}(s) = \\underset{\\tau \\sim \\pi}{\\mathbb{E}}\\left[ R(\\tau) \\mid s_0 = s \\right]",[169,8270,8272,8329],{"className":8271,"ariaHidden":199},[198],[169,8273,8275,8278,8311,8314,8317,8320,8323,8326],{"className":8274},[203],[169,8276],{"className":8277,"style":528},[207],[169,8279,8281,8284],{"className":8280},[212],[169,8282,8073],{"className":8283,"style":2981},[212,213],[169,8285,8287],{"className":8286},[463],[169,8288,8290],{"className":8289},[467],[169,8291,8293],{"className":8292},[472],[169,8294,8297],{"className":8295,"style":8296},[476],"height:0.7144em;",[169,8298,8299,8302],{"style":6753},[169,8300],{"className":8301,"style":485},[484],[169,8303,8305],{"className":8304},[489,490,491,492],[169,8306,8308],{"className":8307},[212,492],[169,8309,597],{"className":8310,"style":613},[212,213,492],[169,8312,431],{"className":8313},[535],[169,8315,190],{"className":8316},[212,213],[169,8318,440],{"className":8319},[579],[169,8321],{"className":8322,"style":514},[513],[169,8324,425],{"className":8325},[518],[169,8327],{"className":8328,"style":514},[513],[169,8330,8332,8335,8391,8394],{"className":8331},[203],[169,8333],{"className":8334,"style":7709},[207],[169,8336,8338],{"className":8337},[212],[169,8339,8341],{"className":8340},[1483,3731],[169,8342,8344,8383],{"className":8343},[467,468],[169,8345,8347,8380],{"className":8346},[472],[169,8348,8350,8370],{"className":8349,"style":7725},[476],[169,8351,8352,8355],{"style":7728},[169,8353],{"className":8354,"style":999},[484],[169,8356,8358],{"className":8357},[489,490,491,492],[169,8359,8361,8364,8367],{"className":8360},[212,492],[169,8362,4174],{"className":8363,"style":4190},[212,213,492],[169,8365,637],{"className":8366},[518,492],[169,8368,597],{"className":8369,"style":613},[212,213,492],[169,8371,8372,8375],{"style":7749},[169,8373],{"className":8374,"style":999},[484],[169,8376,8377],{},[169,8378,7553],{"className":8379},[1483,7758],[169,8381,500],{"className":8382},[499],[169,8384,8386],{"className":8385},[472],[169,8387,8389],{"className":8388,"style":7768},[476],[169,8390],{},[169,8392],{"className":8393,"style":1622},[513],[169,8395,8397,8400,8403,8406,8409,8412,8415,8418,8421,8461,8464,8467,8470,8473],{"className":8396},[2019],[169,8398,1896],{"className":8399,"style":2027},[535,2026],[169,8401,5583],{"className":8402,"style":5598},[212,213],[169,8404,431],{"className":8405},[535],[169,8407,4174],{"className":8408,"style":4190},[212,213],[169,8410,440],{"className":8411},[579],[169,8413],{"className":8414,"style":514},[513],[169,8416,648],{"className":8417},[518],[169,8419],{"className":8420,"style":514},[513],[169,8422,8424,8427],{"className":8423},[212],[169,8425,190],{"className":8426},[212,213],[169,8428,8430],{"className":8429},[463],[169,8431,8433,8453],{"className":8432},[467,468],[169,8434,8436,8450],{"className":8435},[472],[169,8437,8439],{"className":8438,"style":4305},[476],[169,8440,8441,8444],{"style":480},[169,8442],{"className":8443,"style":485},[484],[169,8445,8447],{"className":8446},[489,490,491,492],[169,8448,885],{"className":8449},[212,492],[169,8451,500],{"className":8452},[499],[169,8454,8456],{"className":8455},[472],[169,8457,8459],{"className":8458,"style":507},[476],[169,8460],{},[169,8462],{"className":8463,"style":514},[513],[169,8465,425],{"className":8466},[518],[169,8468],{"className":8469,"style":514},[513],[169,8471,190],{"className":8472},[212,213],[169,8474,1911],{"className":8475,"style":2027},[579,2026],[38,8477,8478,167,8481,8136,8572,8600,8601,8629,8630,8194,8658],{},[117,8479,8480],{},"同策略动作价值函数（on-policy action-value function）",[169,8482,8484,8513],{"className":8483},[172],[169,8485,8487],{"className":8486},[176],[178,8488,8489],{"xmlns":180},[182,8490,8491,8510],{},[185,8492,8493,8500,8502,8504,8506,8508],{},[3502,8494,8495,8498],{},[188,8496,8497],{},"Q",[188,8499,597],{},[423,8501,431],{"stretchy":430},[188,8503,190],{},[423,8505,2505],{"separator":199},[188,8507,18],{},[423,8509,440],{"stretchy":430},[192,8511,8512],{"encoding":194},"Q^{\\pi}(s,a)",[169,8514,8516],{"className":8515,"ariaHidden":199},[198],[169,8517,8519,8522,8554,8557,8560,8563,8566,8569],{"className":8518},[203],[169,8520],{"className":8521,"style":528},[207],[169,8523,8525,8528],{"className":8524},[212],[169,8526,8497],{"className":8527},[212,213],[169,8529,8531],{"className":8530},[463],[169,8532,8534],{"className":8533},[467],[169,8535,8537],{"className":8536},[472],[169,8538,8540],{"className":8539,"style":8112},[476],[169,8541,8542,8545],{"style":4025},[169,8543],{"className":8544,"style":485},[484],[169,8546,8548],{"className":8547},[489,490,491,492],[169,8549,8551],{"className":8550},[212,492],[169,8552,597],{"className":8553,"style":613},[212,213,492],[169,8555,431],{"className":8556},[535],[169,8558,190],{"className":8559},[212,213],[169,8561,2505],{"className":8562},[2534],[169,8564],{"className":8565,"style":1622},[513],[169,8567,18],{"className":8568},[212,213],[169,8570,440],{"className":8571},[579],[169,8573,8575,8588],{"className":8574},[172],[169,8576,8578],{"className":8577},[176],[178,8579,8580],{"xmlns":180},[182,8581,8582,8586],{},[185,8583,8584],{},[188,8585,190],{},[192,8587,190],{"encoding":194},[169,8589,8591],{"className":8590,"ariaHidden":199},[198],[169,8592,8594,8597],{"className":8593},[203],[169,8595],{"className":8596,"style":208},[207],[169,8598,190],{"className":8599},[212,213]," 出发，先采取一个任意动作 ",[169,8602,8604,8617],{"className":8603},[172],[169,8605,8607],{"className":8606},[176],[178,8608,8609],{"xmlns":180},[182,8610,8611,8615],{},[185,8612,8613],{},[188,8614,18],{},[192,8616,18],{"encoding":194},[169,8618,8620],{"className":8619,"ariaHidden":199},[198],[169,8621,8623,8626],{"className":8622},[203],[169,8624],{"className":8625,"style":208},[207],[169,8627,18],{"className":8628},[212,213],"（它可以不来自策略），之后一直按策略 ",[169,8631,8633,8646],{"className":8632},[172],[169,8634,8636],{"className":8635},[176],[178,8637,8638],{"xmlns":180},[182,8639,8640,8644],{},[185,8641,8642],{},[188,8643,597],{},[192,8645,600],{"encoding":194},[169,8647,8649],{"className":8648,"ariaHidden":199},[198],[169,8650,8652,8655],{"className":8651},[203],[169,8653],{"className":8654,"style":208},[207],[169,8656,597],{"className":8657,"style":613},[212,213],[169,8659,8661],{"className":8660},[400],[169,8662,8664,8748],{"className":8663},[172],[169,8665,8667],{"className":8666},[176],[178,8668,8669],{"xmlns":180,"display":409},[182,8670,8671,8745],{},[185,8672,8673,8679,8681,8683,8685,8687,8689,8691,8707],{},[3502,8674,8675,8677],{},[188,8676,8497],{},[188,8678,597],{},[423,8680,431],{"stretchy":430},[188,8682,190],{},[423,8684,2505],{"separator":199},[188,8686,18],{},[423,8688,440],{"stretchy":430},[423,8690,425],{},[188,8692,8693],{},[7546,8694,8695,8699],{},[423,8696,8697],{},[188,8698,7553],{"mathvariant":7552},[185,8700,8701,8703,8705],{},[188,8702,4174],{},[423,8704,637],{},[188,8706,597],{},[185,8708,8709,8711,8713,8715,8717,8719,8721,8727,8729,8731,8733,8739,8741,8743],{},[423,8710,1896],{"fence":199},[188,8712,5583],{},[423,8714,431],{"stretchy":430},[188,8716,4174],{},[423,8718,440],{"stretchy":430},[423,8720,648],{},[415,8722,8723,8725],{},[188,8724,190],{},[2781,8726,885],{},[423,8728,425],{},[188,8730,190],{},[423,8732,2505],{"separator":199},[415,8734,8735,8737],{},[188,8736,18],{},[2781,8738,885],{},[423,8740,425],{},[188,8742,18],{},[423,8744,1911],{"fence":199},[192,8746,8747],{"encoding":194},"Q^{\\pi}(s,a) = \\underset{\\tau \\sim \\pi}{\\mathbb{E}}\\left[ R(\\tau) \\mid s_0 = s, a_0 = a \\right]",[169,8749,8751,8816],{"className":8750,"ariaHidden":199},[198],[169,8752,8754,8757,8789,8792,8795,8798,8801,8804,8807,8810,8813],{"className":8753},[203],[169,8755],{"className":8756,"style":528},[207],[169,8758,8760,8763],{"className":8759},[212],[169,8761,8497],{"className":8762},[212,213],[169,8764,8766],{"className":8765},[463],[169,8767,8769],{"className":8768},[467],[169,8770,8772],{"className":8771},[472],[169,8773,8775],{"className":8774,"style":8296},[476],[169,8776,8777,8780],{"style":6753},[169,8778],{"className":8779,"style":485},[484],[169,8781,8783],{"className":8782},[489,490,491,492],[169,8784,8786],{"className":8785},[212,492],[169,8787,597],{"className":8788,"style":613},[212,213,492],[169,8790,431],{"className":8791},[535],[169,8793,190],{"className":8794},[212,213],[169,8796,2505],{"className":8797},[2534],[169,8799],{"className":8800,"style":1622},[513],[169,8802,18],{"className":8803},[212,213],[169,8805,440],{"className":8806},[579],[169,8808],{"className":8809,"style":514},[513],[169,8811,425],{"className":8812},[518],[169,8814],{"className":8815,"style":514},[513],[169,8817,8819,8822,8878,8881],{"className":8818},[203],[169,8820],{"className":8821,"style":7709},[207],[169,8823,8825],{"className":8824},[212],[169,8826,8828],{"className":8827},[1483,3731],[169,8829,8831,8870],{"className":8830},[467,468],[169,8832,8834,8867],{"className":8833},[472],[169,8835,8837,8857],{"className":8836,"style":7725},[476],[169,8838,8839,8842],{"style":7728},[169,8840],{"className":8841,"style":999},[484],[169,8843,8845],{"className":8844},[489,490,491,492],[169,8846,8848,8851,8854],{"className":8847},[212,492],[169,8849,4174],{"className":8850,"style":4190},[212,213,492],[169,8852,637],{"className":8853},[518,492],[169,8855,597],{"className":8856,"style":613},[212,213,492],[169,8858,8859,8862],{"style":7749},[169,8860],{"className":8861,"style":999},[484],[169,8863,8864],{},[169,8865,7553],{"className":8866},[1483,7758],[169,8868,500],{"className":8869},[499],[169,8871,8873],{"className":8872},[472],[169,8874,8876],{"className":8875,"style":7768},[476],[169,8877],{},[169,8879],{"className":8880,"style":1622},[513],[169,8882,8884,8887,8890,8893,8896,8899,8902,8905,8908,8948,8951,8954,8957,8960,8963,8966,9006,9009,9012,9015,9018],{"className":8883},[2019],[169,8885,1896],{"className":8886,"style":2027},[535,2026],[169,8888,5583],{"className":8889,"style":5598},[212,213],[169,8891,431],{"className":8892},[535],[169,8894,4174],{"className":8895,"style":4190},[212,213],[169,8897,440],{"className":8898},[579],[169,8900],{"className":8901,"style":514},[513],[169,8903,648],{"className":8904},[518],[169,8906],{"className":8907,"style":514},[513],[169,8909,8911,8914],{"className":8910},[212],[169,8912,190],{"className":8913},[212,213],[169,8915,8917],{"className":8916},[463],[169,8918,8920,8940],{"className":8919},[467,468],[169,8921,8923,8937],{"className":8922},[472],[169,8924,8926],{"className":8925,"style":4305},[476],[169,8927,8928,8931],{"style":480},[169,8929],{"className":8930,"style":485},[484],[169,8932,8934],{"className":8933},[489,490,491,492],[169,8935,885],{"className":8936},[212,492],[169,8938,500],{"className":8939},[499],[169,8941,8943],{"className":8942},[472],[169,8944,8946],{"className":8945,"style":507},[476],[169,8947],{},[169,8949],{"className":8950,"style":514},[513],[169,8952,425],{"className":8953},[518],[169,8955],{"className":8956,"style":514},[513],[169,8958,190],{"className":8959},[212,213],[169,8961,2505],{"className":8962},[2534],[169,8964],{"className":8965,"style":1622},[513],[169,8967,8969,8972],{"className":8968},[212],[169,8970,18],{"className":8971},[212,213],[169,8973,8975],{"className":8974},[463],[169,8976,8978,8998],{"className":8977},[467,468],[169,8979,8981,8995],{"className":8980},[472],[169,8982,8984],{"className":8983,"style":4305},[476],[169,8985,8986,8989],{"style":480},[169,8987],{"className":8988,"style":485},[484],[169,8990,8992],{"className":8991},[489,490,491,492],[169,8993,885],{"className":8994},[212,492],[169,8996,500],{"className":8997},[499],[169,8999,9001],{"className":9000},[472],[169,9002,9004],{"className":9003,"style":507},[476],[169,9005],{},[169,9007],{"className":9008,"style":514},[513],[169,9010,425],{"className":9011},[518],[169,9013],{"className":9014,"style":514},[513],[169,9016,18],{"className":9017},[212,213],[169,9019,1911],{"className":9020,"style":2027},[579,2026],[38,9022,9023,167,9026,8136,9100,9128,9129,9132,9133],{},[117,9024,9025],{},"最优价值函数（optimal value function）",[169,9027,9029,9053],{"className":9028},[172],[169,9030,9032],{"className":9031},[176],[178,9033,9034],{"xmlns":180},[182,9035,9036,9050],{},[185,9037,9038,9044,9046,9048],{},[3502,9039,9040,9042],{},[188,9041,8073],{},[423,9043,7811],{},[423,9045,431],{"stretchy":430},[188,9047,190],{},[423,9049,440],{"stretchy":430},[192,9051,9052],{"encoding":194},"V^*(s)",[169,9054,9056],{"className":9055,"ariaHidden":199},[198],[169,9057,9059,9062,9091,9094,9097],{"className":9058},[203],[169,9060],{"className":9061,"style":528},[207],[169,9063,9065,9068],{"className":9064},[212],[169,9066,8073],{"className":9067,"style":2981},[212,213],[169,9069,9071],{"className":9070},[463],[169,9072,9074],{"className":9073},[467],[169,9075,9077],{"className":9076},[472],[169,9078,9080],{"className":9079,"style":7999},[476],[169,9081,9082,9085],{"style":4025},[169,9083],{"className":9084,"style":485},[484],[169,9086,9088],{"className":9087},[489,490,491,492],[169,9089,7811],{"className":9090},[2985,492],[169,9092,431],{"className":9093},[535],[169,9095,190],{"className":9096},[212,213],[169,9098,440],{"className":9099},[579],[169,9101,9103,9116],{"className":9102},[172],[169,9104,9106],{"className":9105},[176],[178,9107,9108],{"xmlns":180},[182,9109,9110,9114],{},[185,9111,9112],{},[188,9113,190],{},[192,9115,190],{"encoding":194},[169,9117,9119],{"className":9118,"ariaHidden":199},[198],[169,9120,9122,9125],{"className":9121},[203],[169,9123],{"className":9124,"style":208},[207],[169,9126,190],{"className":9127},[212,213]," 出发、之后一直按环境中",[45,9130,9131],{},"最优","的策略行动，能得到的期望回报：",[169,9134,9136],{"className":9135},[400],[169,9137,9139,9217],{"className":9138},[172],[169,9140,9142],{"className":9141},[176],[178,9143,9144],{"xmlns":180,"display":409},[182,9145,9146,9214],{},[185,9147,9148,9154,9156,9158,9160,9162,9172,9188],{},[3502,9149,9150,9152],{},[188,9151,8073],{},[423,9153,7811],{},[423,9155,431],{"stretchy":430},[188,9157,190],{},[423,9159,440],{"stretchy":430},[423,9161,425],{},[7546,9163,9164,9170],{},[185,9165,9166,9168],{},[188,9167,7825],{},[423,9169,1467],{},[188,9171,597],{},[188,9173,9174],{},[7546,9175,9176,9180],{},[423,9177,9178],{},[188,9179,7553],{"mathvariant":7552},[185,9181,9182,9184,9186],{},[188,9183,4174],{},[423,9185,637],{},[188,9187,597],{},[185,9189,9190,9192,9194,9196,9198,9200,9202,9208,9210,9212],{},[423,9191,1896],{"fence":199},[188,9193,5583],{},[423,9195,431],{"stretchy":430},[188,9197,4174],{},[423,9199,440],{"stretchy":430},[423,9201,648],{},[415,9203,9204,9206],{},[188,9205,190],{},[2781,9207,885],{},[423,9209,425],{},[188,9211,190],{},[423,9213,1911],{"fence":199},[192,9215,9216],{"encoding":194},"V^*(s) = \\max_{\\pi} \\underset{\\tau \\sim \\pi}{\\mathbb{E}}\\left[ R(\\tau) \\mid s_0 = s \\right]",[169,9218,9220,9273],{"className":9219,"ariaHidden":199},[198],[169,9221,9223,9226,9255,9258,9261,9264,9267,9270],{"className":9222},[203],[169,9224],{"className":9225,"style":528},[207],[169,9227,9229,9232],{"className":9228},[212],[169,9230,8073],{"className":9231,"style":2981},[212,213],[169,9233,9235],{"className":9234},[463],[169,9236,9238],{"className":9237},[467],[169,9239,9241],{"className":9240},[472],[169,9242,9244],{"className":9243,"style":7850},[476],[169,9245,9246,9249],{"style":6753},[169,9247],{"className":9248,"style":485},[484],[169,9250,9252],{"className":9251},[489,490,491,492],[169,9253,7811],{"className":9254},[2985,492],[169,9256,431],{"className":9257},[535],[169,9259,190],{"className":9260},[212,213],[169,9262,440],{"className":9263},[579],[169,9265],{"className":9266,"style":514},[513],[169,9268,425],{"className":9269},[518],[169,9271],{"className":9272,"style":514},[513],[169,9274,9276,9279,9326,9329,9385,9388],{"className":9275},[203],[169,9277],{"className":9278,"style":7709},[207],[169,9280,9282],{"className":9281},[1483,3731],[169,9283,9285,9318],{"className":9284},[467,468],[169,9286,9288,9315],{"className":9287},[472],[169,9289,9291,9305],{"className":9290,"style":208},[476],[169,9292,9293,9296],{"style":7728},[169,9294],{"className":9295,"style":999},[484],[169,9297,9299],{"className":9298},[489,490,491,492],[169,9300,9302],{"className":9301},[212,492],[169,9303,597],{"className":9304,"style":613},[212,213,492],[169,9306,9307,9310],{"style":7749},[169,9308],{"className":9309,"style":999},[484],[169,9311,9312],{},[169,9313,7825],{"className":9314},[1483],[169,9316,500],{"className":9317},[499],[169,9319,9321],{"className":9320},[472],[169,9322,9324],{"className":9323,"style":7768},[476],[169,9325],{},[169,9327],{"className":9328,"style":1622},[513],[169,9330,9332],{"className":9331},[212],[169,9333,9335],{"className":9334},[1483,3731],[169,9336,9338,9377],{"className":9337},[467,468],[169,9339,9341,9374],{"className":9340},[472],[169,9342,9344,9364],{"className":9343,"style":7725},[476],[169,9345,9346,9349],{"style":7728},[169,9347],{"className":9348,"style":999},[484],[169,9350,9352],{"className":9351},[489,490,491,492],[169,9353,9355,9358,9361],{"className":9354},[212,492],[169,9356,4174],{"className":9357,"style":4190},[212,213,492],[169,9359,637],{"className":9360},[518,492],[169,9362,597],{"className":9363,"style":613},[212,213,492],[169,9365,9366,9369],{"style":7749},[169,9367],{"className":9368,"style":999},[484],[169,9370,9371],{},[169,9372,7553],{"className":9373},[1483,7758],[169,9375,500],{"className":9376},[499],[169,9378,9380],{"className":9379},[472],[169,9381,9383],{"className":9382,"style":7768},[476],[169,9384],{},[169,9386],{"className":9387,"style":1622},[513],[169,9389,9391,9394,9397,9400,9403,9406,9409,9412,9415,9455,9458,9461,9464,9467],{"className":9390},[2019],[169,9392,1896],{"className":9393,"style":2027},[535,2026],[169,9395,5583],{"className":9396,"style":5598},[212,213],[169,9398,431],{"className":9399},[535],[169,9401,4174],{"className":9402,"style":4190},[212,213],[169,9404,440],{"className":9405},[579],[169,9407],{"className":9408,"style":514},[513],[169,9410,648],{"className":9411},[518],[169,9413],{"className":9414,"style":514},[513],[169,9416,9418,9421],{"className":9417},[212],[169,9419,190],{"className":9420},[212,213],[169,9422,9424],{"className":9423},[463],[169,9425,9427,9447],{"className":9426},[467,468],[169,9428,9430,9444],{"className":9429},[472],[169,9431,9433],{"className":9432,"style":4305},[476],[169,9434,9435,9438],{"style":480},[169,9436],{"className":9437,"style":485},[484],[169,9439,9441],{"className":9440},[489,490,491,492],[169,9442,885],{"className":9443},[212,492],[169,9445,500],{"className":9446},[499],[169,9448,9450],{"className":9449},[472],[169,9451,9453],{"className":9452,"style":507},[476],[169,9454],{},[169,9456],{"className":9457,"style":514},[513],[169,9459,425],{"className":9460},[518],[169,9462],{"className":9463,"style":514},[513],[169,9465,190],{"className":9466},[212,213],[169,9468,1911],{"className":9469,"style":2027},[579,2026],[38,9471,9472,167,9475,8136,9562,8600,9590,9618,9619,9132,9621],{},[117,9473,9474],{},"最优动作价值函数（optimal action-value function）",[169,9476,9478,9506],{"className":9477},[172],[169,9479,9481],{"className":9480},[176],[178,9482,9483],{"xmlns":180},[182,9484,9485,9503],{},[185,9486,9487,9493,9495,9497,9499,9501],{},[3502,9488,9489,9491],{},[188,9490,8497],{},[423,9492,7811],{},[423,9494,431],{"stretchy":430},[188,9496,190],{},[423,9498,2505],{"separator":199},[188,9500,18],{},[423,9502,440],{"stretchy":430},[192,9504,9505],{"encoding":194},"Q^*(s,a)",[169,9507,9509],{"className":9508,"ariaHidden":199},[198],[169,9510,9512,9515,9544,9547,9550,9553,9556,9559],{"className":9511},[203],[169,9513],{"className":9514,"style":528},[207],[169,9516,9518,9521],{"className":9517},[212],[169,9519,8497],{"className":9520},[212,213],[169,9522,9524],{"className":9523},[463],[169,9525,9527],{"className":9526},[467],[169,9528,9530],{"className":9529},[472],[169,9531,9533],{"className":9532,"style":7999},[476],[169,9534,9535,9538],{"style":4025},[169,9536],{"className":9537,"style":485},[484],[169,9539,9541],{"className":9540},[489,490,491,492],[169,9542,7811],{"className":9543},[2985,492],[169,9545,431],{"className":9546},[535],[169,9548,190],{"className":9549},[212,213],[169,9551,2505],{"className":9552},[2534],[169,9554],{"className":9555,"style":1622},[513],[169,9557,18],{"className":9558},[212,213],[169,9560,440],{"className":9561},[579],[169,9563,9565,9578],{"className":9564},[172],[169,9566,9568],{"className":9567},[176],[178,9569,9570],{"xmlns":180},[182,9571,9572,9576],{},[185,9573,9574],{},[188,9575,190],{},[192,9577,190],{"encoding":194},[169,9579,9581],{"className":9580,"ariaHidden":199},[198],[169,9582,9584,9587],{"className":9583},[203],[169,9585],{"className":9586,"style":208},[207],[169,9588,190],{"className":9589},[212,213],[169,9591,9593,9606],{"className":9592},[172],[169,9594,9596],{"className":9595},[176],[178,9597,9598],{"xmlns":180},[182,9599,9600,9604],{},[185,9601,9602],{},[188,9603,18],{},[192,9605,18],{"encoding":194},[169,9607,9609],{"className":9608,"ariaHidden":199},[198],[169,9610,9612,9615],{"className":9611},[203],[169,9613],{"className":9614,"style":208},[207],[169,9616,18],{"className":9617},[212,213],"，之后一直按环境中",[45,9620,9131],{},[169,9622,9624],{"className":9623},[400],[169,9625,9627,9721],{"className":9626},[172],[169,9628,9630],{"className":9629},[176],[178,9631,9632],{"xmlns":180,"display":409},[182,9633,9634,9718],{},[185,9635,9636,9642,9644,9646,9648,9650,9652,9654,9664,9680],{},[3502,9637,9638,9640],{},[188,9639,8497],{},[423,9641,7811],{},[423,9643,431],{"stretchy":430},[188,9645,190],{},[423,9647,2505],{"separator":199},[188,9649,18],{},[423,9651,440],{"stretchy":430},[423,9653,425],{},[7546,9655,9656,9662],{},[185,9657,9658,9660],{},[188,9659,7825],{},[423,9661,1467],{},[188,9663,597],{},[188,9665,9666],{},[7546,9667,9668,9672],{},[423,9669,9670],{},[188,9671,7553],{"mathvariant":7552},[185,9673,9674,9676,9678],{},[188,9675,4174],{},[423,9677,637],{},[188,9679,597],{},[185,9681,9682,9684,9686,9688,9690,9692,9694,9700,9702,9704,9706,9712,9714,9716],{},[423,9683,1896],{"fence":199},[188,9685,5583],{},[423,9687,431],{"stretchy":430},[188,9689,4174],{},[423,9691,440],{"stretchy":430},[423,9693,648],{},[415,9695,9696,9698],{},[188,9697,190],{},[2781,9699,885],{},[423,9701,425],{},[188,9703,190],{},[423,9705,2505],{"separator":199},[415,9707,9708,9710],{},[188,9709,18],{},[2781,9711,885],{},[423,9713,425],{},[188,9715,18],{},[423,9717,1911],{"fence":199},[192,9719,9720],{"encoding":194},"Q^*(s,a) = \\max_{\\pi} \\underset{\\tau \\sim \\pi}{\\mathbb{E}}\\left[ R(\\tau) \\mid s_0 = s, a_0 = a \\right]",[169,9722,9724,9786],{"className":9723,"ariaHidden":199},[198],[169,9725,9727,9730,9759,9762,9765,9768,9771,9774,9777,9780,9783],{"className":9726},[203],[169,9728],{"className":9729,"style":528},[207],[169,9731,9733,9736],{"className":9732},[212],[169,9734,8497],{"className":9735},[212,213],[169,9737,9739],{"className":9738},[463],[169,9740,9742],{"className":9741},[467],[169,9743,9745],{"className":9744},[472],[169,9746,9748],{"className":9747,"style":7850},[476],[169,9749,9750,9753],{"style":6753},[169,9751],{"className":9752,"style":485},[484],[169,9754,9756],{"className":9755},[489,490,491,492],[169,9757,7811],{"className":9758},[2985,492],[169,9760,431],{"className":9761},[535],[169,9763,190],{"className":9764},[212,213],[169,9766,2505],{"className":9767},[2534],[169,9769],{"className":9770,"style":1622},[513],[169,9772,18],{"className":9773},[212,213],[169,9775,440],{"className":9776},[579],[169,9778],{"className":9779,"style":514},[513],[169,9781,425],{"className":9782},[518],[169,9784],{"className":9785,"style":514},[513],[169,9787,9789,9792,9839,9842,9898,9901],{"className":9788},[203],[169,9790],{"className":9791,"style":7709},[207],[169,9793,9795],{"className":9794},[1483,3731],[169,9796,9798,9831],{"className":9797},[467,468],[169,9799,9801,9828],{"className":9800},[472],[169,9802,9804,9818],{"className":9803,"style":208},[476],[169,9805,9806,9809],{"style":7728},[169,9807],{"className":9808,"style":999},[484],[169,9810,9812],{"className":9811},[489,490,491,492],[169,9813,9815],{"className":9814},[212,492],[169,9816,597],{"className":9817,"style":613},[212,213,492],[169,9819,9820,9823],{"style":7749},[169,9821],{"className":9822,"style":999},[484],[169,9824,9825],{},[169,9826,7825],{"className":9827},[1483],[169,9829,500],{"className":9830},[499],[169,9832,9834],{"className":9833},[472],[169,9835,9837],{"className":9836,"style":7768},[476],[169,9838],{},[169,9840],{"className":9841,"style":1622},[513],[169,9843,9845],{"className":9844},[212],[169,9846,9848],{"className":9847},[1483,3731],[169,9849,9851,9890],{"className":9850},[467,468],[169,9852,9854,9887],{"className":9853},[472],[169,9855,9857,9877],{"className":9856,"style":7725},[476],[169,9858,9859,9862],{"style":7728},[169,9860],{"className":9861,"style":999},[484],[169,9863,9865],{"className":9864},[489,490,491,492],[169,9866,9868,9871,9874],{"className":9867},[212,492],[169,9869,4174],{"className":9870,"style":4190},[212,213,492],[169,9872,637],{"className":9873},[518,492],[169,9875,597],{"className":9876,"style":613},[212,213,492],[169,9878,9879,9882],{"style":7749},[169,9880],{"className":9881,"style":999},[484],[169,9883,9884],{},[169,9885,7553],{"className":9886},[1483,7758],[169,9888,500],{"className":9889},[499],[169,9891,9893],{"className":9892},[472],[169,9894,9896],{"className":9895,"style":7768},[476],[169,9897],{},[169,9899],{"className":9900,"style":1622},[513],[169,9902,9904,9907,9910,9913,9916,9919,9922,9925,9928,9968,9971,9974,9977,9980,9983,9986,10026,10029,10032,10035,10038],{"className":9903},[2019],[169,9905,1896],{"className":9906,"style":2027},[535,2026],[169,9908,5583],{"className":9909,"style":5598},[212,213],[169,9911,431],{"className":9912},[535],[169,9914,4174],{"className":9915,"style":4190},[212,213],[169,9917,440],{"className":9918},[579],[169,9920],{"className":9921,"style":514},[513],[169,9923,648],{"className":9924},[518],[169,9926],{"className":9927,"style":514},[513],[169,9929,9931,9934],{"className":9930},[212],[169,9932,190],{"className":9933},[212,213],[169,9935,9937],{"className":9936},[463],[169,9938,9940,9960],{"className":9939},[467,468],[169,9941,9943,9957],{"className":9942},[472],[169,9944,9946],{"className":9945,"style":4305},[476],[169,9947,9948,9951],{"style":480},[169,9949],{"className":9950,"style":485},[484],[169,9952,9954],{"className":9953},[489,490,491,492],[169,9955,885],{"className":9956},[212,492],[169,9958,500],{"className":9959},[499],[169,9961,9963],{"className":9962},[472],[169,9964,9966],{"className":9965,"style":507},[476],[169,9967],{},[169,9969],{"className":9970,"style":514},[513],[169,9972,425],{"className":9973},[518],[169,9975],{"className":9976,"style":514},[513],[169,9978,190],{"className":9979},[212,213],[169,9981,2505],{"className":9982},[2534],[169,9984],{"className":9985,"style":1622},[513],[169,9987,9989,9992],{"className":9988},[212],[169,9990,18],{"className":9991},[212,213],[169,9993,9995],{"className":9994},[463],[169,9996,9998,10018],{"className":9997},[467,468],[169,9999,10001,10015],{"className":10000},[472],[169,10002,10004],{"className":10003,"style":4305},[476],[169,10005,10006,10009],{"style":480},[169,10007],{"className":10008,"style":485},[484],[169,10010,10012],{"className":10011},[489,490,491,492],[169,10013,885],{"className":10014},[212,492],[169,10016,500],{"className":10017},[499],[169,10019,10021],{"className":10020},[472],[169,10022,10024],{"className":10023,"style":507},[476],[169,10025],{},[169,10027],{"className":10028,"style":514},[513],[169,10030,425],{"className":10031},[518],[169,10033],{"className":10034,"style":514},[513],[169,10036,18],{"className":10037},[212,213],[169,10039,1911],{"className":10040,"style":2027},[579,2026],[10,10042,10043],{"title":264},[14,10044,10045,10046,10049],{},"谈价值函数时，如果没有专门提到对时间的依赖，指的都是期望",[117,10047,10048],{},"无限视界折扣回报","。有限视界不折扣回报的价值函数必须把时间作为参数传入。你能想到为什么吗？提示：时间到了会发生什么？",[10,10051,10052,10055,10308,10311,10525],{"title":264},[14,10053,10054],{},"价值函数与动作价值函数之间有两个经常用到的关键联系：",[169,10056,10058],{"className":10057},[400],[169,10059,10061,10125],{"className":10060},[172],[169,10062,10064],{"className":10063},[176],[178,10065,10066],{"xmlns":180,"display":409},[182,10067,10068,10122],{},[185,10069,10070,10076,10078,10080,10082,10084,10100],{},[3502,10071,10072,10074],{},[188,10073,8073],{},[188,10075,597],{},[423,10077,431],{"stretchy":430},[188,10079,190],{},[423,10081,440],{"stretchy":430},[423,10083,425],{},[188,10085,10086],{},[7546,10087,10088,10092],{},[423,10089,10090],{},[188,10091,7553],{"mathvariant":7552},[185,10093,10094,10096,10098],{},[188,10095,18],{},[423,10097,637],{},[188,10099,597],{},[185,10101,10102,10104,10110,10112,10114,10116,10118,10120],{},[423,10103,1896],{"fence":199},[3502,10105,10106,10108],{},[188,10107,8497],{},[188,10109,597],{},[423,10111,431],{"stretchy":430},[188,10113,190],{},[423,10115,2505],{"separator":199},[188,10117,18],{},[423,10119,440],{"stretchy":430},[423,10121,1911],{"fence":199},[192,10123,10124],{"encoding":194},"V^{\\pi}(s) = \\underset{a\\sim \\pi}{\\mathbb{E}}\\left[ Q^{\\pi}(s,a) \\right]",[169,10126,10128,10184],{"className":10127,"ariaHidden":199},[198],[169,10129,10131,10134,10166,10169,10172,10175,10178,10181],{"className":10130},[203],[169,10132],{"className":10133,"style":528},[207],[169,10135,10137,10140],{"className":10136},[212],[169,10138,8073],{"className":10139,"style":2981},[212,213],[169,10141,10143],{"className":10142},[463],[169,10144,10146],{"className":10145},[467],[169,10147,10149],{"className":10148},[472],[169,10150,10152],{"className":10151,"style":8296},[476],[169,10153,10154,10157],{"style":6753},[169,10155],{"className":10156,"style":485},[484],[169,10158,10160],{"className":10159},[489,490,491,492],[169,10161,10163],{"className":10162},[212,492],[169,10164,597],{"className":10165,"style":613},[212,213,492],[169,10167,431],{"className":10168},[535],[169,10170,190],{"className":10171},[212,213],[169,10173,440],{"className":10174},[579],[169,10176],{"className":10177,"style":514},[513],[169,10179,425],{"className":10180},[518],[169,10182],{"className":10183,"style":514},[513],[169,10185,10187,10190,10246,10249],{"className":10186},[203],[169,10188],{"className":10189,"style":7709},[207],[169,10191,10193],{"className":10192},[212],[169,10194,10196],{"className":10195},[1483,3731],[169,10197,10199,10238],{"className":10198},[467,468],[169,10200,10202,10235],{"className":10201},[472],[169,10203,10205,10225],{"className":10204,"style":7725},[476],[169,10206,10207,10210],{"style":7728},[169,10208],{"className":10209,"style":999},[484],[169,10211,10213],{"className":10212},[489,490,491,492],[169,10214,10216,10219,10222],{"className":10215},[212,492],[169,10217,18],{"className":10218},[212,213,492],[169,10220,637],{"className":10221},[518,492],[169,10223,597],{"className":10224,"style":613},[212,213,492],[169,10226,10227,10230],{"style":7749},[169,10228],{"className":10229,"style":999},[484],[169,10231,10232],{},[169,10233,7553],{"className":10234},[1483,7758],[169,10236,500],{"className":10237},[499],[169,10239,10241],{"className":10240},[472],[169,10242,10244],{"className":10243,"style":7768},[476],[169,10245],{},[169,10247],{"className":10248,"style":1622},[513],[169,10250,10252,10255,10287,10290,10293,10296,10299,10302,10305],{"className":10251},[2019],[169,10253,1896],{"className":10254,"style":2027},[535,2026],[169,10256,10258,10261],{"className":10257},[212],[169,10259,8497],{"className":10260},[212,213],[169,10262,10264],{"className":10263},[463],[169,10265,10267],{"className":10266},[467],[169,10268,10270],{"className":10269},[472],[169,10271,10273],{"className":10272,"style":8296},[476],[169,10274,10275,10278],{"style":6753},[169,10276],{"className":10277,"style":485},[484],[169,10279,10281],{"className":10280},[489,490,491,492],[169,10282,10284],{"className":10283},[212,492],[169,10285,597],{"className":10286,"style":613},[212,213,492],[169,10288,431],{"className":10289},[535],[169,10291,190],{"className":10292},[212,213],[169,10294,2505],{"className":10295},[2534],[169,10297],{"className":10298,"style":1622},[513],[169,10300,18],{"className":10301},[212,213],[169,10303,440],{"className":10304},[579],[169,10306,1911],{"className":10307,"style":2027},[579,2026],[14,10309,10310],{},"以及",[169,10312,10314],{"className":10313},[400],[169,10315,10317,10369],{"className":10316},[172],[169,10318,10320],{"className":10319},[176],[178,10321,10322],{"xmlns":180,"display":409},[182,10323,10324,10366],{},[185,10325,10326,10332,10334,10336,10338,10340,10350,10356,10358,10360,10362,10364],{},[3502,10327,10328,10330],{},[188,10329,8073],{},[423,10331,7811],{},[423,10333,431],{"stretchy":430},[188,10335,190],{},[423,10337,440],{"stretchy":430},[423,10339,425],{},[7546,10341,10342,10348],{},[185,10343,10344,10346],{},[188,10345,7825],{},[423,10347,1467],{},[188,10349,18],{},[3502,10351,10352,10354],{},[188,10353,8497],{},[423,10355,7811],{},[423,10357,431],{"stretchy":430},[188,10359,190],{},[423,10361,2505],{"separator":199},[188,10363,18],{},[423,10365,440],{"stretchy":430},[192,10367,10368],{"encoding":194},"V^*(s) = \\max_a Q^*(s,a)",[169,10370,10372,10425],{"className":10371,"ariaHidden":199},[198],[169,10373,10375,10378,10407,10410,10413,10416,10419,10422],{"className":10374},[203],[169,10376],{"className":10377,"style":528},[207],[169,10379,10381,10384],{"className":10380},[212],[169,10382,8073],{"className":10383,"style":2981},[212,213],[169,10385,10387],{"className":10386},[463],[169,10388,10390],{"className":10389},[467],[169,10391,10393],{"className":10392},[472],[169,10394,10396],{"className":10395,"style":7850},[476],[169,10397,10398,10401],{"style":6753},[169,10399],{"className":10400,"style":485},[484],[169,10402,10404],{"className":10403},[489,490,491,492],[169,10405,7811],{"className":10406},[2985,492],[169,10408,431],{"className":10409},[535],[169,10411,190],{"className":10412},[212,213],[169,10414,440],{"className":10415},[579],[169,10417],{"className":10418,"style":514},[513],[169,10420,425],{"className":10421},[518],[169,10423],{"className":10424,"style":514},[513],[169,10426,10428,10431,10475,10478,10507,10510,10513,10516,10519,10522],{"className":10427},[203],[169,10429],{"className":10430,"style":7709},[207],[169,10432,10434],{"className":10433},[1483,3731],[169,10435,10437,10467],{"className":10436},[467,468],[169,10438,10440,10464],{"className":10439},[472],[169,10441,10443,10454],{"className":10442,"style":208},[476],[169,10444,10445,10448],{"style":7728},[169,10446],{"className":10447,"style":999},[484],[169,10449,10451],{"className":10450},[489,490,491,492],[169,10452,18],{"className":10453},[212,213,492],[169,10455,10456,10459],{"style":7749},[169,10457],{"className":10458,"style":999},[484],[169,10460,10461],{},[169,10462,7825],{"className":10463},[1483],[169,10465,500],{"className":10466},[499],[169,10468,10470],{"className":10469},[472],[169,10471,10473],{"className":10472,"style":7768},[476],[169,10474],{},[169,10476],{"className":10477,"style":1622},[513],[169,10479,10481,10484],{"className":10480},[212],[169,10482,8497],{"className":10483},[212,213],[169,10485,10487],{"className":10486},[463],[169,10488,10490],{"className":10489},[467],[169,10491,10493],{"className":10492},[472],[169,10494,10496],{"className":10495,"style":7850},[476],[169,10497,10498,10501],{"style":6753},[169,10499],{"className":10500,"style":485},[484],[169,10502,10504],{"className":10503},[489,490,491,492],[169,10505,7811],{"className":10506},[2985,492],[169,10508,431],{"className":10509},[535],[169,10511,190],{"className":10512},[212,213],[169,10514,2505],{"className":10515},[2534],[169,10517],{"className":10518,"style":1622},[513],[169,10520,18],{"className":10521},[212,213],[169,10523,440],{"className":10524},[579],[14,10526,10527],{},"这些关系可以直接从刚才给出的定义推出来：你能证明吗？",[159,10529,10531],{"id":10530},"最优-q-函数与最优动作","最优 Q 函数与最优动作",[14,10533,10534,10535,10621,10622,10708,10709,10737,10738,10766],{},"最优动作价值函数 ",[169,10536,10538,10565],{"className":10537},[172],[169,10539,10541],{"className":10540},[176],[178,10542,10543],{"xmlns":180},[182,10544,10545,10563],{},[185,10546,10547,10553,10555,10557,10559,10561],{},[3502,10548,10549,10551],{},[188,10550,8497],{},[423,10552,7811],{},[423,10554,431],{"stretchy":430},[188,10556,190],{},[423,10558,2505],{"separator":199},[188,10560,18],{},[423,10562,440],{"stretchy":430},[192,10564,9505],{"encoding":194},[169,10566,10568],{"className":10567,"ariaHidden":199},[198],[169,10569,10571,10574,10603,10606,10609,10612,10615,10618],{"className":10570},[203],[169,10572],{"className":10573,"style":528},[207],[169,10575,10577,10580],{"className":10576},[212],[169,10578,8497],{"className":10579},[212,213],[169,10581,10583],{"className":10582},[463],[169,10584,10586],{"className":10585},[467],[169,10587,10589],{"className":10588},[472],[169,10590,10592],{"className":10591,"style":7999},[476],[169,10593,10594,10597],{"style":4025},[169,10595],{"className":10596,"style":485},[484],[169,10598,10600],{"className":10599},[489,490,491,492],[169,10601,7811],{"className":10602},[2985,492],[169,10604,431],{"className":10605},[535],[169,10607,190],{"className":10608},[212,213],[169,10610,2505],{"className":10611},[2534],[169,10613],{"className":10614,"style":1622},[513],[169,10616,18],{"className":10617},[212,213],[169,10619,440],{"className":10620},[579]," 与最优策略所选的动作之间有一个重要联系。按定义，",[169,10623,10625,10652],{"className":10624},[172],[169,10626,10628],{"className":10627},[176],[178,10629,10630],{"xmlns":180},[182,10631,10632,10650],{},[185,10633,10634,10640,10642,10644,10646,10648],{},[3502,10635,10636,10638],{},[188,10637,8497],{},[423,10639,7811],{},[423,10641,431],{"stretchy":430},[188,10643,190],{},[423,10645,2505],{"separator":199},[188,10647,18],{},[423,10649,440],{"stretchy":430},[192,10651,9505],{"encoding":194},[169,10653,10655],{"className":10654,"ariaHidden":199},[198],[169,10656,10658,10661,10690,10693,10696,10699,10702,10705],{"className":10657},[203],[169,10659],{"className":10660,"style":528},[207],[169,10662,10664,10667],{"className":10663},[212],[169,10665,8497],{"className":10666},[212,213],[169,10668,10670],{"className":10669},[463],[169,10671,10673],{"className":10672},[467],[169,10674,10676],{"className":10675},[472],[169,10677,10679],{"className":10678,"style":7999},[476],[169,10680,10681,10684],{"style":4025},[169,10682],{"className":10683,"style":485},[484],[169,10685,10687],{"className":10686},[489,490,491,492],[169,10688,7811],{"className":10689},[2985,492],[169,10691,431],{"className":10692},[535],[169,10694,190],{"className":10695},[212,213],[169,10697,2505],{"className":10698},[2534],[169,10700],{"className":10701,"style":1622},[513],[169,10703,18],{"className":10704},[212,213],[169,10706,440],{"className":10707},[579]," 给出的是：从状态 ",[169,10710,10712,10725],{"className":10711},[172],[169,10713,10715],{"className":10714},[176],[178,10716,10717],{"xmlns":180},[182,10718,10719,10723],{},[185,10720,10721],{},[188,10722,190],{},[192,10724,190],{"encoding":194},[169,10726,10728],{"className":10727,"ariaHidden":199},[198],[169,10729,10731,10734],{"className":10730},[203],[169,10732],{"className":10733,"style":208},[207],[169,10735,190],{"className":10736},[212,213]," 出发、先采取（任意的）动作 ",[169,10739,10741,10754],{"className":10740},[172],[169,10742,10744],{"className":10743},[176],[178,10745,10746],{"xmlns":180},[182,10747,10748,10752],{},[185,10749,10750],{},[188,10751,18],{},[192,10753,18],{"encoding":194},[169,10755,10757],{"className":10756,"ariaHidden":199},[198],[169,10758,10760,10763],{"className":10759},[203],[169,10761],{"className":10762,"style":208},[207],[169,10764,18],{"className":10765},[212,213],"、之后一直按最优策略行动时的期望回报。",[14,10768,10769,10770,10798,10799,10827,10828,10888,10889,396],{},"状态 ",[169,10771,10773,10786],{"className":10772},[172],[169,10774,10776],{"className":10775},[176],[178,10777,10778],{"xmlns":180},[182,10779,10780,10784],{},[185,10781,10782],{},[188,10783,190],{},[192,10785,190],{"encoding":194},[169,10787,10789],{"className":10788,"ariaHidden":199},[198],[169,10790,10792,10795],{"className":10791},[203],[169,10793],{"className":10794,"style":208},[207],[169,10796,190],{"className":10797},[212,213]," 处的最优策略，会选择使「从 ",[169,10800,10802,10815],{"className":10801},[172],[169,10803,10805],{"className":10804},[176],[178,10806,10807],{"xmlns":180},[182,10808,10809,10813],{},[185,10810,10811],{},[188,10812,190],{},[192,10814,190],{"encoding":194},[169,10816,10818],{"className":10817,"ariaHidden":199},[198],[169,10819,10821,10824],{"className":10820},[203],[169,10822],{"className":10823,"style":208},[207],[169,10825,190],{"className":10826},[212,213]," 出发的期望回报」最大的那个动作。因此，如果我们手里有 ",[169,10829,10831,10849],{"className":10830},[172],[169,10832,10834],{"className":10833},[176],[178,10835,10836],{"xmlns":180},[182,10837,10838,10846],{},[185,10839,10840],{},[3502,10841,10842,10844],{},[188,10843,8497],{},[423,10845,7811],{},[192,10847,10848],{"encoding":194},"Q^*",[169,10850,10852],{"className":10851,"ariaHidden":199},[198],[169,10853,10855,10859],{"className":10854},[203],[169,10856],{"className":10857,"style":10858},[207],"height:0.8831em;vertical-align:-0.1944em;",[169,10860,10862,10865],{"className":10861},[212],[169,10863,8497],{"className":10864},[212,213],[169,10866,10868],{"className":10867},[463],[169,10869,10871],{"className":10870},[467],[169,10872,10874],{"className":10873},[472],[169,10875,10877],{"className":10876,"style":7999},[476],[169,10878,10879,10882],{"style":4025},[169,10880],{"className":10881,"style":485},[484],[169,10883,10885],{"className":10884},[489,490,491,492],[169,10886,7811],{"className":10887},[2985,492],"，就可以直接得到最优动作 ",[169,10890,10892,10916],{"className":10891},[172],[169,10893,10895],{"className":10894},[176],[178,10896,10897],{"xmlns":180},[182,10898,10899,10913],{},[185,10900,10901,10907,10909,10911],{},[3502,10902,10903,10905],{},[188,10904,18],{},[423,10906,7811],{},[423,10908,431],{"stretchy":430},[188,10910,190],{},[423,10912,440],{"stretchy":430},[192,10914,10915],{"encoding":194},"a^*(s)",[169,10917,10919],{"className":10918,"ariaHidden":199},[198],[169,10920,10922,10925,10954,10957,10960],{"className":10921},[203],[169,10923],{"className":10924,"style":528},[207],[169,10926,10928,10931],{"className":10927},[212],[169,10929,18],{"className":10930},[212,213],[169,10932,10934],{"className":10933},[463],[169,10935,10937],{"className":10936},[467],[169,10938,10940],{"className":10939},[472],[169,10941,10943],{"className":10942,"style":7999},[476],[169,10944,10945,10948],{"style":4025},[169,10946],{"className":10947,"style":485},[484],[169,10949,10951],{"className":10950},[489,490,491,492],[169,10952,7811],{"className":10953},[2985,492],[169,10955,431],{"className":10956},[535],[169,10958,190],{"className":10959},[212,213],[169,10961,440],{"className":10962},[579],[169,10964,10966],{"className":10965},[400],[169,10967,10969,11025],{"className":10968},[172],[169,10970,10972],{"className":10971},[176],[178,10973,10974],{"xmlns":180,"display":409},[182,10975,10976,11022],{},[185,10977,10978,10984,10986,10988,10990,10992,10994,10996,11006,11012,11014,11016,11018,11020],{},[3502,10979,10980,10982],{},[188,10981,18],{},[423,10983,7811],{},[423,10985,431],{"stretchy":430},[188,10987,190],{},[423,10989,440],{"stretchy":430},[423,10991,425],{},[188,10993,7816],{},[423,10995,1467],{},[7546,10997,10998,11004],{},[185,10999,11000,11002],{},[188,11001,7825],{},[423,11003,1467],{},[188,11005,18],{},[3502,11007,11008,11010],{},[188,11009,8497],{},[423,11011,7811],{},[423,11013,431],{"stretchy":430},[188,11015,190],{},[423,11017,2505],{"separator":199},[188,11019,18],{},[423,11021,440],{"stretchy":430},[192,11023,11024],{"encoding":194},"a^*(s) = \\arg \\max_a Q^*(s,a)",[169,11026,11028,11081],{"className":11027,"ariaHidden":199},[198],[169,11029,11031,11034,11063,11066,11069,11072,11075,11078],{"className":11030},[203],[169,11032],{"className":11033,"style":528},[207],[169,11035,11037,11040],{"className":11036},[212],[169,11038,18],{"className":11039},[212,213],[169,11041,11043],{"className":11042},[463],[169,11044,11046],{"className":11045},[467],[169,11047,11049],{"className":11048},[472],[169,11050,11052],{"className":11051,"style":7850},[476],[169,11053,11054,11057],{"style":6753},[169,11055],{"className":11056,"style":485},[484],[169,11058,11060],{"className":11059},[489,490,491,492],[169,11061,7811],{"className":11062},[2985,492],[169,11064,431],{"className":11065},[535],[169,11067,190],{"className":11068},[212,213],[169,11070,440],{"className":11071},[579],[169,11073],{"className":11074,"style":514},[513],[169,11076,425],{"className":11077},[518],[169,11079],{"className":11080,"style":514},[513],[169,11082,11084,11087,11092,11095,11139,11142,11171,11174,11177,11180,11183,11186],{"className":11083},[203],[169,11085],{"className":11086,"style":7709},[207],[169,11088,7898,11090],{"className":11089},[1483],[169,11091,1618],{"style":1617},[169,11093],{"className":11094,"style":1622},[513],[169,11096,11098],{"className":11097},[1483,3731],[169,11099,11101,11131],{"className":11100},[467,468],[169,11102,11104,11128],{"className":11103},[472],[169,11105,11107,11118],{"className":11106,"style":208},[476],[169,11108,11109,11112],{"style":7728},[169,11110],{"className":11111,"style":999},[484],[169,11113,11115],{"className":11114},[489,490,491,492],[169,11116,18],{"className":11117},[212,213,492],[169,11119,11120,11123],{"style":7749},[169,11121],{"className":11122,"style":999},[484],[169,11124,11125],{},[169,11126,7825],{"className":11127},[1483],[169,11129,500],{"className":11130},[499],[169,11132,11134],{"className":11133},[472],[169,11135,11137],{"className":11136,"style":7768},[476],[169,11138],{},[169,11140],{"className":11141,"style":1622},[513],[169,11143,11145,11148],{"className":11144},[212],[169,11146,8497],{"className":11147},[212,213],[169,11149,11151],{"className":11150},[463],[169,11152,11154],{"className":11153},[467],[169,11155,11157],{"className":11156},[472],[169,11158,11160],{"className":11159,"style":7850},[476],[169,11161,11162,11165],{"style":6753},[169,11163],{"className":11164,"style":485},[484],[169,11166,11168],{"className":11167},[489,490,491,492],[169,11169,7811],{"className":11170},[2985,492],[169,11172,431],{"className":11173},[535],[169,11175,190],{"className":11176},[212,213],[169,11178,2505],{"className":11179},[2534],[169,11181],{"className":11182,"style":1622},[513],[169,11184,18],{"className":11185},[212,213],[169,11187,440],{"className":11188},[579],[14,11190,11191,11192,11278],{},"注意：可能有多个动作同时使 ",[169,11193,11195,11222],{"className":11194},[172],[169,11196,11198],{"className":11197},[176],[178,11199,11200],{"xmlns":180},[182,11201,11202,11220],{},[185,11203,11204,11210,11212,11214,11216,11218],{},[3502,11205,11206,11208],{},[188,11207,8497],{},[423,11209,7811],{},[423,11211,431],{"stretchy":430},[188,11213,190],{},[423,11215,2505],{"separator":199},[188,11217,18],{},[423,11219,440],{"stretchy":430},[192,11221,9505],{"encoding":194},[169,11223,11225],{"className":11224,"ariaHidden":199},[198],[169,11226,11228,11231,11260,11263,11266,11269,11272,11275],{"className":11227},[203],[169,11229],{"className":11230,"style":528},[207],[169,11232,11234,11237],{"className":11233},[212],[169,11235,8497],{"className":11236},[212,213],[169,11238,11240],{"className":11239},[463],[169,11241,11243],{"className":11242},[467],[169,11244,11246],{"className":11245},[472],[169,11247,11249],{"className":11248,"style":7999},[476],[169,11250,11251,11254],{"style":4025},[169,11252],{"className":11253,"style":485},[484],[169,11255,11257],{"className":11256},[489,490,491,492],[169,11258,7811],{"className":11259},[2985,492],[169,11261,431],{"className":11262},[535],[169,11264,190],{"className":11265},[212,213],[169,11267,2505],{"className":11268},[2534],[169,11270],{"className":11271,"style":1622},[513],[169,11273,18],{"className":11274},[212,213],[169,11276,440],{"className":11277},[579]," 最大，此时它们全都是最优的，最优策略可以随机选择其中任何一个。但总存在一个确定型地选择动作的最优策略。",[159,11280,11281],{"id":11281},"贝尔曼方程",[14,11283,11284,11285,11288],{},"四个价值函数全都满足一类特殊的自洽方程，称为",[117,11286,11287],{},"贝尔曼方程（Bellman equations）","。贝尔曼方程背后的基本想法是：",[11290,11291,11292],"blockquote",{},[14,11293,11294],{},"你出发点的价值，等于你期望从待在那里得到的奖励，加上你下一落脚点的价值。",[14,11296,11297],{},"同策略价值函数的贝尔曼方程为：",[169,11299,11301],{"className":11300},[400],[169,11302,11304,11558],{"className":11303},[172],[169,11305,11307],{"className":11306},[176],[178,11308,11309],{"xmlns":180,"display":409},[182,11310,11311,11555],{},[871,11312,11313,11429],{"rowspacing":873,"columnalign":874,"columnspacing":875},[877,11314,11315,11333],{},[880,11316,11317],{},[883,11318,11319],{"scriptlevel":885,"displaystyle":199},[185,11320,11321,11327,11329,11331],{},[3502,11322,11323,11325],{},[188,11324,8073],{},[188,11326,597],{},[423,11328,431],{"stretchy":430},[188,11330,190],{},[423,11332,440],{"stretchy":430},[880,11334,11335],{},[883,11336,11337],{"scriptlevel":885,"displaystyle":199},[185,11338,11339,11341,11343,11391],{},[185,11340],{},[423,11342,425],{},[188,11344,11345],{},[7546,11346,11347,11351],{},[423,11348,11349],{},[188,11350,7553],{"mathvariant":7552},[883,11352,11353],{"scriptlevel":3452},[871,11354,11358,11372],{"rowspacing":11355,"columnalign":11356,"columnspacing":11357},"0.1em","center","1em",[877,11359,11360],{},[880,11361,11362],{},[883,11363,11364],{"scriptlevel":3452,"displaystyle":430},[185,11365,11366,11368,11370],{},[188,11367,18],{},[423,11369,637],{},[188,11371,597],{},[877,11373,11374],{},[880,11375,11376],{},[883,11377,11378],{"scriptlevel":3452,"displaystyle":430},[185,11379,11380,11387,11389],{},[3502,11381,11382,11384],{},[188,11383,190],{},[423,11385,11386],{"mathvariant":647,"lspace":875,"rspace":875},"′",[423,11388,637],{},[188,11390,1744],{},[185,11392,11393,11395,11397,11399,11401,11403,11405,11407,11409,11411,11417,11419,11425,11427],{},[423,11394,1896],{"fence":199},[188,11396,5619],{},[423,11398,431],{"stretchy":430},[188,11400,190],{},[423,11402,2505],{"separator":199},[188,11404,18],{},[423,11406,440],{"stretchy":430},[423,11408,2878],{},[188,11410,6506],{},[3502,11412,11413,11415],{},[188,11414,8073],{},[188,11416,597],{},[423,11418,431],{"stretchy":430},[3502,11420,11421,11423],{},[188,11422,190],{},[423,11424,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,11426,440],{"stretchy":430},[423,11428,1911],{"fence":199},[877,11430,11431,11453],{},[880,11432,11433],{},[883,11434,11435],{"scriptlevel":885,"displaystyle":199},[185,11436,11437,11443,11445,11447,11449,11451],{},[3502,11438,11439,11441],{},[188,11440,8497],{},[188,11442,597],{},[423,11444,431],{"stretchy":430},[188,11446,190],{},[423,11448,2505],{"separator":199},[188,11450,18],{},[423,11452,440],{"stretchy":430},[880,11454,11455],{},[883,11456,11457],{"scriptlevel":885,"displaystyle":199},[185,11458,11459,11461,11463,11483],{},[185,11460],{},[423,11462,425],{},[188,11464,11465],{},[7546,11466,11467,11471],{},[423,11468,11469],{},[188,11470,7553],{"mathvariant":7552},[185,11472,11473,11479,11481],{},[3502,11474,11475,11477],{},[188,11476,190],{},[423,11478,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,11480,637],{},[188,11482,1744],{},[185,11484,11485,11487,11489,11491,11493,11495,11497,11499,11501,11503,11523,11553],{},[423,11486,1896],{"fence":199},[188,11488,5619],{},[423,11490,431],{"stretchy":430},[188,11492,190],{},[423,11494,2505],{"separator":199},[188,11496,18],{},[423,11498,440],{"stretchy":430},[423,11500,2878],{},[188,11502,6506],{},[188,11504,11505],{},[7546,11506,11507,11511],{},[423,11508,11509],{},[188,11510,7553],{"mathvariant":7552},[185,11512,11513,11519,11521],{},[3502,11514,11515,11517],{},[188,11516,18],{},[423,11518,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,11520,637],{},[188,11522,597],{},[185,11524,11525,11527,11533,11535,11541,11543,11549,11551],{},[423,11526,1896],{"fence":199},[3502,11528,11529,11531],{},[188,11530,8497],{},[188,11532,597],{},[423,11534,431],{"stretchy":430},[3502,11536,11537,11539],{},[188,11538,190],{},[423,11540,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,11542,2505],{"separator":199},[3502,11544,11545,11547],{},[188,11546,18],{},[423,11548,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,11550,440],{"stretchy":430},[423,11552,1911],{"fence":199},[423,11554,1911],{"fence":199},[192,11556,11557],{"encoding":194},"\\begin{aligned}\nV^{\\pi}(s) &= \\underset{\\substack{a \\sim \\pi \\\\ s'\\sim P}}{\\mathbb{E}}\\left[r(s,a) + \\gamma V^{\\pi}(s')\\right] \\\\\nQ^{\\pi}(s,a) &= \\underset{s'\\sim P}{\\mathbb{E}}\\left[r(s,a) + \\gamma \\underset{a'\\sim \\pi}{\\mathbb{E}}\\left[Q^{\\pi}(s',a')\\right]\\right]\n\\end{aligned}",[169,11559,11561],{"className":11560,"ariaHidden":199},[198],[169,11562,11564,11568],{"className":11563},[203],[169,11565],{"className":11566,"style":11567},[207],"height:4.4332em;vertical-align:-1.9666em;",[169,11569,11571],{"className":11570},[212],[169,11572,11574,11709],{"className":11573},[871],[169,11575,11577],{"className":11576},[982],[169,11578,11580,11700],{"className":11579},[467,468],[169,11581,11583,11697],{"className":11582},[472],[169,11584,11587,11638],{"className":11585,"style":11586},[476],"height:2.4666em;",[169,11588,11590,11594],{"style":11589},"top:-4.7766em;",[169,11591],{"className":11592,"style":11593},[484],"height:3.15em;",[169,11595,11597,11629,11632,11635],{"className":11596},[212],[169,11598,11600,11603],{"className":11599},[212],[169,11601,8073],{"className":11602,"style":2981},[212,213],[169,11604,11606],{"className":11605},[463],[169,11607,11609],{"className":11608},[467],[169,11610,11612],{"className":11611},[472],[169,11613,11615],{"className":11614,"style":8296},[476],[169,11616,11617,11620],{"style":6753},[169,11618],{"className":11619,"style":485},[484],[169,11621,11623],{"className":11622},[489,490,491,492],[169,11624,11626],{"className":11625},[212,492],[169,11627,597],{"className":11628,"style":613},[212,213,492],[169,11630,431],{"className":11631},[535],[169,11633,190],{"className":11634},[212,213],[169,11636,440],{"className":11637},[579],[169,11639,11641,11644],{"style":11640},"top:-1.9277em;",[169,11642],{"className":11643,"style":11593},[484],[169,11645,11647,11679,11682,11685,11688,11691,11694],{"className":11646},[212],[169,11648,11650,11653],{"className":11649},[212],[169,11651,8497],{"className":11652},[212,213],[169,11654,11656],{"className":11655},[463],[169,11657,11659],{"className":11658},[467],[169,11660,11662],{"className":11661},[472],[169,11663,11665],{"className":11664,"style":8296},[476],[169,11666,11667,11670],{"style":6753},[169,11668],{"className":11669,"style":485},[484],[169,11671,11673],{"className":11672},[489,490,491,492],[169,11674,11676],{"className":11675},[212,492],[169,11677,597],{"className":11678,"style":613},[212,213,492],[169,11680,431],{"className":11681},[535],[169,11683,190],{"className":11684},[212,213],[169,11686,2505],{"className":11687},[2534],[169,11689],{"className":11690,"style":1622},[513],[169,11692,18],{"className":11693},[212,213],[169,11695,440],{"className":11696},[579],[169,11698,500],{"className":11699},[499],[169,11701,11703],{"className":11702},[472],[169,11704,11707],{"className":11705,"style":11706},[476],"height:1.9666em;",[169,11708],{},[169,11710,11712],{"className":11711},[1107],[169,11713,11715,12380],{"className":11714},[467,468],[169,11716,11718,12377],{"className":11717},[472],[169,11719,11721,12009],{"className":11720,"style":11586},[476],[169,11722,11723,11726],{"style":11589},[169,11724],{"className":11725,"style":11593},[484],[169,11727,11729,11732,11735,11738,11741,11893,11896],{"className":11728},[212],[169,11730],{"className":11731},[212],[169,11733],{"className":11734,"style":514},[513],[169,11736,425],{"className":11737},[518],[169,11739],{"className":11740,"style":514},[513],[169,11742,11744],{"className":11743},[212],[169,11745,11747],{"className":11746},[1483,3731],[169,11748,11750,11884],{"className":11749},[467,468],[169,11751,11753,11881],{"className":11752},[472],[169,11754,11756,11871],{"className":11755,"style":7725},[476],[169,11757,11759,11762],{"style":11758},"top:-2.0926em;margin-left:0em;",[169,11760],{"className":11761,"style":999},[484],[169,11763,11765],{"className":11764},[489,490,491,492],[169,11766,11768],{"className":11767},[212,492],[169,11769,11771],{"className":11770},[212,492],[169,11772,11774],{"className":11773},[871],[169,11775,11778],{"className":11776},[11777],"col-align-c",[169,11779,11781,11862],{"className":11780},[467,468],[169,11782,11784,11859],{"className":11783},[472],[169,11785,11788,11807],{"className":11786,"style":11787},[476],"height:1.0592em;",[169,11789,11791,11795],{"style":11790},"top:-3.4565em;",[169,11792],{"className":11793,"style":11794},[484],"height:2.8278em;",[169,11796,11798,11801,11804],{"className":11797},[212,492],[169,11799,18],{"className":11800},[212,213,492],[169,11802,637],{"className":11803},[518,492],[169,11805,597],{"className":11806,"style":613},[212,213,492],[169,11808,11810,11813],{"style":11809},"top:-2.4486em;",[169,11811],{"className":11812,"style":11794},[484],[169,11814,11816,11853,11856],{"className":11815},[212,492],[169,11817,11819,11822],{"className":11818},[212,492],[169,11820,190],{"className":11821},[212,213,492],[169,11823,11825],{"className":11824},[463],[169,11826,11828],{"className":11827},[467],[169,11829,11831],{"className":11830},[472],[169,11832,11835],{"className":11833,"style":11834},[476],"height:0.8278em;",[169,11836,11838,11842],{"style":11837},"top:-2.931em;margin-right:0.0714em;",[169,11839],{"className":11840,"style":11841},[484],"height:2.5em;",[169,11843,11847],{"className":11844},[489,11845,11846,492],"reset-size3","size1",[169,11848,11850],{"className":11849},[212,492],[169,11851,11386],{"className":11852},[212,492],[169,11854,637],{"className":11855},[518,492],[169,11857,1744],{"className":11858,"style":1771},[212,213,492],[169,11860,500],{"className":11861},[499],[169,11863,11865],{"className":11864},[472],[169,11866,11869],{"className":11867,"style":11868},[476],"height:0.5592em;",[169,11870],{},[169,11872,11873,11876],{"style":7749},[169,11874],{"className":11875,"style":999},[484],[169,11877,11878],{},[169,11879,7553],{"className":11880},[1483,7758],[169,11882,500],{"className":11883},[499],[169,11885,11887],{"className":11886},[472],[169,11888,11891],{"className":11889,"style":11890},[476],"height:1.3989em;",[169,11892],{},[169,11894],{"className":11895,"style":1622},[513],[169,11897,11899,11902,11905,11908,11911,11914,11917,11920,11923,11926,11929,11932,11935,11967,11970,12003,12006],{"className":11898},[2019],[169,11900,1896],{"className":11901,"style":2027},[535,2026],[169,11903,5619],{"className":11904,"style":823},[212,213],[169,11906,431],{"className":11907},[535],[169,11909,190],{"className":11910},[212,213],[169,11912,2505],{"className":11913},[2534],[169,11915],{"className":11916,"style":1622},[513],[169,11918,18],{"className":11919},[212,213],[169,11921,440],{"className":11922},[579],[169,11924],{"className":11925,"style":2981},[513],[169,11927,2878],{"className":11928},[2985],[169,11930],{"className":11931,"style":2981},[513],[169,11933,6506],{"className":11934,"style":6536},[212,213],[169,11936,11938,11941],{"className":11937},[212],[169,11939,8073],{"className":11940,"style":2981},[212,213],[169,11942,11944],{"className":11943},[463],[169,11945,11947],{"className":11946},[467],[169,11948,11950],{"className":11949},[472],[169,11951,11953],{"className":11952,"style":8296},[476],[169,11954,11955,11958],{"style":6753},[169,11956],{"className":11957,"style":485},[484],[169,11959,11961],{"className":11960},[489,490,491,492],[169,11962,11964],{"className":11963},[212,492],[169,11965,597],{"className":11966,"style":613},[212,213,492],[169,11968,431],{"className":11969},[535],[169,11971,11973,11976],{"className":11972},[212],[169,11974,190],{"className":11975},[212,213],[169,11977,11979],{"className":11978},[463],[169,11980,11982],{"className":11981},[467],[169,11983,11985],{"className":11984},[472],[169,11986,11989],{"className":11987,"style":11988},[476],"height:0.8019em;",[169,11990,11991,11994],{"style":6753},[169,11992],{"className":11993,"style":485},[484],[169,11995,11997],{"className":11996},[489,490,491,492],[169,11998,12000],{"className":11999},[212,492],[169,12001,11386],{"className":12002},[212,492],[169,12004,440],{"className":12005},[579],[169,12007,1911],{"className":12008,"style":2027},[579,2026],[169,12010,12011,12014],{"style":11640},[169,12012],{"className":12013,"style":11593},[484],[169,12015,12017,12020,12023,12026,12029,12118,12121],{"className":12016},[212],[169,12018],{"className":12019},[212],[169,12021],{"className":12022,"style":514},[513],[169,12024,425],{"className":12025},[518],[169,12027],{"className":12028,"style":514},[513],[169,12030,12032],{"className":12031},[212],[169,12033,12035],{"className":12034},[1483,3731],[169,12036,12038,12109],{"className":12037},[467,468],[169,12039,12041,12106],{"className":12040},[472],[169,12042,12044,12096],{"className":12043,"style":7725},[476],[169,12045,12047,12050],{"style":12046},"top:-2.3557em;margin-left:0em;",[169,12048],{"className":12049,"style":999},[484],[169,12051,12053],{"className":12052},[489,490,491,492],[169,12054,12056,12090,12093],{"className":12055},[212,492],[169,12057,12059,12062],{"className":12058},[212,492],[169,12060,190],{"className":12061},[212,213,492],[169,12063,12065],{"className":12064},[463],[169,12066,12068],{"className":12067},[467],[169,12069,12071],{"className":12070},[472],[169,12072,12075],{"className":12073,"style":12074},[476],"height:0.6828em;",[169,12076,12078,12081],{"style":12077},"top:-2.786em;margin-right:0.0714em;",[169,12079],{"className":12080,"style":11841},[484],[169,12082,12084],{"className":12083},[489,11845,11846,492],[169,12085,12087],{"className":12086},[212,492],[169,12088,11386],{"className":12089},[212,492],[169,12091,637],{"className":12092},[518,492],[169,12094,1744],{"className":12095,"style":1771},[212,213,492],[169,12097,12098,12101],{"style":7749},[169,12099],{"className":12100,"style":999},[484],[169,12102,12103],{},[169,12104,7553],{"className":12105},[1483,7758],[169,12107,500],{"className":12108},[499],[169,12110,12112],{"className":12111},[472],[169,12113,12116],{"className":12114,"style":12115},[476],"height:0.7443em;",[169,12117],{},[169,12119],{"className":12120,"style":1622},[513],[169,12122,12124,12131,12134,12137,12140,12143,12146,12149,12152,12155,12158,12161,12164,12251,12254,12371],{"className":12123},[2019],[169,12125,12127],{"className":12126,"style":2027},[535,2026],[169,12128,1896],{"className":12129},[3726,12130],"size2",[169,12132,5619],{"className":12133,"style":823},[212,213],[169,12135,431],{"className":12136},[535],[169,12138,190],{"className":12139},[212,213],[169,12141,2505],{"className":12142},[2534],[169,12144],{"className":12145,"style":1622},[513],[169,12147,18],{"className":12148},[212,213],[169,12150,440],{"className":12151},[579],[169,12153],{"className":12154,"style":2981},[513],[169,12156,2878],{"className":12157},[2985],[169,12159],{"className":12160,"style":2981},[513],[169,12162,6506],{"className":12163,"style":6536},[212,213],[169,12165,12167],{"className":12166},[212],[169,12168,12170],{"className":12169},[1483,3731],[169,12171,12173,12242],{"className":12172},[467,468],[169,12174,12176,12239],{"className":12175},[472],[169,12177,12179,12229],{"className":12178,"style":7725},[476],[169,12180,12182,12185],{"style":12181},"top:-2.356em;margin-left:0em;",[169,12183],{"className":12184,"style":999},[484],[169,12186,12188],{"className":12187},[489,490,491,492],[169,12189,12191,12223,12226],{"className":12190},[212,492],[169,12192,12194,12197],{"className":12193},[212,492],[169,12195,18],{"className":12196},[212,213,492],[169,12198,12200],{"className":12199},[463],[169,12201,12203],{"className":12202},[467],[169,12204,12206],{"className":12205},[472],[169,12207,12209],{"className":12208,"style":12074},[476],[169,12210,12211,12214],{"style":12077},[169,12212],{"className":12213,"style":11841},[484],[169,12215,12217],{"className":12216},[489,11845,11846,492],[169,12218,12220],{"className":12219},[212,492],[169,12221,11386],{"className":12222},[212,492],[169,12224,637],{"className":12225},[518,492],[169,12227,597],{"className":12228,"style":613},[212,213,492],[169,12230,12231,12234],{"style":7749},[169,12232],{"className":12233,"style":999},[484],[169,12235,12236],{},[169,12237,7553],{"className":12238},[1483,7758],[169,12240,500],{"className":12241},[499],[169,12243,12245],{"className":12244},[472],[169,12246,12249],{"className":12247,"style":12248},[476],"height:0.744em;",[169,12250],{},[169,12252],{"className":12253,"style":1622},[513],[169,12255,12257,12260,12292,12295,12327,12330,12333,12365,12368],{"className":12256},[2019],[169,12258,1896],{"className":12259,"style":2027},[535,2026],[169,12261,12263,12266],{"className":12262},[212],[169,12264,8497],{"className":12265},[212,213],[169,12267,12269],{"className":12268},[463],[169,12270,12272],{"className":12271},[467],[169,12273,12275],{"className":12274},[472],[169,12276,12278],{"className":12277,"style":8296},[476],[169,12279,12280,12283],{"style":6753},[169,12281],{"className":12282,"style":485},[484],[169,12284,12286],{"className":12285},[489,490,491,492],[169,12287,12289],{"className":12288},[212,492],[169,12290,597],{"className":12291,"style":613},[212,213,492],[169,12293,431],{"className":12294},[535],[169,12296,12298,12301],{"className":12297},[212],[169,12299,190],{"className":12300},[212,213],[169,12302,12304],{"className":12303},[463],[169,12305,12307],{"className":12306},[467],[169,12308,12310],{"className":12309},[472],[169,12311,12313],{"className":12312,"style":11988},[476],[169,12314,12315,12318],{"style":6753},[169,12316],{"className":12317,"style":485},[484],[169,12319,12321],{"className":12320},[489,490,491,492],[169,12322,12324],{"className":12323},[212,492],[169,12325,11386],{"className":12326},[212,492],[169,12328,2505],{"className":12329},[2534],[169,12331],{"className":12332,"style":1622},[513],[169,12334,12336,12339],{"className":12335},[212],[169,12337,18],{"className":12338},[212,213],[169,12340,12342],{"className":12341},[463],[169,12343,12345],{"className":12344},[467],[169,12346,12348],{"className":12347},[472],[169,12349,12351],{"className":12350,"style":11988},[476],[169,12352,12353,12356],{"style":6753},[169,12354],{"className":12355,"style":485},[484],[169,12357,12359],{"className":12358},[489,490,491,492],[169,12360,12362],{"className":12361},[212,492],[169,12363,11386],{"className":12364},[212,492],[169,12366,440],{"className":12367},[579],[169,12369,1911],{"className":12370,"style":2027},[579,2026],[169,12372,12374],{"className":12373,"style":2027},[579,2026],[169,12375,1911],{"className":12376},[3726,12130],[169,12378,500],{"className":12379},[499],[169,12381,12383],{"className":12382},[472],[169,12384,12386],{"className":12385,"style":11706},[476],[169,12387],{},[14,12389,3067,12390,12475,12476,12598,12599,12661,12662,12475,12713,12789,12790,12475,12874,13017],{},[169,12391,12393,12415],{"className":12392},[172],[169,12394,12396],{"className":12395},[176],[178,12397,12398],{"xmlns":180},[182,12399,12400,12412],{},[185,12401,12402,12408,12410],{},[3502,12403,12404,12406],{},[188,12405,190],{},[423,12407,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,12409,637],{},[188,12411,1744],{},[192,12413,12414],{"encoding":194},"s' \\sim P",[169,12416,12418,12466],{"className":12417,"ariaHidden":199},[198],[169,12419,12421,12425,12457,12460,12463],{"className":12420},[203],[169,12422],{"className":12423,"style":12424},[207],"height:0.7519em;",[169,12426,12428,12431],{"className":12427},[212],[169,12429,190],{"className":12430},[212,213],[169,12432,12434],{"className":12433},[463],[169,12435,12437],{"className":12436},[467],[169,12438,12440],{"className":12439},[472],[169,12441,12443],{"className":12442,"style":12424},[476],[169,12444,12445,12448],{"style":4025},[169,12446],{"className":12447,"style":485},[484],[169,12449,12451],{"className":12450},[489,490,491,492],[169,12452,12454],{"className":12453},[212,492],[169,12455,11386],{"className":12456},[212,492],[169,12458],{"className":12459,"style":514},[513],[169,12461,637],{"className":12462},[518],[169,12464],{"className":12465,"style":514},[513],[169,12467,12469,12472],{"className":12468},[203],[169,12470],{"className":12471,"style":2182},[207],[169,12473,1744],{"className":12474,"style":1771},[212,213]," 是 ",[169,12477,12479,12515],{"className":12478},[172],[169,12480,12482],{"className":12481},[176],[178,12483,12484],{"xmlns":180},[182,12485,12486,12512],{},[185,12487,12488,12494,12496,12498,12500,12502,12504,12506,12508,12510],{},[3502,12489,12490,12492],{},[188,12491,190],{},[423,12493,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,12495,637],{},[188,12497,1744],{},[423,12499,431],{"stretchy":430},[423,12501,644],{},[188,12503,648],{"mathvariant":647},[188,12505,190],{},[423,12507,2505],{"separator":199},[188,12509,18],{},[423,12511,440],{"stretchy":430},[192,12513,12514],{"encoding":194},"s' \\sim P(\\cdot|s,a)",[169,12516,12518,12565],{"className":12517,"ariaHidden":199},[198],[169,12519,12521,12524,12556,12559,12562],{"className":12520},[203],[169,12522],{"className":12523,"style":12424},[207],[169,12525,12527,12530],{"className":12526},[212],[169,12528,190],{"className":12529},[212,213],[169,12531,12533],{"className":12532},[463],[169,12534,12536],{"className":12535},[467],[169,12537,12539],{"className":12538},[472],[169,12540,12542],{"className":12541,"style":12424},[476],[169,12543,12544,12547],{"style":4025},[169,12545],{"className":12546,"style":485},[484],[169,12548,12550],{"className":12549},[489,490,491,492],[169,12551,12553],{"className":12552},[212,492],[169,12554,11386],{"className":12555},[212,492],[169,12557],{"className":12558,"style":514},[513],[169,12560,637],{"className":12561},[518],[169,12563],{"className":12564,"style":514},[513],[169,12566,12568,12571,12574,12577,12580,12583,12586,12589,12592,12595],{"className":12567},[203],[169,12569],{"className":12570,"style":528},[207],[169,12572,1744],{"className":12573,"style":1771},[212,213],[169,12575,431],{"className":12576},[535],[169,12578,644],{"className":12579},[212],[169,12581,648],{"className":12582},[212],[169,12584,190],{"className":12585},[212,213],[169,12587,2505],{"className":12588},[2534],[169,12590],{"className":12591,"style":1622},[513],[169,12593,18],{"className":12594},[212,213],[169,12596,440],{"className":12597},[579]," 的简写，表示下一状态 ",[169,12600,12602,12620],{"className":12601},[172],[169,12603,12605],{"className":12604},[176],[178,12606,12607],{"xmlns":180},[182,12608,12609,12617],{},[185,12610,12611],{},[3502,12612,12613,12615],{},[188,12614,190],{},[423,12616,11386],{"mathvariant":647,"lspace":875,"rspace":875},[192,12618,12619],{"encoding":194},"s'",[169,12621,12623],{"className":12622,"ariaHidden":199},[198],[169,12624,12626,12629],{"className":12625},[203],[169,12627],{"className":12628,"style":12424},[207],[169,12630,12632,12635],{"className":12631},[212],[169,12633,190],{"className":12634},[212,213],[169,12636,12638],{"className":12637},[463],[169,12639,12641],{"className":12640},[467],[169,12642,12644],{"className":12643},[472],[169,12645,12647],{"className":12646,"style":12424},[476],[169,12648,12649,12652],{"style":4025},[169,12650],{"className":12651,"style":485},[484],[169,12653,12655],{"className":12654},[489,490,491,492],[169,12656,12658],{"className":12657},[212,492],[169,12659,11386],{"className":12660},[212,492]," 从环境的转移规则中采样；",[169,12663,12665,12683],{"className":12664},[172],[169,12666,12668],{"className":12667},[176],[178,12669,12670],{"xmlns":180},[182,12671,12672,12680],{},[185,12673,12674,12676,12678],{},[188,12675,18],{},[423,12677,637],{},[188,12679,597],{},[192,12681,12682],{"encoding":194},"a \\sim \\pi",[169,12684,12686,12704],{"className":12685,"ariaHidden":199},[198],[169,12687,12689,12692,12695,12698,12701],{"className":12688},[203],[169,12690],{"className":12691,"style":208},[207],[169,12693,18],{"className":12694},[212,213],[169,12696],{"className":12697,"style":514},[513],[169,12699,637],{"className":12700},[518],[169,12702],{"className":12703,"style":514},[513],[169,12705,12707,12710],{"className":12706},[203],[169,12708],{"className":12709,"style":208},[207],[169,12711,597],{"className":12712,"style":613},[212,213],[169,12714,12716,12744],{"className":12715},[172],[169,12717,12719],{"className":12718},[176],[178,12720,12721],{"xmlns":180},[182,12722,12723,12741],{},[185,12724,12725,12727,12729,12731,12733,12735,12737,12739],{},[188,12726,18],{},[423,12728,637],{},[188,12730,597],{},[423,12732,431],{"stretchy":430},[423,12734,644],{},[188,12736,648],{"mathvariant":647},[188,12738,190],{},[423,12740,440],{"stretchy":430},[192,12742,12743],{"encoding":194},"a \\sim \\pi(\\cdot|s)",[169,12745,12747,12765],{"className":12746,"ariaHidden":199},[198],[169,12748,12750,12753,12756,12759,12762],{"className":12749},[203],[169,12751],{"className":12752,"style":208},[207],[169,12754,18],{"className":12755},[212,213],[169,12757],{"className":12758,"style":514},[513],[169,12760,637],{"className":12761},[518],[169,12763],{"className":12764,"style":514},[513],[169,12766,12768,12771,12774,12777,12780,12783,12786],{"className":12767},[203],[169,12769],{"className":12770,"style":528},[207],[169,12772,597],{"className":12773,"style":613},[212,213],[169,12775,431],{"className":12776},[535],[169,12778,644],{"className":12779},[212],[169,12781,648],{"className":12782},[212],[169,12784,190],{"className":12785},[212,213],[169,12787,440],{"className":12788},[579]," 的简写；",[169,12791,12793,12815],{"className":12792},[172],[169,12794,12796],{"className":12795},[176],[178,12797,12798],{"xmlns":180},[182,12799,12800,12812],{},[185,12801,12802,12808,12810],{},[3502,12803,12804,12806],{},[188,12805,18],{},[423,12807,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,12809,637],{},[188,12811,597],{},[192,12813,12814],{"encoding":194},"a' \\sim \\pi",[169,12816,12818,12865],{"className":12817,"ariaHidden":199},[198],[169,12819,12821,12824,12856,12859,12862],{"className":12820},[203],[169,12822],{"className":12823,"style":12424},[207],[169,12825,12827,12830],{"className":12826},[212],[169,12828,18],{"className":12829},[212,213],[169,12831,12833],{"className":12832},[463],[169,12834,12836],{"className":12835},[467],[169,12837,12839],{"className":12838},[472],[169,12840,12842],{"className":12841,"style":12424},[476],[169,12843,12844,12847],{"style":4025},[169,12845],{"className":12846,"style":485},[484],[169,12848,12850],{"className":12849},[489,490,491,492],[169,12851,12853],{"className":12852},[212,492],[169,12854,11386],{"className":12855},[212,492],[169,12857],{"className":12858,"style":514},[513],[169,12860,637],{"className":12861},[518],[169,12863],{"className":12864,"style":514},[513],[169,12866,12868,12871],{"className":12867},[203],[169,12869],{"className":12870,"style":208},[207],[169,12872,597],{"className":12873,"style":613},[212,213],[169,12875,12877,12913],{"className":12876},[172],[169,12878,12880],{"className":12879},[176],[178,12881,12882],{"xmlns":180},[182,12883,12884,12910],{},[185,12885,12886,12892,12894,12896,12898,12900,12902,12908],{},[3502,12887,12888,12890],{},[188,12889,18],{},[423,12891,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,12893,637],{},[188,12895,597],{},[423,12897,431],{"stretchy":430},[423,12899,644],{},[188,12901,648],{"mathvariant":647},[3502,12903,12904,12906],{},[188,12905,190],{},[423,12907,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,12909,440],{"stretchy":430},[192,12911,12912],{"encoding":194},"a' \\sim \\pi(\\cdot|s')",[169,12914,12916,12963],{"className":12915,"ariaHidden":199},[198],[169,12917,12919,12922,12954,12957,12960],{"className":12918},[203],[169,12920],{"className":12921,"style":12424},[207],[169,12923,12925,12928],{"className":12924},[212],[169,12926,18],{"className":12927},[212,213],[169,12929,12931],{"className":12930},[463],[169,12932,12934],{"className":12933},[467],[169,12935,12937],{"className":12936},[472],[169,12938,12940],{"className":12939,"style":12424},[476],[169,12941,12942,12945],{"style":4025},[169,12943],{"className":12944,"style":485},[484],[169,12946,12948],{"className":12947},[489,490,491,492],[169,12949,12951],{"className":12950},[212,492],[169,12952,11386],{"className":12953},[212,492],[169,12955],{"className":12956,"style":514},[513],[169,12958,637],{"className":12959},[518],[169,12961],{"className":12962,"style":514},[513],[169,12964,12966,12970,12973,12976,12979,12982,13014],{"className":12965},[203],[169,12967],{"className":12968,"style":12969},[207],"height:1.0019em;vertical-align:-0.25em;",[169,12971,597],{"className":12972,"style":613},[212,213],[169,12974,431],{"className":12975},[535],[169,12977,644],{"className":12978},[212],[169,12980,648],{"className":12981},[212],[169,12983,12985,12988],{"className":12984},[212],[169,12986,190],{"className":12987},[212,213],[169,12989,12991],{"className":12990},[463],[169,12992,12994],{"className":12993},[467],[169,12995,12997],{"className":12996},[472],[169,12998,13000],{"className":12999,"style":12424},[476],[169,13001,13002,13005],{"style":4025},[169,13003],{"className":13004,"style":485},[484],[169,13006,13008],{"className":13007},[489,490,491,492],[169,13009,13011],{"className":13010},[212,492],[169,13012,11386],{"className":13013},[212,492],[169,13015,440],{"className":13016},[579]," 的简写。",[14,13019,13020],{},"最优价值函数的贝尔曼方程为：",[169,13022,13024],{"className":13023},[400],[169,13025,13027,13251],{"className":13026},[172],[169,13028,13030],{"className":13029},[176],[178,13031,13032],{"xmlns":180,"display":409},[182,13033,13034,13248],{},[871,13035,13036,13134],{"rowspacing":873,"columnalign":874,"columnspacing":875},[877,13037,13038,13056],{},[880,13039,13040],{},[883,13041,13042],{"scriptlevel":885,"displaystyle":199},[185,13043,13044,13050,13052,13054],{},[3502,13045,13046,13048],{},[188,13047,8073],{},[423,13049,7811],{},[423,13051,431],{"stretchy":430},[188,13053,190],{},[423,13055,440],{"stretchy":430},[880,13057,13058],{},[883,13059,13060],{"scriptlevel":885,"displaystyle":199},[185,13061,13062,13064,13066,13076,13096],{},[185,13063],{},[423,13065,425],{},[7546,13067,13068,13074],{},[185,13069,13070,13072],{},[188,13071,7825],{},[423,13073,1467],{},[188,13075,18],{},[188,13077,13078],{},[7546,13079,13080,13084],{},[423,13081,13082],{},[188,13083,7553],{"mathvariant":7552},[185,13085,13086,13092,13094],{},[3502,13087,13088,13090],{},[188,13089,190],{},[423,13091,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,13093,637],{},[188,13095,1744],{},[185,13097,13098,13100,13102,13104,13106,13108,13110,13112,13114,13116,13122,13124,13130,13132],{},[423,13099,1896],{"fence":199},[188,13101,5619],{},[423,13103,431],{"stretchy":430},[188,13105,190],{},[423,13107,2505],{"separator":199},[188,13109,18],{},[423,13111,440],{"stretchy":430},[423,13113,2878],{},[188,13115,6506],{},[3502,13117,13118,13120],{},[188,13119,8073],{},[423,13121,7811],{},[423,13123,431],{"stretchy":430},[3502,13125,13126,13128],{},[188,13127,190],{},[423,13129,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,13131,440],{"stretchy":430},[423,13133,1911],{"fence":199},[877,13135,13136,13158],{},[880,13137,13138],{},[883,13139,13140],{"scriptlevel":885,"displaystyle":199},[185,13141,13142,13148,13150,13152,13154,13156],{},[3502,13143,13144,13146],{},[188,13145,8497],{},[423,13147,7811],{},[423,13149,431],{"stretchy":430},[188,13151,190],{},[423,13153,2505],{"separator":199},[188,13155,18],{},[423,13157,440],{"stretchy":430},[880,13159,13160],{},[883,13161,13162],{"scriptlevel":885,"displaystyle":199},[185,13163,13164,13166,13168,13188],{},[185,13165],{},[423,13167,425],{},[188,13169,13170],{},[7546,13171,13172,13176],{},[423,13173,13174],{},[188,13175,7553],{"mathvariant":7552},[185,13177,13178,13184,13186],{},[3502,13179,13180,13182],{},[188,13181,190],{},[423,13183,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,13185,637],{},[188,13187,1744],{},[185,13189,13190,13192,13194,13196,13198,13200,13202,13204,13206,13208,13222,13228,13230,13236,13238,13244,13246],{},[423,13191,1896],{"fence":199},[188,13193,5619],{},[423,13195,431],{"stretchy":430},[188,13197,190],{},[423,13199,2505],{"separator":199},[188,13201,18],{},[423,13203,440],{"stretchy":430},[423,13205,2878],{},[188,13207,6506],{},[7546,13209,13210,13216],{},[185,13211,13212,13214],{},[188,13213,7825],{},[423,13215,1467],{},[3502,13217,13218,13220],{},[188,13219,18],{},[423,13221,11386],{"mathvariant":647,"lspace":875,"rspace":875},[3502,13223,13224,13226],{},[188,13225,8497],{},[423,13227,7811],{},[423,13229,431],{"stretchy":430},[3502,13231,13232,13234],{},[188,13233,190],{},[423,13235,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,13237,2505],{"separator":199},[3502,13239,13240,13242],{},[188,13241,18],{},[423,13243,11386],{"mathvariant":647,"lspace":875,"rspace":875},[423,13245,440],{"stretchy":430},[423,13247,1911],{"fence":199},[192,13249,13250],{"encoding":194},"\\begin{aligned}\nV^*(s) &= \\max_a \\underset{s'\\sim P}{\\mathbb{E}}\\left[r(s,a) + \\gamma V^*(s')\\right] \\\\\nQ^*(s,a) &= \\underset{s'\\sim P}{\\mathbb{E}}\\left[r(s,a) + \\gamma \\max_{a'} Q^*(s',a')\\right]\n\\end{aligned}",[169,13252,13254],{"className":13253,"ariaHidden":199},[198],[169,13255,13257,13261],{"className":13256},[203],[169,13258],{"className":13259,"style":13260},[207],"height:3.7787em;vertical-align:-1.6393em;",[169,13262,13264],{"className":13263},[212],[169,13265,13267,13395],{"className":13266},[871],[169,13268,13270],{"className":13269},[982],[169,13271,13273,13386],{"className":13272},[467,468],[169,13274,13276,13383],{"className":13275},[472],[169,13277,13280,13327],{"className":13278,"style":13279},[476],"height:2.1393em;",[169,13281,13283,13286],{"style":13282},"top:-4.4493em;",[169,13284],{"className":13285,"style":11593},[484],[169,13287,13289,13318,13321,13324],{"className":13288},[212],[169,13290,13292,13295],{"className":13291},[212],[169,13293,8073],{"className":13294,"style":2981},[212,213],[169,13296,13298],{"className":13297},[463],[169,13299,13301],{"className":13300},[467],[169,13302,13304],{"className":13303},[472],[169,13305,13307],{"className":13306,"style":7850},[476],[169,13308,13309,13312],{"style":6753},[169,13310],{"className":13311,"style":485},[484],[169,13313,13315],{"className":13314},[489,490,491,492],[169,13316,7811],{"className":13317},[2985,492],[169,13319,431],{"className":13320},[535],[169,13322,190],{"className":13323},[212,213],[169,13325,440],{"className":13326},[579],[169,13328,13330,13333],{"style":13329},"top:-2.255em;",[169,13331],{"className":13332,"style":11593},[484],[169,13334,13336,13365,13368,13371,13374,13377,13380],{"className":13335},[212],[169,13337,13339,13342],{"className":13338},[212],[169,13340,8497],{"className":13341},[212,213],[169,13343,13345],{"className":13344},[463],[169,13346,13348],{"className":13347},[467],[169,13349,13351],{"className":13350},[472],[169,13352,13354],{"className":13353,"style":7850},[476],[169,13355,13356,13359],{"style":6753},[169,13357],{"className":13358,"style":485},[484],[169,13360,13362],{"className":13361},[489,490,491,492],[169,13363,7811],{"className":13364},[2985,492],[169,13366,431],{"className":13367},[535],[169,13369,190],{"className":13370},[212,213],[169,13372,2505],{"className":13373},[2534],[169,13375],{"className":13376,"style":1622},[513],[169,13378,18],{"className":13379},[212,213],[169,13381,440],{"className":13382},[579],[169,13384,500],{"className":13385},[499],[169,13387,13389],{"className":13388},[472],[169,13390,13393],{"className":13391,"style":13392},[476],"height:1.6393em;",[169,13394],{},[169,13396,13398],{"className":13397},[1107],[169,13399,13401,14017],{"className":13400},[467,468],[169,13402,13404,14014],{"className":13403},[472],[169,13405,13407,13671],{"className":13406,"style":13279},[476],[169,13408,13409,13412],{"style":13282},[169,13410],{"className":13411,"style":11593},[484],[169,13413,13415,13418,13421,13424,13427,13471,13474,13559,13562],{"className":13414},[212],[169,13416],{"className":13417},[212],[169,13419],{"className":13420,"style":514},[513],[169,13422,425],{"className":13423},[518],[169,13425],{"className":13426,"style":514},[513],[169,13428,13430],{"className":13429},[1483,3731],[169,13431,13433,13463],{"className":13432},[467,468],[169,13434,13436,13460],{"className":13435},[472],[169,13437,13439,13450],{"className":13438,"style":208},[476],[169,13440,13441,13444],{"style":7728},[169,13442],{"className":13443,"style":999},[484],[169,13445,13447],{"className":13446},[489,490,491,492],[169,13448,18],{"className":13449},[212,213,492],[169,13451,13452,13455],{"style":7749},[169,13453],{"className":13454,"style":999},[484],[169,13456,13457],{},[169,13458,7825],{"className":13459},[1483],[169,13461,500],{"className":13462},[499],[169,13464,13466],{"className":13465},[472],[169,13467,13469],{"className":13468,"style":7768},[476],[169,13470],{},[169,13472],{"className":13473,"style":1622},[513],[169,13475,13477],{"className":13476},[212],[169,13478,13480],{"className":13479},[1483,3731],[169,13481,13483,13551],{"className":13482},[467,468],[169,13484,13486,13548],{"className":13485},[472],[169,13487,13489,13538],{"className":13488,"style":7725},[476],[169,13490,13491,13494],{"style":12046},[169,13492],{"className":13493,"style":999},[484],[169,13495,13497],{"className":13496},[489,490,491,492],[169,13498,13500,13532,13535],{"className":13499},[212,492],[169,13501,13503,13506],{"className":13502},[212,492],[169,13504,190],{"className":13505},[212,213,492],[169,13507,13509],{"className":13508},[463],[169,13510,13512],{"className":13511},[467],[169,13513,13515],{"className":13514},[472],[169,13516,13518],{"className":13517,"style":12074},[476],[169,13519,13520,13523],{"style":12077},[169,13521],{"className":13522,"style":11841},[484],[169,13524,13526],{"className":13525},[489,11845,11846,492],[169,13527,13529],{"className":13528},[212,492],[169,13530,11386],{"className":13531},[212,492],[169,13533,637],{"className":13534},[518,492],[169,13536,1744],{"className":13537,"style":1771},[212,213,492],[169,13539,13540,13543],{"style":7749},[169,13541],{"className":13542,"style":999},[484],[169,13544,13545],{},[169,13546,7553],{"className":13547},[1483,7758],[169,13549,500],{"className":13550},[499],[169,13552,13554],{"className":13553},[472],[169,13555,13557],{"className":13556,"style":12115},[476],[169,13558],{},[169,13560],{"className":13561,"style":1622},[513],[169,13563,13565,13568,13571,13574,13577,13580,13583,13586,13589,13592,13595,13598,13601,13630,13633,13665,13668],{"className":13564},[2019],[169,13566,1896],{"className":13567,"style":2027},[535,2026],[169,13569,5619],{"className":13570,"style":823},[212,213],[169,13572,431],{"className":13573},[535],[169,13575,190],{"className":13576},[212,213],[169,13578,2505],{"className":13579},[2534],[169,13581],{"className":13582,"style":1622},[513],[169,13584,18],{"className":13585},[212,213],[169,13587,440],{"className":13588},[579],[169,13590],{"className":13591,"style":2981},[513],[169,13593,2878],{"className":13594},[2985],[169,13596],{"className":13597,"style":2981},[513],[169,13599,6506],{"className":13600,"style":6536},[212,213],[169,13602,13604,13607],{"className":13603},[212],[169,13605,8073],{"className":13606,"style":2981},[212,213],[169,13608,13610],{"className":13609},[463],[169,13611,13613],{"className":13612},[467],[169,13614,13616],{"className":13615},[472],[169,13617,13619],{"className":13618,"style":7850},[476],[169,13620,13621,13624],{"style":6753},[169,13622],{"className":13623,"style":485},[484],[169,13625,13627],{"className":13626},[489,490,491,492],[169,13628,7811],{"className":13629},[2985,492],[169,13631,431],{"className":13632},[535],[169,13634,13636,13639],{"className":13635},[212],[169,13637,190],{"className":13638},[212,213],[169,13640,13642],{"className":13641},[463],[169,13643,13645],{"className":13644},[467],[169,13646,13648],{"className":13647},[472],[169,13649,13651],{"className":13650,"style":11988},[476],[169,13652,13653,13656],{"style":6753},[169,13654],{"className":13655,"style":485},[484],[169,13657,13659],{"className":13658},[489,490,491,492],[169,13660,13662],{"className":13661},[212,492],[169,13663,11386],{"className":13664},[212,492],[169,13666,440],{"className":13667},[579],[169,13669,1911],{"className":13670,"style":2027},[579,2026],[169,13672,13673,13676],{"style":13329},[169,13674],{"className":13675,"style":11593},[484],[169,13677,13679,13682,13685,13688,13691,13776,13779],{"className":13678},[212],[169,13680],{"className":13681},[212],[169,13683],{"className":13684,"style":514},[513],[169,13686,425],{"className":13687},[518],[169,13689],{"className":13690,"style":514},[513],[169,13692,13694],{"className":13693},[212],[169,13695,13697],{"className":13696},[1483,3731],[169,13698,13700,13768],{"className":13699},[467,468],[169,13701,13703,13765],{"className":13702},[472],[169,13704,13706,13755],{"className":13705,"style":7725},[476],[169,13707,13708,13711],{"style":12046},[169,13709],{"className":13710,"style":999},[484],[169,13712,13714],{"className":13713},[489,490,491,492],[169,13715,13717,13749,13752],{"className":13716},[212,492],[169,13718,13720,13723],{"className":13719},[212,492],[169,13721,190],{"className":13722},[212,213,492],[169,13724,13726],{"className":13725},[463],[169,13727,13729],{"className":13728},[467],[169,13730,13732],{"className":13731},[472],[169,13733,13735],{"className":13734,"style":12074},[476],[169,13736,13737,13740],{"style":12077},[169,13738],{"className":13739,"style":11841},[484],[169,13741,13743],{"className":13742},[489,11845,11846,492],[169,13744,13746],{"className":13745},[212,492],[169,13747,11386],{"className":13748},[212,492],[169,13750,637],{"className":13751},[518,492],[169,13753,1744],{"className":13754,"style":1771},[212,213,492],[169,13756,13757,13760],{"style":7749},[169,13758],{"className":13759,"style":999},[484],[169,13761,13762],{},[169,13763,7553],{"className":13764},[1483,7758],[169,13766,500],{"className":13767},[499],[169,13769,13771],{"className":13770},[472],[169,13772,13774],{"className":13773,"style":12115},[476],[169,13775],{},[169,13777],{"className":13778,"style":1622},[513],[169,13780,13782,13788,13791,13794,13797,13800,13803,13806,13809,13812,13815,13818,13821,13824,13900,13903,13932,13935,13967,13970,13973,14005,14008],{"className":13781},[2019],[169,13783,13785],{"className":13784,"style":2027},[535,2026],[169,13786,1896],{"className":13787},[3726,12130],[169,13789,5619],{"className":13790,"style":823},[212,213],[169,13792,431],{"className":13793},[535],[169,13795,190],{"className":13796},[212,213],[169,13798,2505],{"className":13799},[2534],[169,13801],{"className":13802,"style":1622},[513],[169,13804,18],{"className":13805},[212,213],[169,13807,440],{"className":13808},[579],[169,13810],{"className":13811,"style":2981},[513],[169,13813,2878],{"className":13814},[2985],[169,13816],{"className":13817,"style":2981},[513],[169,13819,6506],{"className":13820,"style":6536},[212,213],[169,13822],{"className":13823,"style":1622},[513],[169,13825,13827],{"className":13826},[1483,3731],[169,13828,13830,13892],{"className":13829},[467,468],[169,13831,13833,13889],{"className":13832},[472],[169,13834,13836,13879],{"className":13835,"style":208},[476],[169,13837,13838,13841],{"style":12181},[169,13839],{"className":13840,"style":999},[484],[169,13842,13844],{"className":13843},[489,490,491,492],[169,13845,13847],{"className":13846},[212,492],[169,13848,13850,13853],{"className":13849},[212,492],[169,13851,18],{"className":13852},[212,213,492],[169,13854,13856],{"className":13855},[463],[169,13857,13859],{"className":13858},[467],[169,13860,13862],{"className":13861},[472],[169,13863,13865],{"className":13864,"style":12074},[476],[169,13866,13867,13870],{"style":12077},[169,13868],{"className":13869,"style":11841},[484],[169,13871,13873],{"className":13872},[489,11845,11846,492],[169,13874,13876],{"className":13875},[212,492],[169,13877,11386],{"className":13878},[212,492],[169,13880,13881,13884],{"style":7749},[169,13882],{"className":13883,"style":999},[484],[169,13885,13886],{},[169,13887,7825],{"className":13888},[1483],[169,13890,500],{"className":13891},[499],[169,13893,13895],{"className":13894},[472],[169,13896,13898],{"className":13897,"style":12248},[476],[169,13899],{},[169,13901],{"className":13902,"style":1622},[513],[169,13904,13906,13909],{"className":13905},[212],[169,13907,8497],{"className":13908},[212,213],[169,13910,13912],{"className":13911},[463],[169,13913,13915],{"className":13914},[467],[169,13916,13918],{"className":13917},[472],[169,13919,13921],{"className":13920,"style":7850},[476],[169,13922,13923,13926],{"style":6753},[169,13924],{"className":13925,"style":485},[484],[169,13927,13929],{"className":13928},[489,490,491,492],[169,13930,7811],{"className":13931},[2985,492],[169,13933,431],{"className":13934},[535],[169,13936,13938,13941],{"className":13937},[212],[169,13939,190],{"className":13940},[212,213],[169,13942,13944],{"className":13943},[463],[169,13945,13947],{"className":13946},[467],[169,13948,13950],{"className":13949},[472],[169,13951,13953],{"className":13952,"style":11988},[476],[169,13954,13955,13958],{"style":6753},[169,13956],{"className":13957,"style":485},[484],[169,13959,13961],{"className":13960},[489,490,491,492],[169,13962,13964],{"className":13963},[212,492],[169,13965,11386],{"className":13966},[212,492],[169,13968,2505],{"className":13969},[2534],[169,13971],{"className":13972,"style":1622},[513],[169,13974,13976,13979],{"className":13975},[212],[169,13977,18],{"className":13978},[212,213],[169,13980,13982],{"className":13981},[463],[169,13983,13985],{"className":13984},[467],[169,13986,13988],{"className":13987},[472],[169,13989,13991],{"className":13990,"style":11988},[476],[169,13992,13993,13996],{"style":6753},[169,13994],{"className":13995,"style":485},[484],[169,13997,13999],{"className":13998},[489,490,491,492],[169,14000,14002],{"className":14001},[212,492],[169,14003,11386],{"className":14004},[212,492],[169,14006,440],{"className":14007},[579],[169,14009,14011],{"className":14010,"style":2027},[579,2026],[169,14012,1911],{"className":14013},[3726,12130],[169,14015,500],{"className":14016},[499],[169,14018,14020],{"className":14019},[472],[169,14021,14023],{"className":14022,"style":13392},[476],[169,14024],{},[14,14026,14027,14028,74,14059,14089],{},"同策略价值函数与最优价值函数的贝尔曼方程之间，关键区别在对动作取不取 ",[169,14029,14031,14047],{"className":14030},[172],[169,14032,14034],{"className":14033},[176],[178,14035,14036],{"xmlns":180},[182,14037,14038,14044],{},[185,14039,14040,14042],{},[188,14041,7825],{},[423,14043,1467],{},[192,14045,14046],{"encoding":194},"\\max",[169,14048,14050],{"className":14049,"ariaHidden":199},[198],[169,14051,14053,14056],{"className":14052},[203],[169,14054],{"className":14055,"style":208},[207],[169,14057,7825],{"className":14058},[1483],[169,14060,14062,14077],{"className":14061},[172],[169,14063,14065],{"className":14064},[176],[178,14066,14067],{"xmlns":180},[182,14068,14069,14075],{},[185,14070,14071,14073],{},[188,14072,7825],{},[423,14074,1467],{},[192,14076,14046],{"encoding":194},[169,14078,14080],{"className":14079,"ariaHidden":199},[198],[169,14081,14083,14086],{"className":14082},[203],[169,14084],{"className":14085,"style":208},[207],[169,14087,7825],{"className":14088},[1483]," 的存在反映了这样一个事实：每当智能体有机会自己挑动作，要想最优地行动，就必须挑那个带来最高价值的动作。",[10,14091,14092],{"title":264},[14,14093,14094],{},"「贝尔曼备份（Bellman backup）」这个词在 RL 文献中出现得非常频繁。一个状态或状态-动作对的贝尔曼备份，就是贝尔曼方程的右边：奖励加下一状态的价值。",[159,14096,14097],{"id":14097},"优势函数",[14,14099,14100,14101,14104,14105,14108],{},"有时在 RL 中，我们不需要绝对地描述一个动作有多好，只需要知道它平均而言比别的动作好多少。也就是说，我们想知道该动作的相对",[117,14102,14103],{},"优势（advantage）","。我们用",[117,14106,14107],{},"优势函数（advantage function）"," 把这个概念精确化。",[14,14110,14111,14112,14140,14141,14232,14233,14261,14262,14290,14291,14345,14346,14374],{},"对应策略 ",[169,14113,14115,14128],{"className":14114},[172],[169,14116,14118],{"className":14117},[176],[178,14119,14120],{"xmlns":180},[182,14121,14122,14126],{},[185,14123,14124],{},[188,14125,597],{},[192,14127,600],{"encoding":194},[169,14129,14131],{"className":14130,"ariaHidden":199},[198],[169,14132,14134,14137],{"className":14133},[203],[169,14135],{"className":14136,"style":208},[207],[169,14138,597],{"className":14139,"style":613},[212,213]," 的优势函数 ",[169,14142,14144,14173],{"className":14143},[172],[169,14145,14147],{"className":14146},[176],[178,14148,14149],{"xmlns":180},[182,14150,14151,14170],{},[185,14152,14153,14160,14162,14164,14166,14168],{},[3502,14154,14155,14158],{},[188,14156,14157],{},"A",[188,14159,597],{},[423,14161,431],{"stretchy":430},[188,14163,190],{},[423,14165,2505],{"separator":199},[188,14167,18],{},[423,14169,440],{"stretchy":430},[192,14171,14172],{"encoding":194},"A^{\\pi}(s,a)",[169,14174,14176],{"className":14175,"ariaHidden":199},[198],[169,14177,14179,14182,14214,14217,14220,14223,14226,14229],{"className":14178},[203],[169,14180],{"className":14181,"style":528},[207],[169,14183,14185,14188],{"className":14184},[212],[169,14186,14157],{"className":14187},[212,213],[169,14189,14191],{"className":14190},[463],[169,14192,14194],{"className":14193},[467],[169,14195,14197],{"className":14196},[472],[169,14198,14200],{"className":14199,"style":8112},[476],[169,14201,14202,14205],{"style":4025},[169,14203],{"className":14204,"style":485},[484],[169,14206,14208],{"className":14207},[489,490,491,492],[169,14209,14211],{"className":14210},[212,492],[169,14212,597],{"className":14213,"style":613},[212,213,492],[169,14215,431],{"className":14216},[535],[169,14218,190],{"className":14219},[212,213],[169,14221,2505],{"className":14222},[2534],[169,14224],{"className":14225,"style":1622},[513],[169,14227,18],{"className":14228},[212,213],[169,14230,440],{"className":14231},[579]," 描述的是：在状态 ",[169,14234,14236,14249],{"className":14235},[172],[169,14237,14239],{"className":14238},[176],[178,14240,14241],{"xmlns":180},[182,14242,14243,14247],{},[185,14244,14245],{},[188,14246,190],{},[192,14248,190],{"encoding":194},[169,14250,14252],{"className":14251,"ariaHidden":199},[198],[169,14253,14255,14258],{"className":14254},[203],[169,14256],{"className":14257,"style":208},[207],[169,14259,190],{"className":14260},[212,213]," 采取特定动作 ",[169,14263,14265,14278],{"className":14264},[172],[169,14266,14268],{"className":14267},[176],[178,14269,14270],{"xmlns":180},[182,14271,14272,14276],{},[185,14273,14274],{},[188,14275,18],{},[192,14277,18],{"encoding":194},[169,14279,14281],{"className":14280,"ariaHidden":199},[198],[169,14282,14284,14287],{"className":14283},[203],[169,14285],{"className":14286,"style":208},[207],[169,14288,18],{"className":14289},[212,213],"，比按 ",[169,14292,14294,14318],{"className":14293},[172],[169,14295,14297],{"className":14296},[176],[178,14298,14299],{"xmlns":180},[182,14300,14301,14315],{},[185,14302,14303,14305,14307,14309,14311,14313],{},[188,14304,597],{},[423,14306,431],{"stretchy":430},[423,14308,644],{},[188,14310,648],{"mathvariant":647},[188,14312,190],{},[423,14314,440],{"stretchy":430},[192,14316,14317],{"encoding":194},"\\pi(\\cdot|s)",[169,14319,14321],{"className":14320,"ariaHidden":199},[198],[169,14322,14324,14327,14330,14333,14336,14339,14342],{"className":14323},[203],[169,14325],{"className":14326,"style":528},[207],[169,14328,597],{"className":14329,"style":613},[212,213],[169,14331,431],{"className":14332},[535],[169,14334,644],{"className":14335},[212],[169,14337,648],{"className":14338},[212],[169,14340,190],{"className":14341},[212,213],[169,14343,440],{"className":14344},[579]," 随机选一个动作好多少；这里假设之后一直按 ",[169,14347,14349,14362],{"className":14348},[172],[169,14350,14352],{"className":14351},[176],[178,14353,14354],{"xmlns":180},[182,14355,14356,14360],{},[185,14357,14358],{},[188,14359,597],{},[192,14361,600],{"encoding":194},[169,14363,14365],{"className":14364,"ariaHidden":199},[198],[169,14366,14368,14371],{"className":14367},[203],[169,14369],{"className":14370,"style":208},[207],[169,14372,597],{"className":14373,"style":613},[212,213]," 行动。数学上，优势函数定义为：",[169,14376,14378],{"className":14377},[400],[169,14379,14381,14441],{"className":14380},[172],[169,14382,14384],{"className":14383},[176],[178,14385,14386],{"xmlns":180,"display":409},[182,14387,14388,14438],{},[185,14389,14390,14396,14398,14400,14402,14404,14406,14408,14414,14416,14418,14420,14422,14424,14426,14432,14434,14436],{},[3502,14391,14392,14394],{},[188,14393,14157],{},[188,14395,597],{},[423,14397,431],{"stretchy":430},[188,14399,190],{},[423,14401,2505],{"separator":199},[188,14403,18],{},[423,14405,440],{"stretchy":430},[423,14407,425],{},[3502,14409,14410,14412],{},[188,14411,8497],{},[188,14413,597],{},[423,14415,431],{"stretchy":430},[188,14417,190],{},[423,14419,2505],{"separator":199},[188,14421,18],{},[423,14423,440],{"stretchy":430},[423,14425,2499],{},[3502,14427,14428,14430],{},[188,14429,8073],{},[188,14431,597],{},[423,14433,431],{"stretchy":430},[188,14435,190],{},[423,14437,440],{"stretchy":430},[192,14439,14440],{"encoding":194},"A^{\\pi}(s,a) = Q^{\\pi}(s,a) - V^{\\pi}(s)",[169,14442,14444,14509,14574],{"className":14443,"ariaHidden":199},[198],[169,14445,14447,14450,14482,14485,14488,14491,14494,14497,14500,14503,14506],{"className":14446},[203],[169,14448],{"className":14449,"style":528},[207],[169,14451,14453,14456],{"className":14452},[212],[169,14454,14157],{"className":14455},[212,213],[169,14457,14459],{"className":14458},[463],[169,14460,14462],{"className":14461},[467],[169,14463,14465],{"className":14464},[472],[169,14466,14468],{"className":14467,"style":8296},[476],[169,14469,14470,14473],{"style":6753},[169,14471],{"className":14472,"style":485},[484],[169,14474,14476],{"className":14475},[489,490,491,492],[169,14477,14479],{"className":14478},[212,492],[169,14480,597],{"className":14481,"style":613},[212,213,492],[169,14483,431],{"className":14484},[535],[169,14486,190],{"className":14487},[212,213],[169,14489,2505],{"className":14490},[2534],[169,14492],{"className":14493,"style":1622},[513],[169,14495,18],{"className":14496},[212,213],[169,14498,440],{"className":14499},[579],[169,14501],{"className":14502,"style":514},[513],[169,14504,425],{"className":14505},[518],[169,14507],{"className":14508,"style":514},[513],[169,14510,14512,14515,14547,14550,14553,14556,14559,14562,14565,14568,14571],{"className":14511},[203],[169,14513],{"className":14514,"style":528},[207],[169,14516,14518,14521],{"className":14517},[212],[169,14519,8497],{"className":14520},[212,213],[169,14522,14524],{"className":14523},[463],[169,14525,14527],{"className":14526},[467],[169,14528,14530],{"className":14529},[472],[169,14531,14533],{"className":14532,"style":8296},[476],[169,14534,14535,14538],{"style":6753},[169,14536],{"className":14537,"style":485},[484],[169,14539,14541],{"className":14540},[489,490,491,492],[169,14542,14544],{"className":14543},[212,492],[169,14545,597],{"className":14546,"style":613},[212,213,492],[169,14548,431],{"className":14549},[535],[169,14551,190],{"className":14552},[212,213],[169,14554,2505],{"className":14555},[2534],[169,14557],{"className":14558,"style":1622},[513],[169,14560,18],{"className":14561},[212,213],[169,14563,440],{"className":14564},[579],[169,14566],{"className":14567,"style":2981},[513],[169,14569,2499],{"className":14570},[2985],[169,14572],{"className":14573,"style":2981},[513],[169,14575,14577,14580,14612,14615,14618],{"className":14576},[203],[169,14578],{"className":14579,"style":528},[207],[169,14581,14583,14586],{"className":14582},[212],[169,14584,8073],{"className":14585,"style":2981},[212,213],[169,14587,14589],{"className":14588},[463],[169,14590,14592],{"className":14591},[467],[169,14593,14595],{"className":14594},[472],[169,14596,14598],{"className":14597,"style":8296},[476],[169,14599,14600,14603],{"style":6753},[169,14601],{"className":14602,"style":485},[484],[169,14604,14606],{"className":14605},[489,490,491,492],[169,14607,14609],{"className":14608},[212,492],[169,14610,597],{"className":14611,"style":613},[212,213,492],[169,14613,431],{"className":14614},[535],[169,14616,190],{"className":14617},[212,213],[169,14619,440],{"className":14620},[579],[10,14622,14623],{"title":264},[14,14624,14625],{},"这一点后面还会细讲：优势函数对策略梯度方法至关重要。",[56,14627,14629],{"id":14628},"选读形式化","（选读）形式化",[14,14631,14632,14633,14636,14637,14773],{},"到目前为止，我们一直在非正式地讨论智能体所处的环境。但如果去翻文献，你很可能会遇到这个设定下的标准数学形式化：",[117,14634,14635],{},"MDP（Markov Decision Process，马尔可夫决策过程）","。一个 MDP 是五元组 ",[169,14638,14640,14681],{"className":14639},[172],[169,14641,14643],{"className":14642},[176],[178,14644,14645],{"xmlns":180},[182,14646,14647,14678],{},[185,14648,14649,14652,14655,14657,14659,14661,14663,14665,14667,14669,14675],{},[423,14650,14651],{"stretchy":430},"⟨",[188,14653,14654],{},"S",[423,14656,2505],{"separator":199},[188,14658,14157],{},[423,14660,2505],{"separator":199},[188,14662,5583],{},[423,14664,2505],{"separator":199},[188,14666,1744],{},[423,14668,2505],{"separator":199},[415,14670,14671,14673],{},[188,14672,4573],{},[2781,14674,885],{},[423,14676,14677],{"stretchy":430},"⟩",[192,14679,14680],{"encoding":194},"\\langle S, A, R, P, \\rho_0 \\rangle",[169,14682,14684],{"className":14683,"ariaHidden":199},[198],[169,14685,14687,14690,14693,14697,14700,14703,14706,14709,14712,14715,14718,14721,14724,14727,14730,14770],{"className":14686},[203],[169,14688],{"className":14689,"style":528},[207],[169,14691,14651],{"className":14692},[535],[169,14694,14654],{"className":14695,"style":14696},[212,213],"margin-right:0.0576em;",[169,14698,2505],{"className":14699},[2534],[169,14701],{"className":14702,"style":1622},[513],[169,14704,14157],{"className":14705},[212,213],[169,14707,2505],{"className":14708},[2534],[169,14710],{"className":14711,"style":1622},[513],[169,14713,5583],{"className":14714,"style":5598},[212,213],[169,14716,2505],{"className":14717},[2534],[169,14719],{"className":14720,"style":1622},[513],[169,14722,1744],{"className":14723,"style":1771},[212,213],[169,14725,2505],{"className":14726},[2534],[169,14728],{"className":14729,"style":1622},[513],[169,14731,14733,14736],{"className":14732},[212],[169,14734,4573],{"className":14735},[212,213],[169,14737,14739],{"className":14738},[463],[169,14740,14742,14762],{"className":14741},[467,468],[169,14743,14745,14759],{"className":14744},[472],[169,14746,14748],{"className":14747,"style":4305},[476],[169,14749,14750,14753],{"style":480},[169,14751],{"className":14752,"style":485},[484],[169,14754,14756],{"className":14755},[489,490,491,492],[169,14757,885],{"className":14758},[212,492],[169,14760,500],{"className":14761},[499],[169,14763,14765],{"className":14764},[472],[169,14766,14768],{"className":14767,"style":507},[476],[169,14769],{},[169,14771,14677],{"className":14772},[579],"，其中：",[35,14775,14776,14807,14838,15233,15568],{},[38,14777,14778,14806],{},[169,14779,14781,14794],{"className":14780},[172],[169,14782,14784],{"className":14783},[176],[178,14785,14786],{"xmlns":180},[182,14787,14788,14792],{},[185,14789,14790],{},[188,14791,14654],{},[192,14793,14654],{"encoding":194},[169,14795,14797],{"className":14796,"ariaHidden":199},[198],[169,14798,14800,14803],{"className":14799},[203],[169,14801],{"className":14802,"style":2182},[207],[169,14804,14654],{"className":14805,"style":14696},[212,213]," 是所有合法状态的集合；",[38,14808,14809,14837],{},[169,14810,14812,14825],{"className":14811},[172],[169,14813,14815],{"className":14814},[176],[178,14816,14817],{"xmlns":180},[182,14818,14819,14823],{},[185,14820,14821],{},[188,14822,14157],{},[192,14824,14157],{"encoding":194},[169,14826,14828],{"className":14827,"ariaHidden":199},[198],[169,14829,14831,14834],{"className":14830},[203],[169,14832],{"className":14833,"style":2182},[207],[169,14835,14157],{"className":14836},[212,213]," 是所有合法动作的集合；",[38,14839,14840,14961,14962,15232],{},[169,14841,14843,14876],{"className":14842},[172],[169,14844,14846],{"className":14845},[176],[178,14847,14848],{"xmlns":180},[182,14849,14850,14873],{},[185,14851,14852,14854,14857,14859,14862,14864,14866,14868,14871],{},[188,14853,5583],{},[423,14855,14856],{},":",[188,14858,14654],{},[423,14860,14861],{},"×",[188,14863,14157],{},[423,14865,14861],{},[188,14867,14654],{},[423,14869,14870],{},"→",[188,14872,5583],{"mathvariant":7552},[192,14874,14875],{"encoding":194},"R : S \\times A \\times S \\to \\mathbb{R}",[169,14877,14879,14897,14916,14934,14952],{"className":14878,"ariaHidden":199},[198],[169,14880,14882,14885,14888,14891,14894],{"className":14881},[203],[169,14883],{"className":14884,"style":2182},[207],[169,14886,5583],{"className":14887,"style":5598},[212,213],[169,14889],{"className":14890,"style":514},[513],[169,14892,14856],{"className":14893},[518],[169,14895],{"className":14896,"style":514},[513],[169,14898,14900,14904,14907,14910,14913],{"className":14899},[203],[169,14901],{"className":14902,"style":14903},[207],"height:0.7667em;vertical-align:-0.0833em;",[169,14905,14654],{"className":14906,"style":14696},[212,213],[169,14908],{"className":14909,"style":2981},[513],[169,14911,14861],{"className":14912},[2985],[169,14914],{"className":14915,"style":2981},[513],[169,14917,14919,14922,14925,14928,14931],{"className":14918},[203],[169,14920],{"className":14921,"style":14903},[207],[169,14923,14157],{"className":14924},[212,213],[169,14926],{"className":14927,"style":2981},[513],[169,14929,14861],{"className":14930},[2985],[169,14932],{"className":14933,"style":2981},[513],[169,14935,14937,14940,14943,14946,14949],{"className":14936},[203],[169,14938],{"className":14939,"style":2182},[207],[169,14941,14654],{"className":14942,"style":14696},[212,213],[169,14944],{"className":14945,"style":514},[513],[169,14947,14870],{"className":14948},[518],[169,14950],{"className":14951,"style":514},[513],[169,14953,14955,14958],{"className":14954},[203],[169,14956],{"className":14957,"style":7725},[207],[169,14959,5583],{"className":14960},[212,7758]," 是奖励函数，",[169,14963,14965,15018],{"className":14964},[172],[169,14966,14968],{"className":14967},[176],[178,14969,14970],{"xmlns":180},[182,14971,14972,15016],{},[185,14973,14974,14980,14982,14984,14986,14992,14994,15000,15002,15014],{},[415,14975,14976,14978],{},[188,14977,5619],{},[188,14979,421],{},[423,14981,425],{},[188,14983,5583],{},[423,14985,431],{"stretchy":430},[415,14987,14988,14990],{},[188,14989,190],{},[188,14991,421],{},[423,14993,2505],{"separator":199},[415,14995,14996,14998],{},[188,14997,18],{},[188,14999,421],{},[423,15001,2505],{"separator":199},[415,15003,15004,15006],{},[188,15005,190],{},[185,15007,15008,15010,15012],{},[188,15009,421],{},[423,15011,2878],{},[2781,15013,3452],{},[423,15015,440],{"stretchy":430},[192,15017,5660],{"encoding":194},[169,15019,15021,15076],{"className":15020,"ariaHidden":199},[198],[169,15022,15024,15027,15067,15070,15073],{"className":15023},[203],[169,15025],{"className":15026,"style":453},[207],[169,15028,15030,15033],{"className":15029},[212],[169,15031,5619],{"className":15032,"style":823},[212,213],[169,15034,15036],{"className":15035},[463],[169,15037,15039,15059],{"className":15038},[467,468],[169,15040,15042,15056],{"className":15041},[472],[169,15043,15045],{"className":15044,"style":477},[476],[169,15046,15047,15050],{"style":5690},[169,15048],{"className":15049,"style":485},[484],[169,15051,15053],{"className":15052},[489,490,491,492],[169,15054,421],{"className":15055},[212,213,492],[169,15057,500],{"className":15058},[499],[169,15060,15062],{"className":15061},[472],[169,15063,15065],{"className":15064,"style":507},[476],[169,15066],{},[169,15068],{"className":15069,"style":514},[513],[169,15071,425],{"className":15072},[518],[169,15074],{"className":15075,"style":514},[513],[169,15077,15079,15082,15085,15088,15128,15131,15134,15174,15177,15180,15229],{"className":15078},[203],[169,15080],{"className":15081,"style":528},[207],[169,15083,5583],{"className":15084,"style":5598},[212,213],[169,15086,431],{"className":15087},[535],[169,15089,15091,15094],{"className":15090},[212],[169,15092,190],{"className":15093},[212,213],[169,15095,15097],{"className":15096},[463],[169,15098,15100,15120],{"className":15099},[467,468],[169,15101,15103,15117],{"className":15102},[472],[169,15104,15106],{"className":15105,"style":477},[476],[169,15107,15108,15111],{"style":480},[169,15109],{"className":15110,"style":485},[484],[169,15112,15114],{"className":15113},[489,490,491,492],[169,15115,421],{"className":15116},[212,213,492],[169,15118,500],{"className":15119},[499],[169,15121,15123],{"className":15122},[472],[169,15124,15126],{"className":15125,"style":507},[476],[169,15127],{},[169,15129,2505],{"className":15130},[2534],[169,15132],{"className":15133,"style":1622},[513],[169,15135,15137,15140],{"className":15136},[212],[169,15138,18],{"className":15139},[212,213],[169,15141,15143],{"className":15142},[463],[169,15144,15146,15166],{"className":15145},[467,468],[169,15147,15149,15163],{"className":15148},[472],[169,15150,15152],{"className":15151,"style":477},[476],[169,15153,15154,15157],{"style":480},[169,15155],{"className":15156,"style":485},[484],[169,15158,15160],{"className":15159},[489,490,491,492],[169,15161,421],{"className":15162},[212,213,492],[169,15164,500],{"className":15165},[499],[169,15167,15169],{"className":15168},[472],[169,15170,15172],{"className":15171,"style":507},[476],[169,15173],{},[169,15175,2505],{"className":15176},[2534],[169,15178],{"className":15179,"style":1622},[513],[169,15181,15183,15186],{"className":15182},[212],[169,15184,190],{"className":15185},[212,213],[169,15187,15189],{"className":15188},[463],[169,15190,15192,15221],{"className":15191},[467,468],[169,15193,15195,15218],{"className":15194},[472],[169,15196,15198],{"className":15197,"style":4305},[476],[169,15199,15200,15203],{"style":480},[169,15201],{"className":15202,"style":485},[484],[169,15204,15206],{"className":15205},[489,490,491,492],[169,15207,15209,15212,15215],{"className":15208},[212,492],[169,15210,421],{"className":15211},[212,213,492],[169,15213,2878],{"className":15214},[2985,492],[169,15216,3452],{"className":15217},[212,492],[169,15219,500],{"className":15220},[499],[169,15222,15224],{"className":15223},[472],[169,15225,15227],{"className":15226,"style":5021},[476],[169,15228],{},[169,15230,440],{"className":15231},[579],"；",[38,15234,15235,15346,15347,15447,15448,15476,15477,15505,15506,15567],{},[169,15236,15238,15270],{"className":15237},[172],[169,15239,15241],{"className":15240},[176],[178,15242,15243],{"xmlns":180},[182,15244,15245,15267],{},[185,15246,15247,15249,15251,15253,15255,15257,15259,15261,15263,15265],{},[188,15248,1744],{},[423,15250,14856],{},[188,15252,14654],{},[423,15254,14861],{},[188,15256,14157],{},[423,15258,14870],{},[188,15260,1744],{"mathvariant":2776},[423,15262,431],{"stretchy":430},[188,15264,14654],{},[423,15266,440],{"stretchy":430},[192,15268,15269],{"encoding":194},"P : S \\times A \\to \\mathcal{P}(S)",[169,15271,15273,15291,15309,15327],{"className":15272,"ariaHidden":199},[198],[169,15274,15276,15279,15282,15285,15288],{"className":15275},[203],[169,15277],{"className":15278,"style":2182},[207],[169,15280,1744],{"className":15281,"style":1771},[212,213],[169,15283],{"className":15284,"style":514},[513],[169,15286,14856],{"className":15287},[518],[169,15289],{"className":15290,"style":514},[513],[169,15292,15294,15297,15300,15303,15306],{"className":15293},[203],[169,15295],{"className":15296,"style":14903},[207],[169,15298,14654],{"className":15299,"style":14696},[212,213],[169,15301],{"className":15302,"style":2981},[513],[169,15304,14861],{"className":15305},[2985],[169,15307],{"className":15308,"style":2981},[513],[169,15310,15312,15315,15318,15321,15324],{"className":15311},[203],[169,15313],{"className":15314,"style":2182},[207],[169,15316,14157],{"className":15317},[212,213],[169,15319],{"className":15320,"style":514},[513],[169,15322,14870],{"className":15323},[518],[169,15325],{"className":15326,"style":514},[513],[169,15328,15330,15333,15337,15340,15343],{"className":15329},[203],[169,15331],{"className":15332,"style":528},[207],[169,15334,1744],{"className":15335,"style":15336},[212,2823],"margin-right:0.0822em;",[169,15338,431],{"className":15339},[535],[169,15341,14654],{"className":15342,"style":14696},[212,213],[169,15344,440],{"className":15345},[579]," 是转移概率函数（transition probability function），",[169,15348,15350,15382],{"className":15349},[172],[169,15351,15353],{"className":15352},[176],[178,15354,15355],{"xmlns":180},[182,15356,15357,15379],{},[185,15358,15359,15361,15363,15369,15371,15373,15375,15377],{},[188,15360,1744],{},[423,15362,431],{"stretchy":430},[3502,15364,15365,15367],{},[188,15366,190],{},[423,15368,11386],{"mathvariant":647,"lspace":875,"rspace":875},[188,15370,648],{"mathvariant":647},[188,15372,190],{},[423,15374,2505],{"separator":199},[188,15376,18],{},[423,15378,440],{"stretchy":430},[192,15380,15381],{"encoding":194},"P(s'|s,a)",[169,15383,15385],{"className":15384,"ariaHidden":199},[198],[169,15386,15388,15391,15394,15397,15429,15432,15435,15438,15441,15444],{"className":15387},[203],[169,15389],{"className":15390,"style":12969},[207],[169,15392,1744],{"className":15393,"style":1771},[212,213],[169,15395,431],{"className":15396},[535],[169,15398,15400,15403],{"className":15399},[212],[169,15401,190],{"className":15402},[212,213],[169,15404,15406],{"className":15405},[463],[169,15407,15409],{"className":15408},[467],[169,15410,15412],{"className":15411},[472],[169,15413,15415],{"className":15414,"style":12424},[476],[169,15416,15417,15420],{"style":4025},[169,15418],{"className":15419,"style":485},[484],[169,15421,15423],{"className":15422},[489,490,491,492],[169,15424,15426],{"className":15425},[212,492],[169,15427,11386],{"className":15428},[212,492],[169,15430,648],{"className":15431},[212],[169,15433,190],{"className":15434},[212,213],[169,15436,2505],{"className":15437},[2534],[169,15439],{"className":15440,"style":1622},[513],[169,15442,18],{"className":15443},[212,213],[169,15445,440],{"className":15446},[579]," 表示从状态 ",[169,15449,15451,15464],{"className":15450},[172],[169,15452,15454],{"className":15453},[176],[178,15455,15456],{"xmlns":180},[182,15457,15458,15462],{},[185,15459,15460],{},[188,15461,190],{},[192,15463,190],{"encoding":194},[169,15465,15467],{"className":15466,"ariaHidden":199},[198],[169,15468,15470,15473],{"className":15469},[203],[169,15471],{"className":15472,"style":208},[207],[169,15474,190],{"className":15475},[212,213]," 出发采取动作 ",[169,15478,15480,15493],{"className":15479},[172],[169,15481,15483],{"className":15482},[176],[178,15484,15485],{"xmlns":180},[182,15486,15487,15491],{},[185,15488,15489],{},[188,15490,18],{},[192,15492,18],{"encoding":194},[169,15494,15496],{"className":15495,"ariaHidden":199},[198],[169,15497,15499,15502],{"className":15498},[203],[169,15500],{"className":15501,"style":208},[207],[169,15503,18],{"className":15504},[212,213]," 后转移到状态 ",[169,15507,15509,15526],{"className":15508},[172],[169,15510,15512],{"className":15511},[176],[178,15513,15514],{"xmlns":180},[182,15515,15516,15524],{},[185,15517,15518],{},[3502,15519,15520,15522],{},[188,15521,190],{},[423,15523,11386],{"mathvariant":647,"lspace":875,"rspace":875},[192,15525,12619],{"encoding":194},[169,15527,15529],{"className":15528,"ariaHidden":199},[198],[169,15530,15532,15535],{"className":15531},[203],[169,15533],{"className":15534,"style":12424},[207],[169,15536,15538,15541],{"className":15537},[212],[169,15539,190],{"className":15540},[212,213],[169,15542,15544],{"className":15543},[463],[169,15545,15547],{"className":15546},[467],[169,15548,15550],{"className":15549},[472],[169,15551,15553],{"className":15552,"style":12424},[476],[169,15554,15555,15558],{"style":4025},[169,15556],{"className":15557,"style":485},[484],[169,15559,15561],{"className":15560},[489,490,491,492],[169,15562,15564],{"className":15563},[212,492],[169,15565,11386],{"className":15566},[212,492]," 的概率；",[38,15569,15570,15639],{},[169,15571,15573,15590],{"className":15572},[172],[169,15574,15576],{"className":15575},[176],[178,15577,15578],{"xmlns":180},[182,15579,15580,15588],{},[185,15581,15582],{},[415,15583,15584,15586],{},[188,15585,4573],{},[2781,15587,885],{},[192,15589,4578],{"encoding":194},[169,15591,15593],{"className":15592,"ariaHidden":199},[198],[169,15594,15596,15599],{"className":15595},[203],[169,15597],{"className":15598,"style":392},[207],[169,15600,15602,15605],{"className":15601},[212],[169,15603,4573],{"className":15604},[212,213],[169,15606,15608],{"className":15607},[463],[169,15609,15611,15631],{"className":15610},[467,468],[169,15612,15614,15628],{"className":15613},[472],[169,15615,15617],{"className":15616,"style":4305},[476],[169,15618,15619,15622],{"style":480},[169,15620],{"className":15621,"style":485},[484],[169,15623,15625],{"className":15624},[489,490,491,492],[169,15626,885],{"className":15627},[212,492],[169,15629,500],{"className":15630},[499],[169,15632,15634],{"className":15633},[472],[169,15635,15637],{"className":15636,"style":507},[476],[169,15638],{}," 是初始状态分布。",[14,15641,15642,15643,15648],{},"「马尔可夫决策过程」这个名字来自系统服从",[18,15644,15647],{"href":15645,"rel":15646},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FMarkov_property",[22],"马尔可夫性质（Markov property）","这一事实：转移只依赖最近的状态和动作，不依赖更早的历史。",[15650,15651,15652],"style",{},"html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s4XuR, html code.shiki .s4XuR{--shiki-default:#E36209;--shiki-dark:#FFAB70}",{"title":1375,"searchDepth":1394,"depth":1407,"links":15654},[15655,15656,15671],{"id":58,"depth":1394,"text":59},{"id":103,"depth":1394,"text":103,"children":15657},[15658,15659,15660,15664,15665,15666,15667,15668,15669,15670],{"id":161,"depth":1407,"text":161},{"id":337,"depth":1407,"text":337},{"id":358,"depth":1407,"text":358,"children":15661},[15662,15663],{"id":1357,"depth":1413,"text":1357},{"id":1534,"depth":1413,"text":1534},{"id":4156,"depth":1407,"text":4156},{"id":5565,"depth":1407,"text":5565},{"id":6827,"depth":1407,"text":6828},{"id":6823,"depth":1407,"text":6823},{"id":10530,"depth":1407,"text":10531},{"id":11281,"depth":1407,"text":11281},{"id":14097,"depth":1407,"text":14097},{"id":14628,"depth":1394,"text":14629},"技术","2026-09-28 09:29:40","OpenAI Spinning Up 教程第一部分的全文翻译：智能体与环境、状态与观测、动作空间、策略、轨迹、奖励与回报、价值函数、贝尔曼方程、优势函数，附 MDP 形式化。",false,"md",{},true,"\u002Fposts\u002Fspinning-up-rl-intro",{"title":5,"description":15674},"posts\u002Fspinning-up-rl-intro",[15683,15684,15685,15686],"强化学习","RL","Spinning Up","翻译","mCMb0v4hCj1ureftfeE3Gg4E9ItguqwA-rvDkYe87AM",[15689,15699,15704,15708,15716,15722,15726,15736,15742,15748,15750,15759,15763,15767,15771,15781,15788,15794,15798,15803,15811,15818,15826,15831,15837,15843,15848,15853,15860,15867,15874,15880,15887,15895,15901,15907,15913,15919,15924,15932],{"path":15690,"title":15691,"description":15692,"date":15693,"categories":15694,"tags":15695,"draft":15675},"\u002Fposts\u002Fsurvival-welcome-to-university","生存手册——欢迎来到大学","大学同时生产三样东西：可验证技能、认证信号和驯化资本。你付了四年的钱，真正拿到手的是哪一样，取决于你毕业时要进的那个市场在为什么付费。","2026-10-12 20:00:00","生活杂谈",[15696,15697,15698],"大学","求职","方法论",{"path":15700,"title":15701,"description":1375,"date":15702,"categories":1375,"tags":15703,"draft":15675},"\u002Fposts\u002Fgp-tab-测试","gp-tab-测试","2026-10-12 10:51:22",null,{"path":15705,"title":15706,"description":1375,"date":15707,"categories":1375,"tags":15703,"draft":15678},"\u002Fposts\u002Fnetwork-clean","怎么让网络变得干净","2026-10-11 19:40:02",{"path":15709,"title":15710,"description":15711,"date":15712,"categories":15694,"tags":15713,"draft":15675},"\u002Fposts\u002Fthe-age-of-hesitation","[搬运] 不敢投入的时代","为什么我们现在越来越不敢投入了。很多时候我们以为的选择，比如你有两个工作offer，现在选择好像是我到底应该考公还是考编。明明这两个选项都不是真正的选项，因为你还没有开始，也不知道最后能不能成。这个时候我们就已经开始犹豫了。","2026-10-10 11:44:12",[15686,15714,15715],"搬运","心理学",{"path":15717,"title":15718,"description":15719,"date":15720,"categories":15694,"tags":15721,"draft":15675},"\u002Fposts\u002Fwhy-happiness-look-like-same","[搬运] 为什么大家的幸福都长一个样？","为什么大家的幸福都长一个样？一种认知被广泛传播后似乎变得不断扁平，这种感觉很奇怪。","2026-10-09 13:57:15",[15686,15714,15715],{"path":15723,"title":15724,"description":1375,"date":15725,"categories":15694,"tags":15703,"draft":15675},"\u002Fposts\u002Feverything-is-paid-twice","任何事情都要被消费两次","2026-10-06 12:30:06",{"path":15727,"title":15728,"description":15729,"date":15730,"categories":15731,"tags":15732,"draft":15675},"\u002Fposts\u002Fwhy-llm-seems-worse-after-launch","为什么模型上线后，体感像是“降智”了？","权重与 KV cache 的量化都能降低 serving 成本，也可能改变输出；但体感变弱不足以证明服务后来切换了精度。","2026-09-29 21:44:08","技术杂谈",[15733,15734,15735],"LLMs","Serving","量化",{"path":15737,"title":15738,"description":15739,"date":15740,"categories":15672,"tags":15741,"draft":15675},"\u002Fposts\u002Fspinning-up-policy-optimization","[译] 深度强化学习入门（三）：策略优化入门","OpenAI Spinning Up 教程第三部分的全文翻译：推导最简单的策略梯度、log-derivative trick、EGLP 引理、reward-to-go 与基线，配 PyTorch 示例代码。","2026-09-28 17:49:25",[15683,15684,15685,15686],{"path":15743,"title":15744,"description":15745,"date":15746,"categories":15672,"tags":15747,"draft":15675},"\u002Fposts\u002Fspinning-up-rl-intro-2","[译] 深度强化学习入门（二）：强化学习算法的分类","OpenAI Spinning Up 第二部分的全文翻译：无模型与基于模型的 RL、策略优化与 Q-learning 的取舍，以及规划、专家迭代和经验增强等方法。","2026-09-28 10:20:21",[15683,15684,15685,15686],{"path":15679,"title":5,"description":15674,"date":15673,"categories":15672,"tags":15749,"draft":15675},[15683,15684,15685,15686],{"path":15751,"title":15752,"description":15753,"date":15754,"categories":15694,"tags":15755,"draft":15678},"\u002Fposts\u002Fwhen-machines-come-alive","冰箱之后，生活还会怎样改变？","从厨房里的果蝇想到冰箱、移动互联网与 AI 的多轮调用：AGI 已经到来，只是昂贵的智能会怎样改变生活？","2026-09-25 06:36:22",[15756,15757,15758],"AI","随笔","科技",{"path":15760,"title":15761,"description":1375,"date":15762,"categories":1375,"tags":15703,"draft":15678},"\u002Fposts\u002Ftest","test","2026-09-20 06:37:44",{"path":15764,"title":15765,"description":1375,"date":15766,"categories":1375,"tags":15703,"draft":15678},"\u002Fposts\u002Fmathematical-foundations-of-llms-1","LLM的数学基础（1）：","2026-09-19 22:59:55",{"path":15768,"title":15769,"description":1375,"date":15770,"categories":1375,"tags":15703,"draft":15678},"\u002Fposts\u002Ftowards-maximize-serving-capatity","如何在 offline bench 里最大化 LLM serving system 的吞吐","2026-09-19 05:45:05",{"path":15772,"title":15773,"description":15774,"date":15775,"categories":15731,"tags":15776,"draft":15675},"\u002Fposts\u002Fllm-memory-compute-tradeoff","LLM 的 Memory-Compute Trade-off：从 MHA 到 MLA、Sparse 与 Linear Attention","ds4.1f 上线了。它的架构 novel 在哪，便宜和快又是从哪省出来的。","2026-09-11 12:00:00",[15733,15777,15778,15779,15780],"Attention","MLA","DeepSeek","长上下文",{"path":15782,"title":15783,"description":15784,"date":15785,"categories":15731,"tags":15786,"draft":15678},"\u002Fposts\u002Fmarkdown-extensions","Markdown 扩展语法速查","本站 Markdown 支持的扩展语法演示与速查：admonition 提示块、GitHub 仓库卡片、spoiler 遮罩、ABC 乐谱、GP 乐谱、音频播放器、浏览器可运行代码块。","2026-09-01 15:30:00",[15787],"博客",{"path":15789,"title":15790,"description":15791,"date":15792,"categories":15694,"tags":15793,"draft":15675},"\u002Fposts\u002Fwhy-we-still-need-to-write-a-blog","为什么还需要手写博客？","AI这么强，还要古法写博客吗？","2026-09-01 13:17:53",[15787],{"path":15795,"title":15796,"description":1375,"date":15797,"categories":1375,"tags":15703,"draft":15678},"\u002Fposts\u002Fllm-serving-1","LLM Serving（一）:以在H200上部署GLM-5.2为例","2026-07-21 06:15:44",{"path":15799,"title":15800,"description":1375,"date":15801,"categories":15802,"tags":15703,"draft":15675},"\u002Fposts\u002Finteresting-music-rhythm","有趣的音乐节奏","2026-07-16 12:47:24","音乐",{"path":15804,"title":15805,"description":15806,"date":15807,"categories":15672,"tags":15808,"draft":15675},"\u002Fposts\u002Ftowards-monosemanticity","LLM 可解释性（一）：SAE与单语义","读 Anthropic《Towards Monosemanticity》。从难以解释的神经元出发, 讲清superposition为什么让单个神经元混成一团, SAE怎么把多语义的激活分解成单语义特征。","2026-07-12 22:00:00",[15733,15809,15810],"可解释性","SAE",{"path":15812,"title":15813,"description":15814,"date":15815,"categories":15694,"tags":15816,"draft":15678},"\u002Fposts\u002Fpiketty-ai-china-k-shaped","用皮凯蒂的镜头看 AI 与中国 K 型社会","一次凌晨的对话整理。从 r>g 出发，看 AI 时代的资本分配、中国的世袭化趋势、体制内的二元劳动力市场，以及一个普通 00 后在一线打拼的真实牌面。","2026-06-24 03:00:00",[15757,15817,15756],"经济",{"path":15819,"title":15820,"description":1375,"date":15821,"categories":1375,"tags":15822,"draft":15678},"\u002Fposts\u002Fclaude-code-compact-behavior","cluade code compact 行为","2026-06-23 20:17:42",[15823,15824,15825],"Agent","AI Agent","Claude Code",{"path":15827,"title":15828,"description":1375,"date":15829,"categories":15672,"tags":15830,"draft":15675},"\u002Fposts\u002Fclaude-code-settings","Claude Code 配置分享","2026-06-23 14:22:12",[15733,15824,15823,15825],{"path":15832,"title":15833,"description":15834,"date":15835,"categories":15731,"tags":15836,"draft":15675},"\u002Fposts\u002Fmy-thoughts-on-the-ai-industry","简单聊聊对 AI Agent 行业的理解","写这篇文章的初衷，是我发觉逐渐忘了前两个月的 AI\u002FLLMs 是什么样的能力和情况了。意识中似乎某些在突然加速，但理性和直觉告诉我这不对，作为行内人，我知道这是一个渐进的、累积的过程。我想自己回忆一下，Agent的发展。","2026-06-20 13:14:25",[15733,15824,15823],{"path":15838,"title":15839,"description":1375,"date":15840,"categories":15672,"tags":15841,"draft":15675},"\u002Fposts\u002Frotational-positional-encoding","RoPE","2026-06-17 18:31:40",[15733,15842,15839],"Transformer",{"path":15844,"title":15845,"description":1375,"date":15846,"categories":15672,"tags":15847,"draft":15675},"\u002Fposts\u002Fintroduction-to-large-language-model","大语言模型入门基础","2026-06-16 22:36:11",[15842,15733],{"path":15849,"title":15850,"description":1375,"date":15851,"categories":15731,"tags":15852,"draft":15675},"\u002Fposts\u002Fthought-about-coding-agent","关于 Coding Agent 使用的遐思","2026-06-03 17:54:20",[15823],{"path":15854,"title":15855,"description":15856,"date":15857,"categories":15672,"tags":15858,"draft":15675},"\u002Fposts\u002Fabout-ssh-usage","关于SSH的使用——客户端篇","介绍SSH客户端的用法。","2025-06-17 02:44:57",[15859],"工具",{"path":15861,"title":15862,"description":15863,"date":15864,"categories":15731,"tags":15865,"draft":15675},"\u002Fposts\u002Fopensourcesoftwarestarttutorial","如何开始一个开源项目？","浅谈我对开源项目的认知。","2025-04-17 18:00:00",[15866],"开源",{"path":15868,"title":15869,"description":15870,"date":15871,"categories":15872,"tags":15873,"draft":15675},"\u002Fposts\u002Fmotivation","渴求动力","有意识的生活并主动地行动会使得生活更美好。","2025-03-05 00:44:57","生活-情感",[15757],{"path":15875,"title":15876,"description":15877,"date":15878,"categories":15872,"tags":15879,"draft":15675},"\u002Fposts\u002F2024summary","2024年终总结","啊，痛彻的时光；啊，悲怆的人！","2025-01-01 01:00:00",[15757],{"path":15881,"title":15882,"description":15883,"date":15884,"categories":15672,"tags":15885,"draft":15675},"\u002Fposts\u002Fvps-crash-startup","快速搭建个人VPS","如何快速搭建个人VPS。","2024-12-31 19:00:00",[15886],"VPS",{"path":15888,"title":15889,"description":15890,"date":15891,"categories":15672,"tags":15892,"draft":15675},"\u002Fposts\u002Fdata-label-software","AI数据（自动）标注软件概述","对市面上目前的AI数据（自动）标注软件进行调研。","2024-12-20 14:00:00",[15893,15894],"数据标注","深度学习",{"path":15896,"title":15897,"description":15898,"date":15899,"categories":15872,"tags":15900,"draft":15675},"\u002Fposts\u002Fself-destruction","生活的崩坏与自我毁灭","是什么在牵引着我？我的行为，我的思想？我的感受？我所处的地方？它们在何处？","2024-12-19 12:00:00",[15757],{"path":15902,"title":15903,"description":15904,"date":15905,"categories":15672,"tags":15906,"draft":15675},"\u002Fposts\u002Fcs194-196_llmagents","LLM Agents 课程笔记","UCB CS294\u002F194-196 Large Language Model Agents 的课程笔记。","2024-11-30 20:00:00",[15823],{"path":15908,"title":15909,"description":15910,"date":15911,"categories":15872,"tags":15912,"draft":15675},"\u002Fposts\u002Fwhatsyourworth","价值、位置","你们的生存究竟有何价值？如果毫无价值，你们究竟为何存在？","2024-11-29 02:00:00",[15757],{"path":15914,"title":15915,"description":15916,"date":15917,"categories":15872,"tags":15918,"draft":15675},"\u002Fposts\u002Fblogrelifever2","重启博客v2","忙里偷闲学习前端，还是决定结合想法手搓一个自己的博客。","2024-11-25 00:00:00",[15787],{"path":15920,"title":15921,"description":15922,"date":15923,"categories":15872,"tags":15703,"draft":15675},"\u002Fposts\u002Fblogrelife","重启博客","在我的表达欲与现实所需(问题解决记录)驱动下，我决定重写博客。","2024-10-21 20:44:57",{"path":15925,"title":15926,"description":15927,"date":15928,"categories":15872,"tags":15929,"draft":15675},"\u002Fposts\u002Frevival","240205 复活","好吧，断更了许久，我又终于回来了，我又终于回来了。","2024-02-05 20:00:00",[15757,15930,15931],"生活","返乡",{"path":15933,"title":15934,"description":15935,"date":15936,"categories":15872,"tags":15937,"draft":15675},"\u002Fposts\u002Fclouds","「230717」云","我已经很久忘记了不我已经很少把自己的想法在lofter上发布的。","2023-07-17 20:00:00",[15757,15938],"散文",1791871062125]