39 lines
240 KiB
HTML
39 lines
240 KiB
HTML
<!DOCTYPE html>
|
||
<!-- saved from url=(0091)https://medium.com/applied-data-science/how-to-build-your-own-muzero-in-python-f77d5718061a -->
|
||
<html lang="en" data-rh="lang"><link type="text/css" id="dark-mode" rel="stylesheet" href="https://medium.com/applied-data-science/how-to-build-your-own-muzero-in-python-f77d5718061a"><style type="text/css" id="dark-mode-custom-style"></style><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><script async="" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/branch-latest.min.js"></script><script async="" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/analytics.js"></script><script defer="" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/16180790160.js"></script><title>MuZero: The Walkthrough (Part 1/3) | by David Foster | Applied Data Science | Medium</title><meta data-rh="true" name="viewport" content="width=device-width,minimum-scale=1,initial-scale=1"><meta data-rh="true" name="theme-color" content="#000000"><meta data-rh="true" name="twitter:app:name:iphone" content="Medium"><meta data-rh="true" name="twitter:app:id:iphone" content="828256236"><meta data-rh="true" property="al:ios:app_name" content="Medium"><meta data-rh="true" property="al:ios:app_store_id" content="828256236"><meta data-rh="true" property="al:android:package" content="com.medium.reader"><meta data-rh="true" property="fb:app_id" content="542599432471018"><meta data-rh="true" property="og:site_name" content="Medium"><meta data-rh="true" property="og:type" content="article"><meta data-rh="true" property="article:published_time" content="2021-02-23T09:11:45.969Z"><meta data-rh="true" name="title" content="MuZero: The Walkthrough (Part 1/3) | by David Foster | Applied Data Science | Medium"><meta data-rh="true" property="og:title" content="How To Build Your Own MuZero AI Using Python (Part 1/3)"><meta data-rh="true" property="twitter:title" content="How To Build Your Own MuZero AI Using Python (Part 1/3)"><meta data-rh="true" name="twitter:site" content="@applied_data"><meta data-rh="true" name="twitter:app:url:iphone" content="medium://p/f77d5718061a"><meta data-rh="true" property="al:android:url" content="medium://p/f77d5718061a"><meta data-rh="true" property="al:ios:url" content="medium://p/f77d5718061a"><meta data-rh="true" property="al:android:app_name" content="Medium"><meta data-rh="true" name="description" content="A Walkthrough Of The DeepMind MuZero Pseudocode"><meta data-rh="true" property="og:description" content="Teaching A Machine To Play Games Using Self-Play And Deep Learning…Without Telling It The Rules 🤯"><meta data-rh="true" property="twitter:description" content="Teaching A Machine To Play Games Using Self-Play And Deep Learning…Without Telling It The Rules 🤯"><meta data-rh="true" property="og:url" content="https://medium.com/applied-data-science/how-to-build-your-own-muzero-in-python-f77d5718061a"><meta data-rh="true" property="al:web:url" content="https://medium.com/applied-data-science/how-to-build-your-own-muzero-in-python-f77d5718061a"><meta data-rh="true" property="og:image" content="https://miro.medium.com/max/1200/1*y-FnMby1uCDxw6Br-BeZng.png"><meta data-rh="true" name="twitter:image:src" content="https://miro.medium.com/max/1200/1*y-FnMby1uCDxw6Br-BeZng.png"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="article:author" content="https://medium.com/@dtfoster"><meta data-rh="true" name="author" content="David Foster"><meta data-rh="true" name="robots" content="index,follow,max-image-preview:large"><meta data-rh="true" name="referrer" content="unsafe-url"><meta data-rh="true" name="twitter:label1" value="Reading time"><meta data-rh="true" name="twitter:data1" value="7 min read"><meta data-rh="true" name="parsely-post-id" content="f77d5718061a"><link data-rh="true" rel="search" type="application/opensearchdescription+xml" title="Medium" href="https://medium.com/osd.xml"><link data-rh="true" rel="apple-touch-icon" sizes="152x152" href="https://miro.medium.com/fit/c/152/152/1*sHhtYhaCe2Uc3IU0IgKwIQ.png"><link data-rh="true" rel="apple-touch-icon" sizes="120x120" href="https://miro.medium.com/fit/c/120/120/1*sHhtYhaCe2Uc3IU0IgKwIQ.png"><link data-rh="true" rel="apple-touch-icon" sizes="76x76" href="https://miro.medium.com/fit/c/76/76/1*sHhtYhaCe2Uc3IU0IgKwIQ.png"><link data-rh="true" rel="apple-touch-icon" sizes="60x60" href="https://miro.medium.com/fit/c/60/60/1*sHhtYhaCe2Uc3IU0IgKwIQ.png"><link data-rh="true" rel="mask-icon" href="https://cdn-static-1.medium.com/_/fp/icons/Medium-Avatar-500x500.svg" color="#171717"><link data-rh="true" id="glyph_preload_link" rel="preload" as="style" type="text/css" href="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/unbound.css"><link data-rh="true" id="glyph_link" rel="stylesheet" type="text/css" href="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/unbound.css"><link data-rh="true" rel="author" href="https://medium.com/@dtfoster"><link data-rh="true" rel="canonical" href="https://medium.com/applied-data-science/how-to-build-your-own-muzero-in-python-f77d5718061a"><link data-rh="true" rel="alternate" href="android-app://com.medium.reader/https/medium.com/p/f77d5718061a"><link data-rh="true" rel="icon" href="https://miro.medium.com/1*m-R_BkNf1Qjr1YbyOIJY2w.png"><link rel="preload" href="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/16180790160.js" as="script"><style type="text/css" data-fela-rehydration="608" data-fela-type="STATIC">html{box-sizing:border-box}*, *:before, *:after{box-sizing:inherit}body{margin:0;padding:0;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;color:rgba(0,0,0,0.8);position:relative;min-height:100vh}h1, h2, h3, h4, h5, h6, dl, dd, ol, ul, menu, figure, blockquote, p, pre, form{margin:0}menu, ol, ul{padding:0;list-style:none;list-style-image:none}main{display:block}a{color:inherit;text-decoration:none}a, button, input{-webkit-tap-highlight-color:transparent}img, svg{vertical-align:middle}button{background:transparent;overflow:visible}button, input, optgroup, select, textarea{margin:0}:root{--reach-tabs:1;--reach-menu-button:1}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="KEYFRAME">@-webkit-keyframes k1{0%{opacity:0;transform:translateY(-60px)}100%{opacity:1;transform:translateY(0px)}}@-moz-keyframes k1{0%{opacity:0;transform:translateY(-60px)}100%{opacity:1;transform:translateY(0px)}}@keyframes k1{0%{opacity:0;transform:translateY(-60px)}100%{opacity:1;transform:translateY(0px)}}@-webkit-keyframes k2{0%{opacity:1;transform:translateY(0px)}100%{opacity:0;transform:translateY(-60px)}}@-moz-keyframes k2{0%{opacity:1;transform:translateY(0px)}100%{opacity:0;transform:translateY(-60px)}}@keyframes k2{0%{opacity:1;transform:translateY(0px)}100%{opacity:0;transform:translateY(-60px)}}@-webkit-keyframes k3{0%{opacity:0.8}50%{opacity:0.5}100%{opacity:0.8}}@-moz-keyframes k3{0%{opacity:0.8}50%{opacity:0.5}100%{opacity:0.8}}@keyframes k3{0%{opacity:0.8}50%{opacity:0.5}100%{opacity:0.8}}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE">.a{font-family:medium-content-sans-serif-font, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans", "Helvetica Neue", sans-serif}.b{font-weight:400}.c{background-color:rgba(255, 255, 255, 1)}.l{height:100vh}.m{width:100vw}.n{display:flex}.o{align-items:center}.p{justify-content:center}.q{height:25px}.r{fill:rgba(41, 41, 41, 1)}.s{display:block}.t{margin-bottom:36px}.v{width:100%}.w{overflow-x:scroll}.x{white-space:nowrap}.y{scrollbar-width:none}.z{-ms-overflow-style:none}.ab::-webkit-scrollbar{display:none}.ac{box-shadow:inset 0 -1px 0 rgba(230, 230, 230, 1)}.ae{min-height:184px}.ah{flex-direction:column}.ai{background-color:#292a3aff}.aj{display:none}.al{border-bottom:none}.am{position:relative}.an{z-index:500}.at{max-width:1192px}.au{min-width:0}.av{height:62px}.aw{flex-direction:row}.ax{flex:1 0 auto}.ay{visibility:hidden}.az{margin-right:16px}.ba{font-family:sohne, "Helvetica Neue", Helvetica, Arial, sans-serif}.bb{font-size:14px}.bc{line-height:20px}.bd{color:rgba(238, 236, 242, 1)}.be{padding:7px 16px 9px}.bf{background:0}.bg{fill:rgba(238, 236, 242, 1)}.bh{border-color:rgba(219, 217, 225, 1)}.bm:disabled{cursor:inherit}.bn:disabled{opacity:0.3}.bo:disabled:hover{color:rgba(238, 236, 242, 1)}.bp:disabled:hover{fill:rgba(238, 236, 242, 1)}.bq:disabled:hover{border-color:rgba(219, 217, 225, 1)}.br{border-radius:4px}.bs{border-width:1px}.bt{border-style:solid}.bu{box-sizing:border-box}.bv{display:inline-block}.bw{text-decoration:none}.bx{margin-left:0px}.by{color:rgba(200, 198, 207, 1)}.bz{font-size:inherit}.ca{border:inherit}.cb{font-family:inherit}.cc{letter-spacing:inherit}.cd{font-weight:inherit}.ce{padding:0}.cf{margin:0}.cg:disabled{cursor:default}.ch:disabled{color:rgba(163, 208, 162, 0.5)}.ci:disabled{fill:rgba(163, 208, 162, 0.5)}.cj{min-height:115px}.ck{justify-content:space-between}.cq{align-items:flex-start}.cr{margin-bottom:0px}.cs{margin-top:-32px}.ct{flex-wrap:wrap}.cw{margin-top:32px}.cx{margin-right:24px}.cz{height:35px}.da{width:118px}.db{flex:0 0 auto}.dc{justify-self:flex-end}.dd{margin-bottom:-3px}.de{margin-left:14px}.df{margin-top:-3px}.dg{fill:rgba(255, 254, 255, 1)}.dh{padding-top:1px}.di{height:70px}.dk{font-size:16px}.dl{line-height:24px}.dm:before{margin-bottom:-10px}.dn:before{content:""}.do:before{display:table}.dp:before{border-collapse:collapse}.dq:after{margin-top:-6px}.dr:after{content:""}.ds:after{display:table}.dt:after{border-collapse:collapse}.du{color:rgba(117, 117, 117, 1)}.dv{margin-right:32px}.dw{margin-bottom:-16px}.dx{margin-top:-14px}.dy{color:rgba(255, 255, 255, 1)}.dz{fill:rgba(255, 255, 255, 1)}.ea{background:rgba(130, 132, 143, 1)}.eb{border-color:rgba(130, 132, 143, 1)}.ee:disabled:hover{background:rgba(130, 132, 143, 1)}.ef:disabled:hover{border-color:rgba(130, 132, 143, 1)}.eg{margin-right:12px}.eh{display:inline-flex}.ei{color:inherit}.ej{fill:inherit}.em:disabled{color:rgba(117, 117, 117, 1)}.en:disabled{fill:rgba(117, 117, 117, 1)}.eo{margin-left:12px}.ep{margin:0 12px}.eq{position:absolute}.er{right:24px}.es{margin:0px}.et{border:0px}.eu{padding:0px}.ev{cursor:pointer}.ew{stroke:rgba(117, 117, 117, 1)}.ez{border-top:none}.fa{left:0}.fb{opacity:0}.fc{position:fixed}.fd{right:0}.fe{top:0}.fg{height:60px}.fj{color:rgba(130, 132, 143, 1)}.fk{fill:rgba(130, 132, 143, 1)}.fn:disabled:hover{color:rgba(130, 132, 143, 1)}.fo:disabled:hover{fill:rgba(130, 132, 143, 1)}.fp{margin-left:16px}.fq{padding-left:24px}.fr{padding-right:24px}.fs{margin-left:auto}.ft{margin-right:auto}.fu{max-width:728px}.fv{background:rgba(255, 255, 255, 1)}.fw{border:1px solid rgba(230, 230, 230, 1)}.fx{box-shadow:0 1px 4px rgba(230, 230, 230, 1)}.fy{max-height:100vh}.fz{overflow-y:auto}.ga{top:calc(100vh + 100px)}.gb{bottom:calc(100vh + 100px)}.gc{width:10px}.gd{pointer-events:none}.ge{word-break:break-word}.gf{word-wrap:break-word}.gg:after{display:block}.gh:after{clear:both}.gi{max-width:680px}.gj{max-width:1211px}.gp{clear:both}.gr{cursor:zoom-in}.gs{z-index:auto}.gu{transition:opacity 100ms 400ms}.gv{height:100%}.gw{overflow:hidden}.gx{will-change:transform}.gy{transform:translateZ(0)}.gz{margin:auto}.ha{background-color:rgba(242, 242, 242, 1)}.hb{padding-bottom:54.41783649876135%}.hc{height:0}.hd{filter:blur(20px)}.he{transform:scale(1.1)}.hf{visibility:visible}.hg{line-height:1.23}.hh{letter-spacing:0}.hi{font-style:normal}.hj{font-weight:700}.ie{margin-bottom:-0.27em}.if{color:rgba(41, 41, 41, 1)}.ig{line-height:1.394}.iw{margin-bottom:-0.42em}.ja{width:28px}.jb{height:28px}.jc{fill:rgba(26, 137, 23, 1)}.jd{width:calc(100% + 24px)}.je{height:calc(100% + 24px)}.jf{top:50%}.jg{left:50%}.jh{transform:translateX(-50%) translateY(-50%)}.ji{border-radius:50%}.jj{fill:rgba(61, 61, 61, 1)}.jk{margin-top:-2px}.jl{padding-left:4px}.jm{margin:0 4px}.jn{margin:0 7px}.jo{align-items:flex-end}.jx{padding-right:8px}.jy{margin-right:8px}.jz{fill:rgba(117, 117, 117, 1)}.ka{box-shadow:inset 3px 0 0 0 rgba(41, 41, 41, 1)}.kb{padding-left:23px}.kc{margin-left:-20px}.kd{line-height:1.58}.ke{letter-spacing:-0.004em}.kf{font-style:italic}.kg{font-family:charter, Georgia, Cambria, "Times New Roman", Times, serif}.kz{margin-bottom:-0.46em}.la{text-decoration:underline}.lb{max-width:75px}.lh{padding-bottom:100%}.li{line-height:1.12}.lj{letter-spacing:-0.022em}.lk{font-weight:500}.md{margin-bottom:-0.28em}.mj{box-shadow:inset 0 0 0 1px rgba(230, 230, 230, 1)}.mk{padding:16px 20px}.ml{flex:1 1 auto}.mn{max-height:40px}.mo{text-overflow:ellipsis}.mp{display:-webkit-box}.mq{-webkit-line-clamp:2}.mr{-webkit-box-orient:vertical}.mt{margin-top:8px}.mu{margin-top:12px}.mv{font-size:13px}.mw{width:160px}.mx{background-image:url(https://miro.medium.com/max/320/1*PanQe3XaV_tpFLddTzpz3g.png)}.my{background-origin:border-box}.mz{background-size:cover}.na{height:167px}.nb{background-position:50% 50%}.nc{max-width:100%}.nd{padding:2px 4px}.ne{font-size:75%}.nf> strong{font-family:inherit}.ng{font-family:Menlo, Monaco, "Courier New", Courier, monospace}.np{padding-bottom:5px}.nq{padding-top:5px}.nr{margin-top:10px}.ns{text-align:center}.nv{padding-bottom:NaN%}.nw{line-height:1.18}.oe{margin-bottom:-0.31em}.of{max-width:1212px}.og{padding-bottom:58.003300330033%}.oh{list-style-type:disc}.oi{margin-left:30px}.oj{padding-left:0px}.op{padding-bottom:45.8298926507019%}.oq{margin-bottom:14px}.or{padding-top:24px}.os{padding-bottom:10px}.ot{background-color:rgba(8, 8, 8, 1)}.ou{height:3px}.ov{width:3px}.ow{margin-right:20px}.ox{max-width:700px}.oy{padding-bottom:27.714285714285715%}.oz{will-change:opacity}.pa{width:188px}.pb{transform:translateX(406px)}.pc{top:calc(65px + 54px + 14px)}.pf{will-change:opacity, transform}.pg{transform:translateY(159px)}.pi{width:131px}.pj{padding-bottom:28px}.pk{border-bottom:1px solid rgba(230, 230, 230, 1)}.pl{padding-top:2px}.pm{padding-top:14px}.pn{padding-top:28px}.po{margin-bottom:19px}.pp{margin-left:-3px}.pv{outline:0}.pw{border:0}.px{user-select:none}.py> svg{pointer-events:none}.qa{-webkit-user-select:none}.qk button{text-align:left}.ql{opacity:0.4}.qm{cursor:not-allowed}.qn{padding-right:9px}.qw{margin-top:40px}.qx{padding-bottom:25px}.qy{margin-top:25px}.qz{max-width:155px}.rd{top:1px}.rr{margin-left:-1px}.rs{margin-left:-4px}.sa{padding-bottom:40px}.sb{list-style-type:none}.sc{margin-bottom:8px}.sd{line-height:22px}.se{border-radius:3px}.sf{padding:5px 10px}.sg{background:rgba(242, 242, 242, 1)}.sh{padding-bottom:4px}.si{padding-top:32px}.sj{background-color:rgba(250, 250, 250, 1)}.sz{-webkit-line-clamp:1}.ta{padding-right:168px}.tb{padding-top:25px}.te{margin-bottom:96px}.tf{margin-bottom:40px}.tg{margin-top:24px}.th{padding-bottom:16px}.ti{margin-bottom:24px}.us{flex-grow:0}.ut{padding-bottom:24px}.uu{max-width:500px}.uv{flex:0 1 auto}.uz{padding-bottom:8px}.vu{padding:32px 0}.vv{background-color:rgba(0, 0, 0, 0.9)}.vz:disabled{color:rgba(255, 255, 255, 0.7)}.wa:disabled{fill:rgba(255, 255, 255, 0.7)}.wb{height:22px}.wc{padding-top:8px}.wd{width:200px}.wf{color:rgba(255, 255, 255, 0.98)}.wi{color:rgba(255, 255, 255, 0.7)}.bi:hover{color:rgba(255, 254, 255, 1)}.bj:hover{fill:rgba(255, 254, 255, 1)}.bk:hover{border-color:rgba(255, 254, 255, 1)}.bl:hover{cursor:pointer}.ec:hover{background:rgba(112, 113, 123, 1)}.ed:hover{border-color:rgba(112, 113, 123, 1)}.ek:hover{color:rgba(25, 25, 25, 1)}.el:hover{fill:rgba(25, 25, 25, 1)}.fl:hover{color:rgba(112, 113, 123, 1)}.fm:hover{fill:rgba(112, 113, 123, 1)}.qc:hover{fill:rgba(117, 117, 117, 1)}.vh:hover{text-decoration:underline}.vx:hover{color:rgba(255, 255, 255, 0.99)}.vy:hover{fill:rgba(255, 255, 255, 0.99)}.gt:focus{transform:scale(1.01)}.qb:focus{fill:rgba(117, 117, 117, 1)}.pz:active{border-style:none}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="all and (min-width: 1080px)">.d{display:none}.as{margin:0 64px}.go{margin-top:32px}.ia{font-size:46px}.ib{margin-top:0.6em}.ic{line-height:56px}.id{letter-spacing:-0.011em}.it{font-size:22px}.iu{margin-top:0.92em}.iv{line-height:28px}.jv{margin-left:30px}.kv{font-size:21px}.kw{margin-top:2em}.kx{line-height:32px}.ky{letter-spacing:-0.003em}.lg{margin-top:56px}.lz{font-size:30px}.ma{margin-top:1.95em}.mb{line-height:36px}.mc{letter-spacing:0}.mi{margin-top:0.86em}.no{max-width:1192px}.od{margin-top:1.72em}.oo{margin-top:1.05em}.pu{margin-right:5px}.qj{margin-top:5px}.qv{padding-left:6px}.rf{display:inline-block}.rk{margin-left:7px}.rl{margin-top:8px}.rq{width:25px}.ry{padding-left:7px}.rz{top:3px}.sw{font-size:20px}.sx{line-height:24px}.sy{max-height:24px}.td{margin:0}.tx{width:calc(100% + 32px)}.ty{margin-left:-16px}.tz{margin-right:-16px}.uo{padding-left:16px}.up{padding-right:16px}.uq{flex-basis:25%}.ur{max-width:25%}.ve{font-size:16px}.vf{line-height:20px}.vs{min-width:70px}.vt{min-height:70px}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="all and (max-width: 1079.98px)">.e{display:none}.ju{margin-left:30px}.nt{margin-left:auto}.nu{text-align:center}.pt{margin-right:5px}.qi{margin-top:5px}.qu{padding-left:6px}.re{display:inline-block}.ri{margin-left:7px}.rj{margin-top:8px}.rp{width:25px}.rw{padding-left:7px}.rx{top:3px}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="all and (max-width: 903.98px)">.f{display:none}.jt{margin-left:30px}.ps{margin-right:5px}.qh{margin-top:5px}.qs{padding-left:6px}.qt{top:3px}.rc{display:inline-block}.rg{margin-left:7px}.rh{margin-top:8px}.ro{width:15px}.rv{padding-left:3px}.uy{margin-right:16px}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="all and (max-width: 727.98px)">.g{display:none}.u{margin-bottom:20px}.af{box-shadow:inset 0 -1px 0 rgba(230, 230, 230, 1)}.ag{min-height:230px}.ak{display:block}.cl{min-height:98px}.cm{display:flex}.cn{align-items:flex-start}.co{flex-direction:column}.cp{justify-content:flex-end}.cu{margin-bottom:28px}.cv{margin-top:0px}.cy{margin-top:28px}.dj{margin:0}.ex{border-top:1px solid rgba(230, 230, 230, 1)}.ey{border-bottom:1px solid rgba(230, 230, 230, 1)}.fh{align-items:center}.fi{flex:1 0 auto}.iy{margin-top:32px}.iz{flex-direction:column-reverse}.jr{margin-bottom:30px}.js{margin-left:0px}.mm{padding:10px 12px 10px}.pr{margin-left:8px}.qf{margin-top:2px}.qg{margin-right:8px}.qq{padding-left:6px}.qr{top:3px}.rb{display:inline-block}.rn{width:15px}.ru{padding-left:3px}.tj{padding-bottom:12px}.tk{margin-top:16px}.ux{margin-right:16px}.vi{margin-left:16px}.vj{margin-right:0px}.vw{padding:32px 0}.we{width:140px}.wg{margin-bottom:16px}.wh{margin-top:30px}.wj{width:100%}.wk{flex-direction:row}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="all and (max-width: 551.98px)">.h{display:none}.ao{margin:0 24px}.gk{margin-top:24px}.hk{font-size:32px}.hl{margin-top:0.64em}.hm{line-height:40px}.hn{letter-spacing:-0.016em}.ih{font-size:18px}.ii{margin-top:0.79em}.ij{line-height:24px}.ix{margin-top:32px}.jp{margin-bottom:30px}.jq{margin-left:0px}.kh{margin-top:1.56em}.ki{line-height:28px}.kj{letter-spacing:-0.003em}.lc{margin-top:40px}.ll{font-size:22px}.lm{margin-top:1.2em}.ln{letter-spacing:0}.me{margin-top:0.67em}.nh{margin:0}.ni{max-width:100%}.nx{font-size:20px}.ny{margin-top:1.23em}.ok{margin-top:1.34em}.pq{margin-left:8px}.qd{margin-top:2px}.qe{margin-right:8px}.qo{padding-left:6px}.qp{top:3px}.ra{display:inline-block}.rm{width:15px}.rt{padding-left:3px}.sk{font-size:16px}.sl{line-height:20px}.sm{max-height:20px}.tl{width:calc(100% + 24px)}.tm{margin-left:-12px}.tn{margin-right:-12px}.ua{padding-left:12px}.ub{padding-right:12px}.uc{flex-basis:100%}.uw{margin-right:16px}.vg{margin-bottom:0px}.vk{min-width:48px}.vl{min-height:48px}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="all and (min-width: 904px) and (max-width: 1079.98px)">.i{display:none}.ar{margin:0 64px}.gn{margin-top:32px}.hw{font-size:46px}.hx{margin-top:0.6em}.hy{line-height:56px}.hz{letter-spacing:-0.011em}.iq{font-size:22px}.ir{margin-top:0.92em}.is{line-height:28px}.kr{font-size:21px}.ks{margin-top:2em}.kt{line-height:32px}.ku{letter-spacing:-0.003em}.lf{margin-top:56px}.lv{font-size:30px}.lw{margin-top:1.95em}.lx{line-height:36px}.ly{letter-spacing:0}.mh{margin-top:0.86em}.nn{max-width:1192px}.oc{margin-top:1.72em}.on{margin-top:1.05em}.st{font-size:20px}.su{line-height:24px}.sv{max-height:24px}.tc{margin:0}.tu{width:calc(100% + 32px)}.tv{margin-left:-16px}.tw{margin-right:-16px}.uk{padding-left:16px}.ul{padding-right:16px}.um{flex-basis:25%}.un{max-width:25%}.vc{font-size:16px}.vd{line-height:20px}.vq{min-width:70px}.vr{min-height:70px}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="all and (min-width: 728px) and (max-width: 903.98px)">.j{display:none}.aq{margin:0 48px}.gm{margin-top:32px}.hs{font-size:46px}.ht{margin-top:0.6em}.hu{line-height:56px}.hv{letter-spacing:-0.011em}.in{font-size:22px}.io{margin-top:0.92em}.ip{line-height:28px}.kn{font-size:21px}.ko{margin-top:2em}.kp{line-height:32px}.kq{letter-spacing:-0.003em}.le{margin-top:56px}.lr{font-size:30px}.ls{margin-top:1.95em}.lt{line-height:36px}.lu{letter-spacing:0}.mg{margin-top:0.86em}.nl{margin:0}.nm{max-width:100%}.ob{margin-top:1.72em}.om{margin-top:1.05em}.sq{font-size:20px}.sr{line-height:24px}.ss{max-height:24px}.tr{width:calc(100% + 28px)}.ts{margin-left:-14px}.tt{margin-right:-14px}.ug{padding-left:14px}.uh{padding-right:14px}.ui{flex-basis:50%}.uj{max-width:50%}.va{font-size:16px}.vb{line-height:20px}.vo{min-width:48px}.vp{min-height:48px}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="all and (min-width: 552px) and (max-width: 727.98px)">.k{display:none}.ap{margin:0 24px}.gl{margin-top:24px}.ho{font-size:32px}.hp{margin-top:0.64em}.hq{line-height:40px}.hr{letter-spacing:-0.016em}.ik{font-size:18px}.il{margin-top:0.79em}.im{line-height:24px}.kk{margin-top:1.56em}.kl{line-height:28px}.km{letter-spacing:-0.003em}.ld{margin-top:40px}.lo{font-size:22px}.lp{margin-top:1.2em}.lq{letter-spacing:0}.mf{margin-top:0.67em}.nj{margin:0}.nk{max-width:100%}.nz{font-size:20px}.oa{margin-top:1.23em}.ol{margin-top:1.34em}.sn{font-size:16px}.so{line-height:20px}.sp{max-height:20px}.to{width:calc(100% + 24px)}.tp{margin-left:-12px}.tq{margin-right:-12px}.ud{padding-left:12px}.ue{padding-right:12px}.uf{flex-basis:100%}.vm{min-width:48px}.vn{min-height:48px}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="print">.jw{display:none}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="(prefers-reduced-motion: no-preference)">.ff{animation:k2 .2s ease-in-out both}.gq{transition:transform 300ms cubic-bezier(0.2, 0, 0.2, 1)}.pd{transition:opacity 200ms}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="(orientation: landscape) and (max-width: 903.98px)">.ms{max-height:none}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="all and (max-width: 1230px)">.pe{display:none}</style><style type="text/css" data-fela-rehydration="608" data-fela-type="RULE" media="all and (max-width: 1198px)">.ph{display:none}</style><script type="application/ld+json" data-rh="true">{"@context":"http:\u002F\u002Fschema.org","@type":"NewsArticle","image":["https:\u002F\u002Fmiro.medium.com\u002Fmax\u002F1200\u002F1*y-FnMby1uCDxw6Br-BeZng.png"],"url":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fhow-to-build-your-own-muzero-in-python-f77d5718061a","dateCreated":"2019-12-02T05:01:20.163Z","datePublished":"2019-12-02T05:01:20.163Z","dateModified":"2021-02-23T09:11:49.158Z","headline":"MuZero: The Walkthrough (Part 1\u002F3) - Applied Data Science - Medium","name":"MuZero: The Walkthrough (Part 1\u002F3) - Applied Data Science - Medium","description":"A Walkthrough Of The DeepMind MuZero Pseudocode","identifier":"f77d5718061a","keywords":["Lite:true","Tag:Machine Learning","Tag:Deep Learning","Tag:Data Science","Tag:Artificial Intelligence","Tag:AI","Publication:applied-data-science","Elevated:false","LockedPostSource:LOCKED_POST_SOURCE_UGC_UNENROLLED","LayerCake:4"],"author":{"@type":"Person","name":"David Foster","url":"https:\u002F\u002Fmedium.com\u002F@dtfoster"},"creator":["David Foster"],"publisher":{"@type":"Organization","name":"Applied Data Science","url":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science","logo":{"@type":"ImageObject","width":201,"height":60,"url":"https:\u002F\u002Fmiro.medium.com\u002Fmax\u002F201\u002F1*2A6OcWvQBoboXH2AW0LQMw.png"}},"mainEntityOfPage":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fhow-to-build-your-own-muzero-in-python-f77d5718061a","isAccessibleForFree":"False","hasPart":{"@type":"WebPageElement","isAccessibleForFree":"False","cssSelector":".meteredContent"}}</script><script data-rh="true">(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||
ga('create', 'UA-24232453-2', 'auto');
|
||
ga('send', 'pageview');</script><script type="text/javascript" data-rh="true">(function(b,r,a,n,c,h,_,s,d,k){if(!b[n]||!b[n]._q){for(;s<_.length;)c(h,_[s++]);d=r.createElement(a);d.async=1;d.src="https://cdn.branch.io/branch-latest.min.js";k=r.getElementsByTagName(a)[0];k.parentNode.insertBefore(d,k);b[n]=h}})(window,document,"script","branch",function(b,r){b[r]=function(){b._q.push([r,arguments])}},{_q:[],_v:1},"addListener applyCode autoAppIndex banner closeBanner closeJourney creditHistory credits data deepview deepviewCta first getCode init link logout redeem referrals removeListener sendSMS setBranchViewData setIdentity track validateCode trackCommerceEvent logEvent".split(" "), 0);
|
||
branch.init('key_live_ofxXr2qTrrU9NqURK8ZwEhknBxiI6KBm', {metadata: {}, 'no_journeys': true, 'disable_exit_animation': true, 'disable_entry_animation': true, 'tracking_disabled': null}, function(err, data) {});</script><style id="googleidentityservice_button_styles">.qJTHM{-webkit-user-select:none;color:#202124;direction:ltr;-webkit-touch-callout:none;font-family:'Roboto-Regular',arial,sans-serif;-webkit-font-smoothing:antialiased;font-weight:400;margin:0;overflow:hidden;-webkit-text-size-adjust:100%}.ynRLnc{left:-9999px;position:absolute;top:-9999px}.L6cTce{display:none}.bltWBb{word-break:break-all}.hSRGPd{color:#1a73e8;cursor:pointer;font-weight:500;text-decoration:none}.Bz112c-W3lGp{height:16px;width:16px}.Bz112c-E3DyYd{height:20px;width:20px}.Bz112c-r9oPif{height:24px;width:24px}.Bz112c-uaxL4e{-webkit-border-radius:10px;border-radius:10px}.LgbsSe-Bz112c{display:block}.S9gUrf-YoZ4jf,.S9gUrf-YoZ4jf *{border:none;margin:0;padding:0}.fFW7wc-ibnC6b>.aZ2wEe>div{border-color:#4285f4}.P1ekSe-ZMv3u>div:nth-child(1){background-color:#1a73e8!important}.P1ekSe-ZMv3u>div:nth-child(2),.P1ekSe-ZMv3u>div:nth-child(3){background-image:linear-gradient(to right,rgba(255,255,255,0.7),rgba(255,255,255,0.7)),linear-gradient(to right,#1a73e8,#1a73e8)!important}.haAclf{display:inline-block}.nsm7Bb-HzV7m-LgbsSe{-webkit-border-radius:4px;border-radius:4px;-webkit-box-sizing:border-box;box-sizing:border-box;-webkit-transition:background-color .218s,border-color .218s;transition:background-color .218s,border-color .218s;-webkit-user-select:none;-webkit-appearance:none;background-color:#fff;background-image:none;border:1px solid #dadce0;color:#3c4043;cursor:pointer;font-family:'Google Sans',arial,sans-serif;font-size:14px;height:40px;letter-spacing:.25px;outline:none;overflow:hidden;padding:0 12px;position:relative;text-align:center;vertical-align:middle;white-space:nowrap;width:auto}@media screen and (-ms-high-contrast:active){.nsm7Bb-HzV7m-LgbsSe{border:2px solid windowText;color:windowText}}.nsm7Bb-HzV7m-LgbsSe.pSzOP-SxQuSe{font-size:14px;height:32px;letter-spacing:.25px;padding:0 10px}.nsm7Bb-HzV7m-LgbsSe.purZT-SxQuSe{font-size:11px;height:20px;letter-spacing:.3px;padding:0 8px}.nsm7Bb-HzV7m-LgbsSe.Bz112c-LgbsSe{padding:0;width:40px}.nsm7Bb-HzV7m-LgbsSe.Bz112c-LgbsSe.pSzOP-SxQuSe{width:32px}.nsm7Bb-HzV7m-LgbsSe.Bz112c-LgbsSe.purZT-SxQuSe{width:20px}.nsm7Bb-HzV7m-LgbsSe.JGcpL-RbRzK{-webkit-border-radius:20px;border-radius:20px}.nsm7Bb-HzV7m-LgbsSe.JGcpL-RbRzK.pSzOP-SxQuSe{-webkit-border-radius:16px;border-radius:16px}.nsm7Bb-HzV7m-LgbsSe.JGcpL-RbRzK.purZT-SxQuSe{-webkit-border-radius:10px;border-radius:10px}.nsm7Bb-HzV7m-LgbsSe.MFS4be-Ia7Qfc{border:none;color:#fff}.nsm7Bb-HzV7m-LgbsSe.MFS4be-v3pZbf-Ia7Qfc{background-color:#1a73e8}.nsm7Bb-HzV7m-LgbsSe.MFS4be-JaPV2b-Ia7Qfc{background-color:#202124;color:#e8eaed}.nsm7Bb-HzV7m-LgbsSe .nsm7Bb-HzV7m-LgbsSe-Bz112c{height:18px;margin-right:8px;min-width:18px;width:18px}.nsm7Bb-HzV7m-LgbsSe.pSzOP-SxQuSe .nsm7Bb-HzV7m-LgbsSe-Bz112c{height:14px;min-width:14px;width:14px}.nsm7Bb-HzV7m-LgbsSe.purZT-SxQuSe .nsm7Bb-HzV7m-LgbsSe-Bz112c{height:10px;min-width:10px;width:10px}.nsm7Bb-HzV7m-LgbsSe.jVeSEe .nsm7Bb-HzV7m-LgbsSe-Bz112c{margin-left:8px;margin-right:-4px}.nsm7Bb-HzV7m-LgbsSe.Bz112c-LgbsSe .nsm7Bb-HzV7m-LgbsSe-Bz112c{margin:0;padding:10px}.nsm7Bb-HzV7m-LgbsSe.Bz112c-LgbsSe.pSzOP-SxQuSe .nsm7Bb-HzV7m-LgbsSe-Bz112c{padding:8px}.nsm7Bb-HzV7m-LgbsSe.Bz112c-LgbsSe.purZT-SxQuSe .nsm7Bb-HzV7m-LgbsSe-Bz112c{padding:4px}.nsm7Bb-HzV7m-LgbsSe .nsm7Bb-HzV7m-LgbsSe-Bz112c-haAclf{-webkit-border-top-left-radius:3px;border-top-left-radius:3px;-webkit-border-bottom-left-radius:3px;border-bottom-left-radius:3px;display:-webkit-box;display:-webkit-flex;display:flex;justify-content:center;-webkit-align-items:center;align-items:center;background-color:#fff;height:36px;margin-left:-10px;margin-right:12px;min-width:36px;width:36px}.nsm7Bb-HzV7m-LgbsSe .nsm7Bb-HzV7m-LgbsSe-Bz112c-haAclf .nsm7Bb-HzV7m-LgbsSe-Bz112c,.nsm7Bb-HzV7m-LgbsSe.Bz112c-LgbsSe .nsm7Bb-HzV7m-LgbsSe-Bz112c-haAclf .nsm7Bb-HzV7m-LgbsSe-Bz112c{margin:0;padding:0}.nsm7Bb-HzV7m-LgbsSe.pSzOP-SxQuSe .nsm7Bb-HzV7m-LgbsSe-Bz112c-haAclf{height:28px;margin-left:-8px;margin-right:10px;min-width:28px;width:28px}.nsm7Bb-HzV7m-LgbsSe.purZT-SxQuSe .nsm7Bb-HzV7m-LgbsSe-Bz112c-haAclf{height:16px;margin-left:-6px;margin-right:8px;min-width:16px;width:16px}.nsm7Bb-HzV7m-LgbsSe.Bz112c-LgbsSe .nsm7Bb-HzV7m-LgbsSe-Bz112c-haAclf{-webkit-border-radius:3px;border-radius:3px;margin-left:2px;margin-right:0;padding:0}.nsm7Bb-HzV7m-LgbsSe.JGcpL-RbRzK .nsm7Bb-HzV7m-LgbsSe-Bz112c-haAclf{-webkit-border-radius:18px;border-radius:18px}.nsm7Bb-HzV7m-LgbsSe.pSzOP-SxQuSe.JGcpL-RbRzK .nsm7Bb-HzV7m-LgbsSe-Bz112c-haAclf{-webkit-border-radius:14px;border-radius:14px}.nsm7Bb-HzV7m-LgbsSe.purZT-SxQuSe.JGcpL-RbRzK .nsm7Bb-HzV7m-LgbsSe-Bz112c-haAclf{-webkit-border-radius:8px;border-radius:8px}.nsm7Bb-HzV7m-LgbsSe .nsm7Bb-HzV7m-LgbsSe-bN97Pc-sM5MNb{display:-webkit-box;display:-webkit-flex;display:flex;-webkit-align-items:center;align-items:center;-webkit-flex-direction:row;flex-direction:row;justify-content:space-between;-webkit-flex-wrap:nowrap;flex-wrap:nowrap;height:100%;position:relative;width:100%}.nsm7Bb-HzV7m-LgbsSe .oXtfBe-l4eHX{justify-content:center}.nsm7Bb-HzV7m-LgbsSe .nsm7Bb-HzV7m-LgbsSe-BPrWId{-webkit-flex-grow:1;flex-grow:1;font-family:'Google Sans',arial,sans-serif;font-weight:500;overflow:hidden;text-overflow:ellipsis;vertical-align:top}.nsm7Bb-HzV7m-LgbsSe.purZT-SxQuSe .nsm7Bb-HzV7m-LgbsSe-BPrWId{font-weight:300}.nsm7Bb-HzV7m-LgbsSe .oXtfBe-l4eHX .nsm7Bb-HzV7m-LgbsSe-BPrWId{-webkit-flex-grow:0;flex-grow:0}.nsm7Bb-HzV7m-LgbsSe .nsm7Bb-HzV7m-LgbsSe-MJoBVe{-webkit-transition:background-color .218s;transition:background-color .218s;bottom:0;left:0;position:absolute;right:0;top:0}.nsm7Bb-HzV7m-LgbsSe:hover,.nsm7Bb-HzV7m-LgbsSe:focus{-webkit-box-shadow:none;box-shadow:none;border-color:#d2e3fc;outline:none}.nsm7Bb-HzV7m-LgbsSe:hover .nsm7Bb-HzV7m-LgbsSe-MJoBVe,.nsm7Bb-HzV7m-LgbsSe:focus .nsm7Bb-HzV7m-LgbsSe-MJoBVe{background:rgba(66,133,244,0.04)}.nsm7Bb-HzV7m-LgbsSe:active .nsm7Bb-HzV7m-LgbsSe-MJoBVe{background:rgba(66,133,244,0.1)}.nsm7Bb-HzV7m-LgbsSe.MFS4be-Ia7Qfc:hover .nsm7Bb-HzV7m-LgbsSe-MJoBVe,.nsm7Bb-HzV7m-LgbsSe.MFS4be-Ia7Qfc:focus .nsm7Bb-HzV7m-LgbsSe-MJoBVe{background:rgba(255,255,255,0.24)}.nsm7Bb-HzV7m-LgbsSe.MFS4be-Ia7Qfc:active .nsm7Bb-HzV7m-LgbsSe-MJoBVe{background:rgba(255,255,255,0.32)}.nsm7Bb-HzV7m-LgbsSe .n1UuX-DkfjY{-webkit-border-radius:50%;border-radius:50%;display:-webkit-box;display:-webkit-flex;display:flex;height:20px;margin-left:-4px;margin-right:8px;min-width:20px;width:20px}.nsm7Bb-HzV7m-LgbsSe.jVeSEe .nsm7Bb-HzV7m-LgbsSe-BPrWId{font-family:'Roboto';font-size:12px;text-align:left}.nsm7Bb-HzV7m-LgbsSe.jVeSEe .nsm7Bb-HzV7m-LgbsSe-BPrWId .ssJRIf,.nsm7Bb-HzV7m-LgbsSe.jVeSEe .nsm7Bb-HzV7m-LgbsSe-BPrWId .K4efff .fmcmS{overflow:hidden;text-overflow:ellipsis}.nsm7Bb-HzV7m-LgbsSe.jVeSEe .nsm7Bb-HzV7m-LgbsSe-BPrWId .K4efff{display:-webkit-box;display:-webkit-flex;display:flex;-webkit-align-items:center;align-items:center;color:#5f6368;fill:#5f6368;font-size:11px;font-weight:400}.nsm7Bb-HzV7m-LgbsSe.jVeSEe.MFS4be-Ia7Qfc .nsm7Bb-HzV7m-LgbsSe-BPrWId .K4efff{color:#e8eaed;fill:#e8eaed}.nsm7Bb-HzV7m-LgbsSe.jVeSEe .nsm7Bb-HzV7m-LgbsSe-BPrWId .K4efff .Bz112c{height:18px;margin:-3px -3px -3px 2px;min-width:18px;width:18px}.nsm7Bb-HzV7m-LgbsSe.jVeSEe .nsm7Bb-HzV7m-LgbsSe-Bz112c-haAclf{-webkit-border-top-left-radius:0;border-top-left-radius:0;-webkit-border-bottom-left-radius:0;border-bottom-left-radius:0;-webkit-border-top-right-radius:3px;border-top-right-radius:3px;-webkit-border-bottom-right-radius:3px;border-bottom-right-radius:3px;margin-left:12px;margin-right:-10px}.nsm7Bb-HzV7m-LgbsSe.jVeSEe.JGcpL-RbRzK .nsm7Bb-HzV7m-LgbsSe-Bz112c-haAclf{-webkit-border-radius:18px;border-radius:18px}.L5Fo6c-sM5MNb{border:0;display:block;left:0;position:relative;top:0}.L5Fo6c-bF1uUb{-webkit-border-radius:4px;border-radius:4px;bottom:0;cursor:pointer;left:0;position:absolute;right:0;top:0}.L5Fo6c-bF1uUb:focus{border:none;outline:none}sentinel{}
|
||
/*# sourceURL=/_/gsi/_/ss/k=gsi.gsi.n64yMIEPTMo.L.W.O/am=cg/d=1/ct=zgms/rs=AF0KOtV4Ub2H8p50TbLsIBxnE82DWYYB8g/m=gis_client_button_style */</style><link id="googleidentityservice" type="text/css" media="all" rel="stylesheet" href="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/style"></head><body><div id="root"><div class="a b c"><div class="d e f g h i j k"></div><script>document.domain = document.domain;</script><div><script>if (window.self !== window.top) window.location = "about:blank"</script></div><script>window.PARSELY = window.PARSELY || {autotrack: false}</script><div class="s"><div class="t s u"><div class="ac ae s af ag"><div class="n ah ai"><div class="aj ak"><div class="al s am an"><div class="n p"><div class="ao ap aq ar as at au v"><div class="av n o"><div class="n o aw ax"><div class="hf" id="lo-meta-header-sign-up-button"><div class="az s"><span><button class="ba b bb bc bd be bf bg bh bi bj bk bl bm bn bo bp bq br bs bt bu bv bw">Get started</button></span></div></div><div class="hf" id="lo-ShowPostUnderCollection-navbar-open-in-app-button"><div class="bx aj ak"><span class="ba b bb bc by"><a href="https://rsci.app.link/?%24canonical_url=https%3A%2F%2Fmedium.com%2Fp%2Ff77d5718061a&~feature=LoOpenInAppButton&~channel=ShowPostUnderCollection&~stage=mobileNavBar&source=post_page-----f77d5718061a--------------------------------" class="bd bg bz ca cb cc cd ce cf bl bi bj cg ch ci" rel="noopener nofollow">Open in app</a></span></div></div></div><a aria-label="Homepage" rel="noopener" href="https://medium.com/?source=post_page-----f77d5718061a--------------------------------"><svg viewBox="0 0 1043.63 592.71" class="q bg"><g data-name="Layer 2"><g data-name="Layer 1"><path d="M588.67 296.36c0 163.67-131.78 296.35-294.33 296.35S0 460 0 296.36 131.78 0 294.34 0s294.33 132.69 294.33 296.36M911.56 296.36c0 154.06-65.89 279-147.17 279s-147.17-124.94-147.17-279 65.88-279 147.16-279 147.17 124.9 147.17 279M1043.63 296.36c0 138-23.17 249.94-51.76 249.94s-51.75-111.91-51.75-249.94 23.17-249.94 51.75-249.94 51.76 111.9 51.76 249.94"></path></g></g></svg></a></div></div></div></div></div><div class="n p"><div class="ao ap aq ar as at au v"><div class="cj n o aw ck cl cm cn co cp"><div class="v n cq ck"><div class="n v"><div class="cr cs v n o aw ct cu cv cm cn co"><div class="cw cx s cy"><a aria-label="Publication Homepage" rel="noopener" href="https://medium.com/applied-data-science?source=post_page-----f77d5718061a--------------------------------"><div class="cz da s"><img alt="Applied Data Science" class="" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_2A6OcWvQBoboXH2AW0LQMw.png" width="118" height="35"></div></a></div></div></div><div class="n o db dc an g"><div class="hf" id="lo-meta-header-sign-in-link"><p class="ba b bb bc by"><span><a href="https://medium.com/m/signin?operation=login&redirect=https%3A%2F%2Fmedium.com%2Fapplied-data-science%2Fhow-to-build-your-own-muzero-in-python-f77d5718061a&source=post_page-----f77d5718061a---------------------nav_reg-----------" class="bd bg bz ca cb cc cd ce cf bl bi bj cg ch ci" rel="noopener">Sign in</a></span></p></div><div class="hf" id="lo-meta-header-sign-up-button"><div class="dd de df cx s"><span><button class="ba b bb bc bd be bf bg bh bi bj bk bl bm bn bo bp bq br bs bt bu bv bw">Get started</button></span></div></div><a aria-label="Homepage" rel="noopener" href="https://medium.com/?source=post_page-----f77d5718061a--------------------------------"><svg viewBox="0 0 1043.63 592.71" class="q dg"><g data-name="Layer 2"><g data-name="Layer 1"><path d="M588.67 296.36c0 163.67-131.78 296.35-294.33 296.35S0 460 0 296.36 131.78 0 294.34 0s294.33 132.69 294.33 296.36M911.56 296.36c0 154.06-65.89 279-147.17 279s-147.17-124.94-147.17-279 65.88-279 147.16-279 147.17 124.9 147.17 279M1043.63 296.36c0 138-23.17 249.94-51.76 249.94s-51.75-111.91-51.75-249.94 23.17-249.94 51.75-249.94 51.76 111.9 51.76 249.94"></path></g></g></svg></a></div></div></div></div></div></div><div class="s ak"><div class="n p"><div class="ao ap aq ar as at au v"><div class="w x y z ab"><div class="dh di n o"><div class="s dj"><span class="ba b dk dl dm dn do dp dq dr ds dt du"><div class="n o"><div class="dv s"><div class="dw dx s"><div class="bv" aria-hidden="false" aria-describedby="collectionFollowPopover" aria-labelledby="collectionFollowPopover"><span><button class="ba b bb bc dy be dz ea eb ec ed bl bm bn ee ef br bs bt bu bv bw"><div class="n aw">Follow</div></button></span></div></div></div><div class="eg eh ah"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener" href="https://medium.com/applied-data-science/followers?source=post_page-----f77d5718061a--------------------------------">7.6K Followers</a></div><div class="eo s g">·</div><div class="eo s g"><nav class="n o"><span class="ep n ah"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener" href="https://medium.com/applied-data-science/write-for-applied-data-science-3615cca98e91?source=post_page-----f77d5718061a--------------------------------">Write For Us</a></span><span class="ep n ah"><a href="https://adsp.ai/?source=post_page-----f77d5718061a--------------------------------" class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" target="_blank" rel="noopener nofollow">Website</a></span></nav></div><div class="eo n ah g"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener" href="https://medium.com/applied-data-science/about?source=post_page-----f77d5718061a--------------------------------">About</a></div></div></span></div><div class="aj eq er ak"><button class="n o p es et eu ev" aria-label="Expand navbar"><svg width="14" height="14" class="ew"><path d="M0 .5h14M0 7h14M0 13.5h14"></path></svg></button></div></div></div></div></div></div></div><div class="ex ey ez al c fa xc fc fd fe hf an aco"><div class="n p"><div class="ao ap aq ar as at au v"><div class="fg v aj fe an cm fh"><div class="aj cm fh fi"><div class="hf" id="lo-sticky-header-sign-up-button"><span><button class="ba b bb bc fj be bf fk eb fl fm ed bl bm bn fn fo ef br bs bt bu bv bw">Get started</button></span></div><div class="hf" id="lo-ShowPostUnderCollection-navbar-open-in-app-button"><div class="fp aj ak"><span class="ba b bb bc du"><a href="https://rsci.app.link/?%24canonical_url=https%3A%2F%2Fmedium.com%2Fp%2Ff77d5718061a&~feature=LoOpenInAppButton&~channel=ShowPostUnderCollection&~stage=mobileNavBar&source=post_page-----f77d5718061a--------------------------------" class="fj fk bz ca cb cc cd ce cf bl fl fm cg ch ci" rel="noopener nofollow">Open in app</a></span></div></div></div><a aria-label="Homepage" rel="noopener" href="https://medium.com/?source=post_page-----f77d5718061a--------------------------------"><svg viewBox="0 0 1043.63 592.71" class="q r"><g data-name="Layer 2"><g data-name="Layer 1"><path d="M588.67 296.36c0 163.67-131.78 296.35-294.33 296.35S0 460 0 296.36 131.78 0 294.34 0s294.33 132.69 294.33 296.36M911.56 296.36c0 154.06-65.89 279-147.17 279s-147.17-124.94-147.17-279 65.88-279 147.16-279 147.17 124.9 147.17 279M1043.63 296.36c0 138-23.17 249.94-51.76 249.94s-51.75-111.91-51.75-249.94 23.17-249.94 51.75-249.94 51.76 111.9 51.76 249.94"></path></g></g></svg></a></div></div></div></div></div><div class="s jw"><div class="wq wr v gv fc ws wt ev fb wu gd" aria-hidden="true"></div><div class="wv fc ww wx wy wq gv bu wz xa xb xc xd xe xf wj xg xh xi xj aj" aria-hidden="true"><div class="xm xn n o aw ck"><h2 class="ba lk xo dl hh if">Responses (4)</h2><div class="n aw"><div><div class="bv" role="tooltip" aria-hidden="false" aria-describedby="5" aria-labelledby="5"><a href="https://policy.medium.com/medium-rules-30e5502c4eb4?source=responses-----f77d5718061a--------------------------------" class="jj el" target="_blank" rel="noopener"><svg width="25" height="25" viewBox="0 0 25 25"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.99 5.04c.26-.21.64-.22.91-.01.97.72 1.77 1.21 2.6 1.54.83.32 1.72.48 2.89.5.41.01.74.35.74.76-.02 3.62-.43 6.26-1.45 8.21-1.03 1.98-2.66 3.21-4.97 4.08a.75.75 0 0 1-.53 0c-2.25-.87-3.86-2.1-4.9-4.07-1.02-1.95-1.46-4.59-1.48-8.22 0-.41.33-.75.75-.76 1.19-.02 2.1-.18 2.92-.5.82-.32 1.6-.81 2.52-1.53zm.46.9c-.9.69-1.71 1.21-2.62 1.56a8.9 8.9 0 0 1-3.02.57c.03 3.45.46 5.82 1.36 7.51.88 1.69 2.25 2.77 4.28 3.57 2.1-.8 3.47-1.89 4.34-3.57.89-1.7 1.3-4.07 1.34-7.51a8.8 8.8 0 0 1-3-.57 11.8 11.8 0 0 1-2.68-1.56zm0 9.15a2.67 2.67 0 1 0 0-5.34 2.67 2.67 0 0 0 0 5.34zm0 1a3.67 3.67 0 1 0 0-7.34 3.67 3.67 0 0 0 0 7.34zm-1.82-3.77l.53-.53.91.92 1.63-1.63.52.53-2.15 2.15-1.44-1.44z"></path></svg></a></div></div><div class="s am xp"><div class="s am fe fd"><button class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" data-testid="close-button" aria-label="close"><svg width="25" height="25" viewBox="0 0 25 25" class="jz"><path d="M18.13 6.11l-5.61 5.61-5.6-5.61-.81.8 5.61 5.61-5.61 5.61.8.8 5.61-5.6 5.61 5.6.8-.8-5.6-5.6 5.6-5.62"></path></svg></button></div></div></div></div><div><div class="ba b bb bc if"><div class="dl"><div class="abm yc abn s"><div class="wx abf br n ah abg abh abi"><div class="n ah"><span><a href="https://medium.com/m/signin?operation=register&redirect=https%3A%2F%2Fmedium.com%2Fapplied-data-science%2Fhow-to-build-your-own-muzero-in-python-f77d5718061a&source=responses-----f77d5718061a---------------------respond_sidebar-----------" class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener"><div class="abo s"><p class="ba b bb bc du">What are your thoughts?</p></div></a></span><div class="ck n abj abk fb abl"><label class="n o"><div class="jy ace acf n acg am"><input class="abp abq abr eq fb abs abt abu abv abw abx aby" type="checkbox"><span class="o fv abz aca bu dy ev n dz acb p acc acd"><svg width="11" height="11" viewBox="0 0 11 11" class="dz"><path d="M0 6.31l3.7 3.7.9.91.67-1.1 5.3-8.79L8.84 0l-5.3 8.8 1.57-.2-3.7-3.7L0 6.3z"></path></svg></span></div><h4 class="ba b mv bc du">Also publish to my profile</h4></label><button class="ba b bb bc dy ach dz aci acj ack acl bl bm bn acm acn br bs bt bu bv bw" disabled="">Respond</button></div></div></div></div></div></div></div><div class="s"><div class="s"><div class="pk yu s"><div><div class="v gv"><div class="th tb s"><div class="n aw ck"><div class="n o aw"><div class="am yg yf"><div class="jc n aw o p eq yw yx jf jg jh gd"><svg width="39" height="39" viewBox="0 0 39 39"><path fill-rule="evenodd" clip-rule="evenodd" d="M19.5 1C11.83 1 5.17 5.75 1.9 12.71L1 12.3C4.4 5.02 11.4 0 19.5 0S34.6 5.02 38 12.29l-.9.42C33.82 5.75 27.16 1 19.5 1zM1.9 26.29C5.18 33.25 11.84 38 19.5 38c7.67 0 14.33-4.75 17.6-11.71l.9.42C34.6 33.98 27.6 39 19.5 39S4.4 33.98 1 26.71l.9-.42z"></path></svg></div><img alt="Pratyush Srivastava" class="s ji yf yg" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/0_xNChTRUKPfeYOIcZ" width="32" height="32"></div><div class="yy s"><div class="n aw"><div><div class="bv" role="tooltip" aria-hidden="false" aria-describedby="53" aria-labelledby="53"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener" href="https://medium.com/@pratyushucla?source=responses-----f77d5718061a----0----------------------------"><p class="ba b bb bc if">Pratyush Srivastava</p></a></div></div></div><a class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener nofollow" href="https://medium.com/@pratyushucla/shouldnt-it-be-dynamics-followed-by-prediction-d176e55c2efa?source=responses-----f77d5718061a----0----------------------------"><p class="ba b bb bc du">9 months ago</p></a></div></div><button class="ev pw jj el yv"><svg class="overflow-dots-filled-25px_svg__svgIcon-use" width="25" height="25"><path d="M5 12.5c0 .55.2 1.02.59 1.41.39.4.86.59 1.41.59.55 0 1.02-.2 1.41-.59.4-.39.59-.86.59-1.41 0-.55-.2-1.02-.59-1.41A1.93 1.93 0 0 0 7 10.5c-.55 0-1.02.2-1.41.59-.4.39-.59.86-.59 1.41zm5.62 0c0 .55.2 1.02.58 1.41.4.4.87.59 1.42.59.55 0 1.02-.2 1.41-.59.4-.39.59-.86.59-1.41 0-.55-.2-1.02-.59-1.41a1.93 1.93 0 0 0-1.41-.59c-.55 0-1.03.2-1.42.59-.39.39-.58.86-.58 1.41zm5.6 0c0 .55.2 1.02.58 1.41.4.4.87.59 1.43.59.56 0 1.03-.2 1.42-.59.39-.39.58-.86.58-1.41 0-.55-.2-1.02-.58-1.41a1.93 1.93 0 0 0-1.42-.59c-.56 0-1.04.2-1.43.59-.39.39-.58.86-.58 1.41z" fill-rule="evenodd"></path></svg></button></div><div role="button" tabindex="0" class="yz za se zb zc zd ev c"><p id="embedded-quote-cc481f99129a-7f8d" class="ze zf hi kg b ih zg ik zh zi zj zk zl zm zn kz ge zo if" data-selectable-paragraph=""><mark class="zp yq ev">representation followed by dynamics</mark></p></div><div class="zq s"><pre class="zr"><div class="zs s"><div class="ba b bb bc if"><div class="dl">Shouldn’t it be dynamics followed by prediction?</div></div></div></pre></div><div class="zu n o aw ck"><div class="n o"><div class="n o"><div class="s am jq js ps pt pu"><span><a href="https://medium.com/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fvote%2Fp%2Fd176e55c2efa&operation=register&redirect=https%3A%2F%2Fmedium.com%2Fapplied-data-science%2Fhow-to-build-your-own-muzero-in-python-f77d5718061a&source=responses-----d176e55c2efa----0-----------------respond_sidebar-----------" class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener"><div class="ce pv pw px ev py pz wp r qb qc"><svg width="25" height="25" viewBox="0 0 25 25" aria-label="clap"><g fill-rule="evenodd"><path d="M11.74 0l.76 2.97.76-2.97zM14.81 3.78l1.84-2.56-1.42-.47zM8.38 1.22l1.84 2.56L9.8.75zM20.38 21.62a5.11 5.11 0 0 1-3.16 1.61l.49-.45c2.88-2.89 3.45-5.98 1.69-9.21l-1.1-1.94-.96-2.02c-.31-.67-.23-1.18.25-1.55a.84.84 0 0 1 .66-.16c.34.05.66.28.88.6l2.85 5.02c1.18 1.97 1.38 5.12-1.6 8.1M7.1 21.1l-5.02-5.02a1 1 0 0 1 .7-1.7 1 1 0 0 1 .72.3l2.6 2.6a.44.44 0 0 0 .63-.62L4.1 14.04l-1.75-1.75a1 1 0 1 1 1.41-1.41l4.15 4.15a.44.44 0 0 0 .63 0 .44.44 0 0 0 0-.62L4.4 10.26 3.22 9.08a1 1 0 0 1 0-1.4 1.02 1.02 0 0 1 1.41 0l1.18 1.16L9.96 13a.44.44 0 0 0 .62 0 .44.44 0 0 0 0-.63L6.43 8.22a.99.99 0 0 1-.3-.7.99.99 0 0 1 .3-.7 1 1 0 0 1 1.41 0l7 6.98a.44.44 0 0 0 .7-.5l-1.35-2.85c-.31-.68-.23-1.19.25-1.56a.85.85 0 0 1 .66-.16c.34.06.66.28.88.6L18.63 14c1.57 2.88 1.07 5.54-1.55 8.16a5.62 5.62 0 0 1-5.06 1.65 9.35 9.35 0 0 1-4.93-2.72zM11 5.98l2.56 2.56c-.5.6-.56 1.41-.15 2.28l.26.56-4.25-4.25a.98.98 0 0 1-.12-.45 1 1 0 0 1 .29-.7 1.02 1.02 0 0 1 1.41 0zm8.89 2.06c-.38-.56-.9-.92-1.49-1.01a1.74 1.74 0 0 0-1.34.33c-.38.29-.61.65-.71 1.06a2.1 2.1 0 0 0-1.1-.56 1.78 1.78 0 0 0-.99.13l-2.64-2.64a1.88 1.88 0 0 0-2.65 0 1.86 1.86 0 0 0-.48.85 1.89 1.89 0 0 0-2.67-.01 1.87 1.87 0 0 0-.5.9c-.76-.75-2-.75-2.7-.04a1.88 1.88 0 0 0 0 2.66c-.3.12-.61.29-.87.55a1.88 1.88 0 0 0 0 2.66l.62.62a1.88 1.88 0 0 0-.9 3.16l5.01 5.02c1.6 1.6 3.52 2.64 5.4 2.96a7.16 7.16 0 0 0 1.18.1c1.03 0 2-.25 2.9-.7A5.9 5.9 0 0 0 21 22.24c3.34-3.34 3.08-6.93 1.74-9.17l-2.87-5.04z"></path></g></svg></div></a></span></div></div><div class="eo q n o"><button class="ei ej bz ca cb cc cd ce cf bl ek el cg em en"><div class="n o"><svg width="29" height="29" class="r q zt" aria-label="responses"><path d="M21.27 20.06a9.04 9.04 0 0 0 2.75-6.68C24.02 8.21 19.67 4 14.1 4S4 8.21 4 13.38c0 5.18 4.53 9.39 10.1 9.39 1 0 2-.14 2.95-.41.28.25.6.49.92.7a7.46 7.46 0 0 0 4.19 1.3c.27 0 .5-.13.6-.35a.63.63 0 0 0-.05-.65 8.08 8.08 0 0 1-1.29-2.58 5.42 5.42 0 0 1-.15-.75zm-3.85 1.32l-.08-.28-.4.12a9.72 9.72 0 0 1-2.84.43c-4.96 0-9-3.71-9-8.27 0-4.55 4.04-8.26 9-8.26 4.95 0 8.77 3.71 8.77 8.27 0 2.25-.75 4.35-2.5 5.92l-.24.21v.32a5.59 5.59 0 0 0 .21 1.29c.19.7.49 1.4.89 2.08a6.43 6.43 0 0 1-2.67-1.06c-.34-.22-.88-.48-1.16-.74z"></path></svg><span class="yh zv bv"><span class="ba b mv bc if">1 reply</span></span></div></button></div></div><div class="zw s"><p class="ba b bb bc if"><button class="ei ej bz ca cb cc cd ce cf bl vh cg em en">Reply</button></p></div></div></div></div></div></div><div class="pk yu s"><div><div class="v gv"><div class="th tb s"><div class="n aw ck"><div class="n o aw"><img alt="London L." class="s ji yf yg" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/0_oFm6Y4jNaIuHDgBE" width="32" height="32"><div class="yy s"><div class="n aw"><div><div class="bv" role="tooltip" aria-hidden="false" aria-describedby="54" aria-labelledby="54"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener" href="https://medium.com/@happylondon4?source=responses-----f77d5718061a----1----------------------------"><p class="ba b bb bc if">London L.</p></a></div></div></div><a class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener nofollow" href="https://medium.com/@happylondon4/in-the-paper-it-says-for-the-initial-step-the-representation-function-h-receives-as-input-the-84a7580ac231?source=responses-----f77d5718061a----1----------------------------"><p class="ba b bb bc du">about 1 year ago</p></a></div></div><button class="ev pw jj el yv"><svg class="overflow-dots-filled-25px_svg__svgIcon-use" width="25" height="25"><path d="M5 12.5c0 .55.2 1.02.59 1.41.39.4.86.59 1.41.59.55 0 1.02-.2 1.41-.59.4-.39.59-.86.59-1.41 0-.55-.2-1.02-.59-1.41A1.93 1.93 0 0 0 7 10.5c-.55 0-1.02.2-1.41.59-.4.39-.59.86-.59 1.41zm5.62 0c0 .55.2 1.02.58 1.41.4.4.87.59 1.42.59.55 0 1.02-.2 1.41-.59.4-.39.59-.86.59-1.41 0-.55-.2-1.02-.59-1.41a1.93 1.93 0 0 0-1.41-.59c-.55 0-1.03.2-1.42.59-.39.39-.58.86-.58 1.41zm5.6 0c0 .55.2 1.02.58 1.41.4.4.87.59 1.43.59.56 0 1.03-.2 1.42-.59.39-.39.58-.86.58-1.41 0-.55-.2-1.02-.58-1.41a1.93 1.93 0 0 0-1.42-.59c-.56 0-1.04.2-1.43.59-.39.39-.58.86-.58 1.41z" fill-rule="evenodd"></path></svg></button></div><div class="zq s"><pre class="zr"><div class="zs s"><div class="ba b bb bc if"><div class="dl">In the paper it says “ For the initial step, the representation function h receives as input the past observations o_1, …, o_t from the selected trajectory.” However, in the pseudocode, and in your explanation here, the representation function just…...</div></div></div></pre></div><button class="zx zy ev"><span class="ba b mv bc zz">Read More</span></button><div class="zu n o aw ck"><div class="n o"><div class="n o"><div class="s am jq js ps pt pu"><span><a href="https://medium.com/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fvote%2Fp%2F84a7580ac231&operation=register&redirect=https%3A%2F%2Fmedium.com%2Fapplied-data-science%2Fhow-to-build-your-own-muzero-in-python-f77d5718061a&source=responses-----84a7580ac231----1-----------------respond_sidebar-----------" class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener"><div class="ce pv pw px ev py pz wp r qb qc"><svg width="25" height="25" viewBox="0 0 25 25" aria-label="clap"><g fill-rule="evenodd"><path d="M11.74 0l.76 2.97.76-2.97zM14.81 3.78l1.84-2.56-1.42-.47zM8.38 1.22l1.84 2.56L9.8.75zM20.38 21.62a5.11 5.11 0 0 1-3.16 1.61l.49-.45c2.88-2.89 3.45-5.98 1.69-9.21l-1.1-1.94-.96-2.02c-.31-.67-.23-1.18.25-1.55a.84.84 0 0 1 .66-.16c.34.05.66.28.88.6l2.85 5.02c1.18 1.97 1.38 5.12-1.6 8.1M7.1 21.1l-5.02-5.02a1 1 0 0 1 .7-1.7 1 1 0 0 1 .72.3l2.6 2.6a.44.44 0 0 0 .63-.62L4.1 14.04l-1.75-1.75a1 1 0 1 1 1.41-1.41l4.15 4.15a.44.44 0 0 0 .63 0 .44.44 0 0 0 0-.62L4.4 10.26 3.22 9.08a1 1 0 0 1 0-1.4 1.02 1.02 0 0 1 1.41 0l1.18 1.16L9.96 13a.44.44 0 0 0 .62 0 .44.44 0 0 0 0-.63L6.43 8.22a.99.99 0 0 1-.3-.7.99.99 0 0 1 .3-.7 1 1 0 0 1 1.41 0l7 6.98a.44.44 0 0 0 .7-.5l-1.35-2.85c-.31-.68-.23-1.19.25-1.56a.85.85 0 0 1 .66-.16c.34.06.66.28.88.6L18.63 14c1.57 2.88 1.07 5.54-1.55 8.16a5.62 5.62 0 0 1-5.06 1.65 9.35 9.35 0 0 1-4.93-2.72zM11 5.98l2.56 2.56c-.5.6-.56 1.41-.15 2.28l.26.56-4.25-4.25a.98.98 0 0 1-.12-.45 1 1 0 0 1 .29-.7 1.02 1.02 0 0 1 1.41 0zm8.89 2.06c-.38-.56-.9-.92-1.49-1.01a1.74 1.74 0 0 0-1.34.33c-.38.29-.61.65-.71 1.06a2.1 2.1 0 0 0-1.1-.56 1.78 1.78 0 0 0-.99.13l-2.64-2.64a1.88 1.88 0 0 0-2.65 0 1.86 1.86 0 0 0-.48.85 1.89 1.89 0 0 0-2.67-.01 1.87 1.87 0 0 0-.5.9c-.76-.75-2-.75-2.7-.04a1.88 1.88 0 0 0 0 2.66c-.3.12-.61.29-.87.55a1.88 1.88 0 0 0 0 2.66l.62.62a1.88 1.88 0 0 0-.9 3.16l5.01 5.02c1.6 1.6 3.52 2.64 5.4 2.96a7.16 7.16 0 0 0 1.18.1c1.03 0 2-.25 2.9-.7A5.9 5.9 0 0 0 21 22.24c3.34-3.34 3.08-6.93 1.74-9.17l-2.87-5.04z"></path></g></svg></div></a></span></div></div><div class="eo q n o"></div></div><div class="zw s"><p class="ba b bb bc if"><button class="ei ej bz ca cb cc cd ce cf bl vh cg em en">Reply</button></p></div></div></div></div></div></div><div class="pk yu s"><div><div class="v gv"><div class="abd tb s"><div class="n aw ck"><div class="n o aw"><img alt="London L." class="s ji yf yg" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/0_oFm6Y4jNaIuHDgBE" width="32" height="32"><div class="yy s"><div class="n aw"><div><div class="bv" role="tooltip" aria-hidden="false" aria-describedby="55" aria-labelledby="55"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener" href="https://medium.com/@happylondon4?source=responses-----f77d5718061a----2----------------------------"><p class="ba b bb bc if">London L.</p></a></div></div></div><a class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener nofollow" href="https://medium.com/@happylondon4/why-is-there-a-reward-from-the-initial-inference-you-have-it-in-parentheses-as-0-449350b4f80d?source=responses-----f77d5718061a----2----------------------------"><p class="ba b bb bc du">about 1 year ago</p></a></div></div><button class="ev pw jj el yv"><svg class="overflow-dots-filled-25px_svg__svgIcon-use" width="25" height="25"><path d="M5 12.5c0 .55.2 1.02.59 1.41.39.4.86.59 1.41.59.55 0 1.02-.2 1.41-.59.4-.39.59-.86.59-1.41 0-.55-.2-1.02-.59-1.41A1.93 1.93 0 0 0 7 10.5c-.55 0-1.02.2-1.41.59-.4.39-.59.86-.59 1.41zm5.62 0c0 .55.2 1.02.58 1.41.4.4.87.59 1.42.59.55 0 1.02-.2 1.41-.59.4-.39.59-.86.59-1.41 0-.55-.2-1.02-.59-1.41a1.93 1.93 0 0 0-1.41-.59c-.55 0-1.03.2-1.42.59-.39.39-.58.86-.58 1.41zm5.6 0c0 .55.2 1.02.58 1.41.4.4.87.59 1.43.59.56 0 1.03-.2 1.42-.59.39-.39.58-.86.58-1.41 0-.55-.2-1.02-.58-1.41a1.93 1.93 0 0 0-1.42-.59c-.56 0-1.04.2-1.43.59-.39.39-.58.86-.58 1.41z" fill-rule="evenodd"></path></svg></button></div><a rel="noopener" href="https://medium.com/@happylondon4/why-is-there-a-reward-from-the-initial-inference-you-have-it-in-parentheses-as-0-449350b4f80d?source=responses-----f77d5718061a----2----------------------------"><div class="yz za se aba abb c"><div class="jz abc"><div class="ba b mv bc du"><div class="abe n o db"></div></div></div><p class="ba b bb bc gw mn mo mp mq mr ms if"><div class="s">Why is there a reward from the initial inference? (You have it in parentheses as “0”.)</div><div class="if">This seems like a related question on Stack Overflow: https://stackoverflow.com/questions/60234530/is-the-reward-value-in-muzeros-pseudocode-misaligned</div></p></div></a></div></div></div></div><div class="al yu s"><div><div class="v gv"><div class="th tb s"><div class="n aw ck"><div class="n o aw"><img alt="MMG" class="s ji yf yg" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_4uVRY7OQNt-m7YKJRB2sVw.jpeg" width="32" height="32"><div class="yy s"><div class="n aw"><div><div class="bv" role="tooltip" aria-hidden="false" aria-describedby="56" aria-labelledby="56"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener" href="https://medium.com/@vgoklani?source=responses-----f77d5718061a----3----------------------------"><p class="ba b bb bc if">MMG</p></a></div></div></div><a class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener nofollow" href="https://medium.com/@vgoklani/can-you-please-please-please-write-a-book-on-this-subject-10b9dfce82ef?source=responses-----f77d5718061a----3----------------------------"><p class="ba b bb bc du">over 1 year ago</p></a></div></div><button class="ev pw jj el yv"><svg class="overflow-dots-filled-25px_svg__svgIcon-use" width="25" height="25"><path d="M5 12.5c0 .55.2 1.02.59 1.41.39.4.86.59 1.41.59.55 0 1.02-.2 1.41-.59.4-.39.59-.86.59-1.41 0-.55-.2-1.02-.59-1.41A1.93 1.93 0 0 0 7 10.5c-.55 0-1.02.2-1.41.59-.4.39-.59.86-.59 1.41zm5.62 0c0 .55.2 1.02.58 1.41.4.4.87.59 1.42.59.55 0 1.02-.2 1.41-.59.4-.39.59-.86.59-1.41 0-.55-.2-1.02-.59-1.41a1.93 1.93 0 0 0-1.41-.59c-.55 0-1.03.2-1.42.59-.39.39-.58.86-.58 1.41zm5.6 0c0 .55.2 1.02.58 1.41.4.4.87.59 1.43.59.56 0 1.03-.2 1.42-.59.39-.39.58-.86.58-1.41 0-.55-.2-1.02-.58-1.41a1.93 1.93 0 0 0-1.42-.59c-.56 0-1.04.2-1.43.59-.39.39-.58.86-.58 1.41z" fill-rule="evenodd"></path></svg></button></div><div class="zq s"><pre class="zr"><div class="zs s"><div class="ba b bb bc if"><div class="dl">Can you please-please-please write a book on this subject! Your GANs book is awesome, and I’m sure this would be too!</div></div></div></pre></div><div class="zu n o aw ck"><div class="n o"><div class="n o"><div class="s am jq js ps pt pu"><span><a href="https://medium.com/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fvote%2Fp%2F10b9dfce82ef&operation=register&redirect=https%3A%2F%2Fmedium.com%2Fapplied-data-science%2Fhow-to-build-your-own-muzero-in-python-f77d5718061a&source=responses-----10b9dfce82ef----3-----------------respond_sidebar-----------" class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener"><div class="ce pv pw px ev py pz wp r qb qc"><svg width="25" height="25" viewBox="0 0 25 25" aria-label="clap"><g fill-rule="evenodd"><path d="M11.74 0l.76 2.97.76-2.97zM14.81 3.78l1.84-2.56-1.42-.47zM8.38 1.22l1.84 2.56L9.8.75zM20.38 21.62a5.11 5.11 0 0 1-3.16 1.61l.49-.45c2.88-2.89 3.45-5.98 1.69-9.21l-1.1-1.94-.96-2.02c-.31-.67-.23-1.18.25-1.55a.84.84 0 0 1 .66-.16c.34.05.66.28.88.6l2.85 5.02c1.18 1.97 1.38 5.12-1.6 8.1M7.1 21.1l-5.02-5.02a1 1 0 0 1 .7-1.7 1 1 0 0 1 .72.3l2.6 2.6a.44.44 0 0 0 .63-.62L4.1 14.04l-1.75-1.75a1 1 0 1 1 1.41-1.41l4.15 4.15a.44.44 0 0 0 .63 0 .44.44 0 0 0 0-.62L4.4 10.26 3.22 9.08a1 1 0 0 1 0-1.4 1.02 1.02 0 0 1 1.41 0l1.18 1.16L9.96 13a.44.44 0 0 0 .62 0 .44.44 0 0 0 0-.63L6.43 8.22a.99.99 0 0 1-.3-.7.99.99 0 0 1 .3-.7 1 1 0 0 1 1.41 0l7 6.98a.44.44 0 0 0 .7-.5l-1.35-2.85c-.31-.68-.23-1.19.25-1.56a.85.85 0 0 1 .66-.16c.34.06.66.28.88.6L18.63 14c1.57 2.88 1.07 5.54-1.55 8.16a5.62 5.62 0 0 1-5.06 1.65 9.35 9.35 0 0 1-4.93-2.72zM11 5.98l2.56 2.56c-.5.6-.56 1.41-.15 2.28l.26.56-4.25-4.25a.98.98 0 0 1-.12-.45 1 1 0 0 1 .29-.7 1.02 1.02 0 0 1 1.41 0zm8.89 2.06c-.38-.56-.9-.92-1.49-1.01a1.74 1.74 0 0 0-1.34.33c-.38.29-.61.65-.71 1.06a2.1 2.1 0 0 0-1.1-.56 1.78 1.78 0 0 0-.99.13l-2.64-2.64a1.88 1.88 0 0 0-2.65 0 1.86 1.86 0 0 0-.48.85 1.89 1.89 0 0 0-2.67-.01 1.87 1.87 0 0 0-.5.9c-.76-.75-2-.75-2.7-.04a1.88 1.88 0 0 0 0 2.66c-.3.12-.61.29-.87.55a1.88 1.88 0 0 0 0 2.66l.62.62a1.88 1.88 0 0 0-.9 3.16l5.01 5.02c1.6 1.6 3.52 2.64 5.4 2.96a7.16 7.16 0 0 0 1.18.1c1.03 0 2-.25 2.9-.7A5.9 5.9 0 0 0 21 22.24c3.34-3.34 3.08-6.93 1.74-9.17l-2.87-5.04z"></path></g></svg></div></a></span></div></div><div class="eo q n o"></div></div><div class="zw s"><p class="ba b bb bc if"><button class="ei ej bz ca cb cc cd ce cf bl vh cg em en">Reply</button></p></div></div></div></div></div></div></div></div></div></div><article class="meteredContent"><section class="fq fr fs ft v fu bu s"><div><div class="hf" id="lo-highlight-meter-1-highlight-box"><div class="xq br xr am xs"><div class="xt xu n ck xv"><p class="ba b dk dl if"><span class="hf" id="lo-highlight-meter-1-copy">You have <b>2</b> free member-only stories left this month. </span><div class="xw x"><span class="hf" id="lo-highlight-meter-1-link"><span><a href="https://medium.com/m/signin?operation=register&redirect=https%3A%2F%2Fmedium.com%2Fapplied-data-science%2Fhow-to-build-your-own-muzero-in-python-f77d5718061a&source=-----f77d5718061a---------------------smart_meter-----------" class="ei ej bz ca cb cc cd ce cf bl cg em en la" rel="noopener">Sign up for Medium and get an extra one</a></span></span></div></p></div></div></div></div></section><span class="s"></span><div><div class="eq fa ya gb gc gd"></div><div class="fs ft fu am"><div class="s h g f e"><aside class="wy eq fe" style="width: 535.5px;"><div class="yr nc eq acw x v"><p class="ba b mv bc du"><span class="bv nc x gw mo">Top highlight</span></p></div></aside></div></div><section class="ge gf gg dr gh"><div class="n p"><div class="ao ap aq ar as gi au v"><figure class="gk gl gm gn go gp fs ft paragraph-image"><div role="button" tabindex="0" class="gq gr am gs v gt"><div class="fs ft gj"><div class="gz s am ha"><div class="hb hc s"><div class="fb gu eq fe fa gv v gw gx gy"><img alt="Image for post" class="eq fe fa gv v hd he ay yt" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_y-FnMby1uCDxw6Br-BeZng.png" width="1211" height="659"></div><img alt="Image for post" class="xc xy eq fe fa gv v c" width="1211" height="659" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_y-FnMby1uCDxw6Br-BeZng(1).png" srcset="https://miro.medium.com/max/276/1*y-FnMby1uCDxw6Br-BeZng.png 276w, https://miro.medium.com/max/552/1*y-FnMby1uCDxw6Br-BeZng.png 552w, https://miro.medium.com/max/640/1*y-FnMby1uCDxw6Br-BeZng.png 640w, https://miro.medium.com/max/700/1*y-FnMby1uCDxw6Br-BeZng.png 700w" sizes="700px"><noscript><img alt="Image for post" class="eq fe fa gv v" src="https://miro.medium.com/max/2422/1*y-FnMby1uCDxw6Br-BeZng.png" width="1211" height="659" srcSet="https://miro.medium.com/max/552/1*y-FnMby1uCDxw6Br-BeZng.png 276w, https://miro.medium.com/max/1104/1*y-FnMby1uCDxw6Br-BeZng.png 552w, https://miro.medium.com/max/1280/1*y-FnMby1uCDxw6Br-BeZng.png 640w, https://miro.medium.com/max/1400/1*y-FnMby1uCDxw6Br-BeZng.png 700w" sizes="700px"/></noscript></div></div></div></div></figure><div class=""><h1 id="5321" class="hg hh hi ba hj hk hl hm hn ho hp hq hr hs ht hu hv hw hx hy hz ia ib ic id ie if">MuZero: The Walkthrough (Part 1/3)</h1></div><div class=""><h2 id="0289" class="ig hh hi ba b ih ii ij ik il im in io ip iq ir is it iu iv iw du">Teaching a machine to play games using self-play and deep learning…without telling it the rules 🤯</h2><div class="cw"><div class="n ck ix iy iz"><div class="o n"><div><a rel="noopener" href="https://medium.com/@dtfoster?source=post_page-----f77d5718061a--------------------------------"><div class="am ja jb"><div class="jc n aw o p eq jd je jf jg jh gd"><svg width="36" height="36" viewBox="0 0 36 36"><path fill-rule="evenodd" clip-rule="evenodd" d="M18 1.87c-6.63 0-12.4 4.14-15.21 10.21L2 11.71C4.94 5.37 11 1 18 1s13.06 4.37 16 10.71l-.79.37C30.4 6.01 24.63 1.88 18 1.88zM2.79 23.92c2.81 6.07 8.58 10.2 15.21 10.2 6.63 0 12.4-4.13 15.21-10.2l.79.37C31.06 30.63 25 35 18 35S4.94 30.63 2 24.29l.79-.37z"></path></svg></div><img alt="David Foster" class="s ji jb ja" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_QqMD1rth7ahcS7Gc415R9w.jpeg" width="28" height="28"></div></a></div><div class="eo v n ct"><div class="n"><div style="flex:1"><span class="ba b bb bc if"><div><div class="bv" role="tooltip" aria-hidden="false" aria-describedby="1" aria-labelledby="1"><a class="" rel="noopener" href="https://medium.com/@dtfoster?source=post_page-----f77d5718061a--------------------------------"><p class="ba b bb bc fj">David Foster</p></a></div></div></span></div></div><span class="ba b bb bc du"><a class="" rel="noopener" href="https://medium.com/applied-data-science/how-to-build-your-own-muzero-in-python-f77d5718061a?source=post_page-----f77d5718061a--------------------------------"><p class="ba b bb bc du"><span class="jm"></span>Dec 1, 2019<span class="jn">·</span>7 min read<svg class="jj jk jl" width="15" height="15" viewBox="0 0 15 15"><path d="M7.44 2.32c.03-.1.09-.1.12 0l1.2 3.53a.29.29 0 0 0 .26.2h3.88c.11 0 .13.04.04.1L9.8 8.33a.27.27 0 0 0-.1.29l1.2 3.53c.03.1-.01.13-.1.07l-3.14-2.18a.3.3 0 0 0-.32 0L4.2 12.22c-.1.06-.14.03-.1-.07l1.2-3.53a.27.27 0 0 0-.1-.3L2.06 6.16c-.1-.06-.07-.12.03-.12h3.89a.29.29 0 0 0 .26-.19l1.2-3.52z"></path></svg></p></a></span></div></div><div class="n jo jp jq jr js jt ju jv jw"><div class="n o"><div class="jx s"><div class="bv" aria-hidden="false" aria-describedby="postFooterSocialMenu" aria-labelledby="postFooterSocialMenu"><div><div class="bv" role="tooltip" aria-hidden="false" aria-describedby="2" aria-labelledby="2"><button class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" aria-controls="postFooterSocialMenu" aria-expanded="false" aria-label="Share Post"><svg width="25" height="25" class="r"><g fill-rule="evenodd"><path d="M15.6 5a.42.42 0 0 0 .17-.3.42.42 0 0 0-.12-.33l-2.8-2.79a.5.5 0 0 0-.7 0l-2.8 2.8a.4.4 0 0 0-.1.32c0 .12.07.23.16.3h.02a.45.45 0 0 0 .57-.04l2-2V10c0 .28.23.5.5.5s.5-.22.5-.5V2.93l2.02 2.02c.08.07.18.12.3.13.11.01.21-.02.3-.08v.01"></path><path d="M18 7h-1.5a.5.5 0 0 0 0 1h1.6c.5 0 .9.4.9.9v10.2c0 .5-.4.9-.9.9H6.9a.9.9 0 0 1-.9-.9V8.9c0-.5.4-.9.9-.9h1.6a.5.5 0 0 0 .35-.15A.5.5 0 0 0 9 7.5a.5.5 0 0 0-.15-.35A.5.5 0 0 0 8.5 7H7a2 2 0 0 0-2 2v10c0 1.1.9 2 2 2h11a2 2 0 0 0 2-2V9a2 2 0 0 0-2-2"></path></g></svg></button></div></div></div></div><div class="jy s"><div class="jz"><span><a href="https://medium.com/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fbookmark%2Fp%2Ff77d5718061a&operation=register&redirect=https%3A%2F%2Fmedium.com%2Fapplied-data-science%2Fhow-to-build-your-own-muzero-in-python-f77d5718061a&source=post_actions_header--------------------------bookmark_preview-----------" class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener"><svg width="25" height="25" viewBox="0 0 25 25"><path d="M19 6a2 2 0 0 0-2-2H8a2 2 0 0 0-2 2v14.66h.01c.01.1.05.2.12.28a.5.5 0 0 0 .7.03l5.67-4.12 5.66 4.13a.5.5 0 0 0 .71-.03.5.5 0 0 0 .12-.29H19V6zm-6.84 9.97L7 19.64V6a1 1 0 0 1 1-1h9a1 1 0 0 1 1 1v13.64l-5.16-3.67a.49.49 0 0 0-.68 0z" fill-rule="evenodd"></path></svg></a></span></div></div><div class="s ax"></div></div></div></div></div></div><blockquote class="ka kb kc"><p id="f918" class="kd ke kf kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">If you want to learn how one of the most sophisticated AI systems ever built works, you’ve come to the right place.</p></blockquote><p id="2bfb" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">In this three part series, we’ll explore the inner workings of the DeepMind MuZero model — the younger (and even more impressive) brother of AlphaZero.</p><p id="4340" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">👉 <a class="ei la" rel="noopener" href="https://medium.com/applied-data-science/how-to-build-your-own-deepmind-muzero-in-python-part-2-3-f99dad7a7ad">Part 2</a></p><p id="1a04" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">👉 <a class="ei la" rel="noopener" href="https://medium.com/applied-data-science/how-to-build-your-own-deepmind-muzero-in-python-part-3-3-ccea6b03538b">Part 3</a></p><p id="ab29" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">Also check out my latest post, about how to train reinforcement learning agents for multi-player board games, using self-play!</p><p id="efd8" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">👉 <a class="ei la" rel="noopener" href="https://medium.com/applied-data-science/how-to-train-ai-agents-to-play-multiplayer-games-using-self-play-deep-reinforcement-learning-247d0b440717">Self-Play in Multiplayer Environments</a></p><p id="4295" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">We’ll be walking through the <a href="https://arxiv.org/src/1911.08265v1/anc/pseudocode.py" class="ei la" rel="noopener nofollow">pseudocode</a> that accompanies the MuZero <a href="https://arxiv.org/abs/1911.08265" class="ei la" rel="noopener nofollow">paper</a> — so grab yourself a cup of tea and a comfy chair and let’s begin.</p><figure class="lc ld le lf lg gp fs ft paragraph-image"><div class="fs ft lb"><div class="gz s am ha"><div class="lh hc s"><div class="fb gu eq fe fa gv v gw gx gy"><img alt="Image for post" class="eq fe fa gv v hd he ay yt" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_62G0bnqJoqujSIO4hizzKw.png" width="75" height="75"></div><img alt="Image for post" class="xc xy eq fe fa gv v c" width="75" height="75" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_62G0bnqJoqujSIO4hizzKw(1).png" srcset="" sizes="75px"><noscript><img alt="Image for post" class="eq fe fa gv v" src="https://miro.medium.com/max/150/1*62G0bnqJoqujSIO4hizzKw.png" width="75" height="75"/></noscript></div></div></div></figure><h1 id="e82d" class="li lj hi ba lk ll lm ki ln lo lp kl lq lr ls lt lu lv lw lx ly lz ma mb mc md if" data-selectable-paragraph="">The story so far…</h1><p id="0bb5" class="kd ke hi kg b ih me ki kj ik mf kl km kn mg kp kq kr mh kt ku kv mi kx ky kz ge if" data-selectable-paragraph="">On 19th November 2019 DeepMind released their latest model-based reinforcement learning algorithm to the world — <a href="https://arxiv.org/abs/1911.08265" class="ei la" rel="noopener nofollow">MuZero</a>.</p><p id="225e" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">Th<span id="rmm"><span id="rmm"><span id="rmm"><span id="rmm"><span id="rmm"><span id="rmm"><span id="rmm"><span id="rmm"><span id="rmm"><span id="rmm"><span id="rmm"><span id="rmm"><span id="rmm">i</span></span></span></span></span></span></span></span></span></span></span></span></span>s is the fourth in a line of DeepMind reinforcement learning papers that have continually smashed through the barriers of possibility, starting with AlphaGo in 2016.</p><p id="b5bc" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">To read about the full history from AlphaGo through to AlphaZero — check out my previous blog 👇</p><div class="gk gl gm gn go mj"><a target="_blank" rel="noopener" href="https://medium.com/applied-data-science/how-to-build-your-own-alphazero-ai-using-python-and-keras-7f664945c188"><div class="eu n db"><div class="mk n ah p ml mm"><h2 class="ba hj dk bc gw mn mo mp mq mr ms hh if">How to build your own AlphaZero AI using Python and Keras</h2><div class="mt s"><h3 class="ba b dk bc gw mn mo mp mq mr ms du">Teach a machine to learn Connect4 strategy through self-play and deep learning.</h3></div><div class="mu s"><p class="ba b mv bc gw mn mo mp mq mr ms du">medium.com</p></div></div><div class="mw s"><div class="mx s my mz na mw nb nc mj"></div></div></div></a></div><p id="f065" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">AlphaZero was hailed as the general algorithm for getting good at something, quickly, without any prior knowledge of human expert strategy.</p><p id="c320" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">So…what now?</p><figure class="lc ld le lf lg gp fs ft paragraph-image"><div class="fs ft lb"><div class="gz s am ha"><div class="lh hc s"><div class="fb gu eq fe fa gv v gw gx gy"><img alt="Image for post" class="eq fe fa gv v hd he ay yt" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_VVGmi0SSqHVJPtpuOw-aVw.png" width="75" height="75"></div><img alt="Image for post" class="xc xy eq fe fa gv v c" width="75" height="75" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_VVGmi0SSqHVJPtpuOw-aVw(1).png" srcset="" sizes="75px"><noscript><img alt="Image for post" class="eq fe fa gv v" src="https://miro.medium.com/max/150/1*VVGmi0SSqHVJPtpuOw-aVw.png" width="75" height="75"/></noscript></div></div></div></figure><h1 id="9fe1" class="li lj hi ba lk ll lm ki ln lo lp kl lq lr ls lt lu lv lw lx ly lz ma mb mc md if" data-selectable-paragraph="">MuZero</h1><p id="5249" class="kd ke hi kg b ih me ki kj ik mf kl km kn mg kp kq kr mh kt ku kv mi kx ky kz ge if" data-selectable-paragraph=""><a href="https://arxiv.org/abs/1911.08265" class="ei la" rel="noopener nofollow">Mastering Atari, Go, Chess and Shogi by Planning with a Learned Model</a></p><p id="63d2" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">MuZero takes the ultimate next step. Not only does MuZero deny itself human strategy to learn from. It isn’t even shown the rules of the game.</p><p id="766f" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">In other words, for chess, AlphaZero is set the following challenge:</p><blockquote class="ka kb kc"><p id="80a1" class="kd ke kf kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">Learn how to play this game on your own — here’s the rulebook that explains how each piece moves and which moves are legal. Also it tells you how to tell if a position is checkmate (or a draw).</p></blockquote><p id="4f85" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">MuZero on the other hand, is set this challenge:</p><blockquote class="ka kb kc"><p id="e932" class="kd ke kf kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">Learn how to play this game on your own — I’ll tell you what moves are legal in the current position and when one side has won (or it’s a draw), but I won’t tell you the overall rules of the game.</p></blockquote><p id="3b5e" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">Alongside developing winning strategies, MuZero must therefore also develop its own dynamic model of the environment so that it can understand the implications of its choices and plan ahead.</p><p id="bf72" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">Imagine trying to become better than the world champion at a game where you are never told the rules. MuZero achieves precisely this.</p><p id="f3b8" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">In the next section we will explore how MuZero achieves this amazing feat, by walking through the codebase in detail.</p><figure class="lc ld le lf lg gp fs ft paragraph-image"><div class="fs ft lb"><div class="gz s am ha"><div class="lh hc s"><div class="fb gu eq fe fa gv v gw gx gy"><img alt="Image for post" class="eq fe fa gv v hd he ay yt" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_zZvaEuY5-OhKB2-GYzuUdw.png" width="75" height="75"></div><img alt="Image for post" class="xc xy eq fe fa gv v c" width="75" height="75" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_zZvaEuY5-OhKB2-GYzuUdw(1).png" srcset="" sizes="75px"><noscript><img alt="Image for post" class="eq fe fa gv v" src="https://miro.medium.com/max/150/1*zZvaEuY5-OhKB2-GYzuUdw.png" width="75" height="75"/></noscript></div></div></div></figure><h1 id="b162" class="li lj hi ba lk ll lm ki ln lo lp kl lq lr ls lt lu lv lw lx ly lz ma mb mc md if" data-selectable-paragraph="">The MuZero pseudocode</h1><p id="8a62" class="kd ke hi kg b ih me ki kj ik mf kl km kn mg kp kq kr mh kt ku kv mi kx ky kz ge if" data-selectable-paragraph="">Alongside the MuZero preprint <a href="https://arxiv.org/abs/1911.08265" class="ei la" rel="noopener nofollow">paper</a>, DeepMind have released Python <a href="https://arxiv.org/src/1911.08265v1/anc/pseudocode.py" class="ei la" rel="noopener nofollow">pseudocode</a> detailing the interactions between each part of the algorithm.</p><p id="fd51" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">In this section, we’ll pick apart each function and class in a logical order, and I’ll explain what each part is doing and why. We’ll assume MuZero is learning to play chess, but the process is the same for any game, just with different parameters. All code is from the open-sourced DeepMind <a href="https://arxiv.org/src/1911.08265v1/anc/pseudocode.py" class="ei la" rel="noopener nofollow">pseudocode</a>.</p><p id="7ec3" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">Let’s start with an overview of the entire process, starting with the entrypoint function, <code class="ha nd ne nf ng b">muzero</code>.</p></div></div><div class="gp"><div class="n p"><div class="nh ni nj nk nl nm ar nn as no au v"><figure class="lc ld le lf lg gp np nq paragraph-image"><div role="button" tabindex="0" class="gq gr am gs v gt"><div class="fs ft gj"><div class="gz s am ha"><div class="hb hc s"><div class="fb gu eq fe fa gv v gw gx gy"><img alt="Image for post" class="eq fe fa gv v hd he ay yt" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_ajFyjeF-1hVbmtlAsSoT2Q.png" width="1211" height="659"></div><img alt="Image for post" class="xc xy eq fe fa gv v c" width="1211" height="659" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_ajFyjeF-1hVbmtlAsSoT2Q(1).png" srcset="https://miro.medium.com/max/276/1*ajFyjeF-1hVbmtlAsSoT2Q.png 276w, https://miro.medium.com/max/552/1*ajFyjeF-1hVbmtlAsSoT2Q.png 552w, https://miro.medium.com/max/640/1*ajFyjeF-1hVbmtlAsSoT2Q.png 640w, https://miro.medium.com/max/728/1*ajFyjeF-1hVbmtlAsSoT2Q.png 728w, https://miro.medium.com/max/816/1*ajFyjeF-1hVbmtlAsSoT2Q.png 816w, https://miro.medium.com/max/904/1*ajFyjeF-1hVbmtlAsSoT2Q.png 904w, https://miro.medium.com/max/992/1*ajFyjeF-1hVbmtlAsSoT2Q.png 992w, https://miro.medium.com/max/1000/1*ajFyjeF-1hVbmtlAsSoT2Q.png 1000w" sizes="1000px"><noscript><img alt="Image for post" class="eq fe fa gv v" src="https://miro.medium.com/max/2422/1*ajFyjeF-1hVbmtlAsSoT2Q.png" width="1211" height="659" srcSet="https://miro.medium.com/max/552/1*ajFyjeF-1hVbmtlAsSoT2Q.png 276w, https://miro.medium.com/max/1104/1*ajFyjeF-1hVbmtlAsSoT2Q.png 552w, https://miro.medium.com/max/1280/1*ajFyjeF-1hVbmtlAsSoT2Q.png 640w, https://miro.medium.com/max/1456/1*ajFyjeF-1hVbmtlAsSoT2Q.png 728w, https://miro.medium.com/max/1632/1*ajFyjeF-1hVbmtlAsSoT2Q.png 816w, https://miro.medium.com/max/1808/1*ajFyjeF-1hVbmtlAsSoT2Q.png 904w, https://miro.medium.com/max/1984/1*ajFyjeF-1hVbmtlAsSoT2Q.png 992w, https://miro.medium.com/max/2000/1*ajFyjeF-1hVbmtlAsSoT2Q.png 1000w" sizes="1000px"/></noscript></div></div></div></div><figcaption class="nr ns fu fs ft nt nu ba b bb bc du" data-selectable-paragraph="">Overview of the MuZero self-play and training process</figcaption></figure></div></div></div><div class="n p"><div class="ao ap aq ar as gi au v"><figure class="lc ld le lf lg gp"><div class="gz s am"><div class="acr hc s"><iframe src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/37989df7e0c024c98a85ab9154886677.html" allowfullscreen="" frameborder="0" height="257" width="680" title="MuZero pseudocode (https://arxiv.org/src/1911.08265v1/anc/pseudocode.py)" class="eq fe fa gv v" scrolling="auto"></iframe></div></div></figure><p id="05af" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">The entrypoint function <code class="ha nd ne nf ng b">muzero</code> is passed a <code class="ha nd ne nf ng b">MuZeroConfig</code> object, which stores important information about the parameterisation of the run, such as the <code class="ha nd ne nf ng b">action_space_size</code> (number of possible actions) and <code class="ha nd ne nf ng b">num_actors</code> (the number of parallel game simulations to spin up). We’ll go through these parameters in more detail as we encounter them in other functions.</p><p id="1e7b" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">At a high level, there are two independent parts to the MuZero algorithm — <em class="kf">self-play</em> (creating game data) and <em class="kf">training</em> (producing improved versions of the neural network). The<code class="ha nd ne nf ng b">SharedStorage</code> and <code class="ha nd ne nf ng b">ReplayBuffer</code> objects can be accessed by both halves of the algorithm and store neural network versions and game data respectively.</p><figure class="lc ld le lf lg gp fs ft paragraph-image"><div class="fs ft lb"><div class="gz s am ha"><div class="lh hc s"><div class="fb gu eq fe fa gv v gw gx gy"><img alt="Image for post" class="eq fe fa gv v hd he ay yt" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_sWVAnhQvj2EmapE1UqIZDg.png" width="75" height="75"></div><img alt="Image for post" class="xc xy eq fe fa gv v c" width="75" height="75" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_sWVAnhQvj2EmapE1UqIZDg(1).png" srcset="" sizes="75px"><noscript><img alt="Image for post" class="eq fe fa gv v" src="https://miro.medium.com/max/150/1*sWVAnhQvj2EmapE1UqIZDg.png" width="75" height="75"/></noscript></div></div></div></figure><h2 id="8e1a" class="nw lj hi ba lk nx ny ij ln nz oa im lq in ob ip lu iq oc is ly it od iv mc oe if" data-selectable-paragraph="">Shared Storage and the Replay Buffer</h2><p id="4e18" class="kd ke hi kg b ih me ki kj ik mf kl km kn mg kp kq kr mh kt ku kv mi kx ky kz ge if" data-selectable-paragraph="">The <code class="ha nd ne nf ng b">SharedStorage</code> object contains methods for saving a version of the neural network and retrieving the latest neural network from the store.</p><figure class="lc ld le lf lg gp"><div class="gz s am"><div class="act hc s"><iframe src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/57eaa5c07fa0d54806a3c01079a6365c.html" allowfullscreen="" frameborder="0" height="345" width="680" title="SharedStorage (https://arxiv.org/src/1911.08265v1/anc/pseudocode.py)" class="eq fe fa gv v" scrolling="auto"></iframe></div></div></figure><p id="bedb" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">We also need a <code class="ha nd ne nf ng b">ReplayBuffer</code> to store data from previous games. This takes the following form:</p><figure class="lc ld le lf lg gp"><div class="gz s am"><div class="acv hc s"><iframe src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/83100f618299e154eaa27d3b5cfef9e4.html" allowfullscreen="" frameborder="0" height="323" width="680" title="ReplayBuffer (https://arxiv.org/src/1911.08265v1/anc/pseudocode.py)" class="eq fe fa gv v" scrolling="auto"></iframe></div></div></figure><p id="44ce" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">Notice how the <code class="ha nd ne nf ng b">window_size</code> parameter limits the maximum number of games stored in the buffer. In MuZero, this is set to the latest 1,000,000 games.</p><figure class="lc ld le lf lg gp fs ft paragraph-image"><div class="fs ft lb"><div class="gz s am ha"><div class="lh hc s"><div class="fb gu eq fe fa gv v gw gx gy"><img alt="Image for post" class="eq fe fa gv v hd he ay yt" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_WNkAYNV92eJNZqxc9EfcBw.png" width="75" height="75"></div><img alt="Image for post" class="xc xy eq fe fa gv v c" width="75" height="75" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_WNkAYNV92eJNZqxc9EfcBw(1).png" srcset="" sizes="75px"><noscript><img alt="Image for post" class="eq fe fa gv v" src="https://miro.medium.com/max/150/1*WNkAYNV92eJNZqxc9EfcBw.png" width="75" height="75"/></noscript></div></div></div></figure><h2 id="b3ce" class="nw lj hi ba lk nx ny ij ln nz oa im lq in ob ip lu iq oc is ly it od iv mc oe if" data-selectable-paragraph="">Self-play (run_selfplay)</h2><p id="05b3" class="kd ke hi kg b ih me ki kj ik mf kl km kn mg kp kq kr mh kt ku kv mi kx ky kz ge if" data-selectable-paragraph="">After creating the shared storage and replay buffer, MuZero launches <code class="ha nd ne nf ng b">num_actors</code> parallel game environments, that run independently. For chess, <code class="ha nd ne nf ng b">num_actors</code> is set to 3000. Each is running a function <code class="ha nd ne nf ng b">run_selfplay</code> that grabs the latest version of the network from the store, plays a game with it (<code class="ha nd ne nf ng b">play_game</code>) and saves the game data to the shared buffer.</p><figure class="lc ld le lf lg gp"><div class="gz s am"><div class="nv hc s"></div></div></figure><p id="b6fe" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">So in summary, MuZero is playing thousands of games against itself, saving these to a buffer and then training itself on data from those games. So far, this is no different to AlphaZero.</p><p id="5970" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">To end Part 1, we will cover one of the key differences between AlphaZero and MuZero — why does MuZero have three neural networks, whereas AlphaZero only has one?</p><figure class="lc ld le lf lg gp fs ft paragraph-image"><div class="fs ft lb"><div class="gz s am ha"><div class="lh hc s"><div class="fb gu eq fe fa gv v gw gx gy"><img alt="Image for post" class="eq fe fa gv v hd he ay yt" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_XgPxsnGohRYsJZftwNNUJQ.png" width="75" height="75"></div><img alt="Image for post" class="xc xy eq fe fa gv v c" width="75" height="75" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_XgPxsnGohRYsJZftwNNUJQ(1).png" srcset="" sizes="75px"><noscript><img alt="Image for post" class="eq fe fa gv v" src="https://miro.medium.com/max/150/1*XgPxsnGohRYsJZftwNNUJQ.png" width="75" height="75"/></noscript></div></div></div></figure><h2 id="0aad" class="nw lj hi ba lk nx ny ij ln nz oa im lq in ob ip lu iq oc is ly it od iv mc oe if" data-selectable-paragraph=""><strong class="cd">The 3 Neural Networks of MuZero</strong></h2><p id="859c" class="kd ke hi kg b ih me ki kj ik mf kl km kn mg kp kq kr mh kt ku kv mi kx ky kz ge if" data-selectable-paragraph="">Both AlphaZero and MuZero utilise a technique known as <strong class="kg hj">Monte Carlo Tree Search (MCTS)</strong> to select the next best move.</p><p id="3224" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">The idea is that in order to select the next best move, it makes sense to ‘play out’ likely future scenarios from the current position, evaluate their value using a neural network and choose the action that maximises the future expected value. This seems to be what we humans are doing in our head when playing chess, and the AI is also designed to make use of this technique.</p><p id="84ee" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">However, MuZero has a problem. As it doesn’t know the rules of the game, it has no idea how a given action will affect the game state, so it cannot imagine future scenarios in the MCTS. It doesn’t even know how to work out what moves are legal from a given position, or whether one side has won.</p><blockquote class="ka kb kc"><p id="ac7b" class="kd ke kf kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">The stunning development in the MuZero paper is to show that this doesn’t matter. MuZero learns how to play the game by creating a dynamic model of the environment within its own imagination and optimising within this model.</p></blockquote><p id="5154" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">The diagram below shows a comparison between the MCTS processes in AlphaZero and MuZero:</p></div></div><div class="gp"><div class="n p"><div class="nh ni nj nk nl nm ar nn as no au v"><figure class="lc ld le lf lg gp np nq paragraph-image"><div role="button" tabindex="0" class="gq gr am gs v gt"><div class="fs ft of"><div class="gz s am ha"><div class="og hc s"><div class="xc xy eq fe fa gv v gw gx gy"><img alt="Image for post" class="eq fe fa gv v hd he hf" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_NowOwxV5SQ9aLKbjdz41lQ.png" width="1212" height="703"></div><img alt="Image for post" class="fb gu eq fe fa gv v c" width="1212" height="703"><noscript><img alt="Image for post" class="eq fe fa gv v" src="https://miro.medium.com/max/2424/1*NowOwxV5SQ9aLKbjdz41lQ.png" width="1212" height="703" srcSet="https://miro.medium.com/max/552/1*NowOwxV5SQ9aLKbjdz41lQ.png 276w, https://miro.medium.com/max/1104/1*NowOwxV5SQ9aLKbjdz41lQ.png 552w, https://miro.medium.com/max/1280/1*NowOwxV5SQ9aLKbjdz41lQ.png 640w, https://miro.medium.com/max/1456/1*NowOwxV5SQ9aLKbjdz41lQ.png 728w, https://miro.medium.com/max/1632/1*NowOwxV5SQ9aLKbjdz41lQ.png 816w, https://miro.medium.com/max/1808/1*NowOwxV5SQ9aLKbjdz41lQ.png 904w, https://miro.medium.com/max/1984/1*NowOwxV5SQ9aLKbjdz41lQ.png 992w, https://miro.medium.com/max/2000/1*NowOwxV5SQ9aLKbjdz41lQ.png 1000w" sizes="1000px"/></noscript></div></div></div></div></figure></div></div></div><div class="n p"><div class="ao ap aq ar as gi au v"><p id="e0a3" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph=""><mark class="yp yq ev">Whereas AlphaZero only has only one neural network (</mark><mark class="yp yq ev"><strong class="kg hj">prediction</strong></mark><mark class="yp yq ev">), MuZero needs three (</mark><mark class="yp yq ev"><strong class="kg hj">prediction</strong></mark><mark class="yp yq ev">, </mark><mark class="yp yq ev"><strong class="kg hj">dynamics</strong></mark><mark class="yp yq ev">, </mark><mark class="yp yq ev"><strong class="kg hj">representation</strong></mark><mark class="yp yq ev">)</mark></p><p id="f763" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">The job of the AlphaZero <strong class="kg hj">prediction</strong> neural network <code class="ha nd ne nf ng b">f</code> is to predict the policy <code class="ha nd ne nf ng b">p</code> and value <code class="ha nd ne nf ng b">v</code> of a given game state. The policy is a probability distribution over all moves and the value is just a single number that estimates the future rewards. This prediction is made every time the MCTS hits an unexplored leaf node, so that it can immediately assign an estimated value to the new position and also assign a probability to each subsequent action. The values are backfilled up the tree, back to the root node, so that after many simulations, the root node has a good idea of the future value of the current state, having explored lots of different possible futures.</p><p id="36ac" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">MuZero also has a <strong class="kg hj">prediction</strong> neural network <code class="ha nd ne nf ng b">f</code>, but now the ‘game state’ that it operates on is a hidden representation that MuZero learns how to evolve through a <strong class="kg hj">dynamics</strong> neural network <code class="ha nd ne nf ng b">g</code>. The dynamics network takes the current hidden state <code class="ha nd ne nf ng b">s</code> and chosen action <code class="ha nd ne nf ng b">a</code> and outputs a reward <code class="ha nd ne nf ng b">r</code> and new state. Notice how in AlphaZero, moving between states in the MCTS tree is simply a case of asking the environment. MuZero doesn’t have this luxury, so needs to build its own dynamic model!</p><p id="0942" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">Lastly, in order to map from the current observed game state to the initial representation, MuZero uses a third <strong class="kg hj">representation</strong> neural network, <code class="ha nd ne nf ng b">h</code>.</p><p id="7939" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">There are therefore two inference functions MuZero needs, in order to move through the MCTS tree making predictions:</p><ul class=""><li id="aad9" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz oh oi oj if" data-selectable-paragraph=""><code class="ha nd ne nf ng b">initial_inference</code> for the current state. <code class="ha nd ne nf ng b">h</code> followed by <code class="ha nd ne nf ng b">f</code> (representation followed by prediction) .</li><li id="7f8d" class="kd ke hi kg b ih ok ki kj ik ol kl km kn om kp kq kr on kt ku kv oo kx ky kz oh oi oj if" data-selectable-paragraph=""><code class="ha nd ne nf ng b">recurrent_inference</code> for moving between states inside the MCTS tree.<code class="ha nd ne nf ng b">g</code> followed by <code class="ha nd ne nf ng b">f</code> (representation followed by dynamics).</li></ul></div></div><div class="gp"><div class="n p"><div class="nh ni nj nk nl nm ar nn as no au v"><figure class="lc ld le lf lg gp np nq paragraph-image"><div role="button" tabindex="0" class="gq gr am gs v gt"><div class="fs ft gj"><div class="gz s am ha"><div class="op hc s"><div class="xc xy eq fe fa gv v gw gx gy"><img alt="Image for post" class="eq fe fa gv v hd he hf" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_GA72IpY7ZciGshmVvtl8kQ.png" width="1211" height="555"></div><img alt="Image for post" class="fb gu eq fe fa gv v c" width="1211" height="555"><noscript><img alt="Image for post" class="eq fe fa gv v" src="https://miro.medium.com/max/2422/1*GA72IpY7ZciGshmVvtl8kQ.png" width="1211" height="555" srcSet="https://miro.medium.com/max/552/1*GA72IpY7ZciGshmVvtl8kQ.png 276w, https://miro.medium.com/max/1104/1*GA72IpY7ZciGshmVvtl8kQ.png 552w, https://miro.medium.com/max/1280/1*GA72IpY7ZciGshmVvtl8kQ.png 640w, https://miro.medium.com/max/1456/1*GA72IpY7ZciGshmVvtl8kQ.png 728w, https://miro.medium.com/max/1632/1*GA72IpY7ZciGshmVvtl8kQ.png 816w, https://miro.medium.com/max/1808/1*GA72IpY7ZciGshmVvtl8kQ.png 904w, https://miro.medium.com/max/1984/1*GA72IpY7ZciGshmVvtl8kQ.png 992w, https://miro.medium.com/max/2000/1*GA72IpY7ZciGshmVvtl8kQ.png 1000w" sizes="1000px"/></noscript></div></div></div></div><figcaption class="nr ns fu fs ft nt nu ba b bb bc du" data-selectable-paragraph="">The two types of inference in MuZero</figcaption></figure></div></div></div><div class="n p"><div class="ao ap aq ar as gi au v"><p id="101a" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">The exact models aren’t provided in the pseudocode, but detailed descriptions are given in the accompanying paper.</p><figure class="lc ld le lf lg gp"><div class="gz s am"><div class="nv hc s"></div></div></figure><p id="4df5" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">In summary, in the absence of the actual rules of chess, MuZero creates a new game inside its mind that it can control and uses this to plan into the future. The three networks (<strong class="kg hj">prediction</strong>, <strong class="kg hj">dynamics</strong> and <strong class="kg hj">representation</strong>) are optimised together so that strategies that perform well inside the imagined environment, also perform well in the real environment.</p><p id="376f" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">Amazing stuff.</p></div></div></section><div class="n p cw oq or os" role="separator"><span class="ot ji bv ou ov ow"></span><span class="ot ji bv ou ov ow"></span><span class="ot ji bv ou ov"></span></div><section class="ge gf gg dr gh"><div class="n p"><div class="ao ap aq ar as gi au v"><p id="ae0f" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">This is the end of Part 1 — in <a class="ei la" rel="noopener" href="https://medium.com/applied-data-science/how-to-build-your-own-deepmind-muzero-in-python-part-2-3-f99dad7a7ad">Part 2</a>, we’ll start by walking through the <code class="ha nd ne nf ng b">play_game</code> function and see how MuZero makes a decision about the next best move at each turn.</p><p id="f1a4" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">Please clap if you’ve enjoyed this post and I’ll see you in <a class="ei la" rel="noopener" href="https://medium.com/applied-data-science/how-to-build-your-own-deepmind-muzero-in-python-part-2-3-f99dad7a7ad">Part 2</a>!</p></div></div></section><div class="n p cw oq or os" role="separator"><span class="ot ji bv ou ov ow"></span><span class="ot ji bv ou ov ow"></span><span class="ot ji bv ou ov"></span></div><section class="ge gf gg dr gh"><div class="n p"><div class="ao ap aq ar as gi au v"><figure class="lc ld le lf lg gp fs ft paragraph-image"><div class="fs ft ox"><div class="gz s am ha"><div class="oy hc s"><div class="xc xy eq fe fa gv v gw gx gy"><img alt="Image for post" class="eq fe fa gv v hd he hf" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/0_x1sJqcMlPO9p-R-0.png" width="700" height="194"></div><img alt="Image for post" class="fb gu eq fe fa gv v c" width="700" height="194"><noscript><img alt="Image for post" class="eq fe fa gv v" src="https://miro.medium.com/max/1400/0*x1sJqcMlPO9p-R-0.png" width="700" height="194" srcSet="https://miro.medium.com/max/552/0*x1sJqcMlPO9p-R-0.png 276w, https://miro.medium.com/max/1104/0*x1sJqcMlPO9p-R-0.png 552w, https://miro.medium.com/max/1280/0*x1sJqcMlPO9p-R-0.png 640w, https://miro.medium.com/max/1400/0*x1sJqcMlPO9p-R-0.png 700w" sizes="700px"/></noscript></div></div></div></figure><p id="0737" class="kd ke hi kg b ih kh ki kj ik kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz ge if" data-selectable-paragraph="">This is the blog of Applied Data Science Partners, a consultancy that develops innovative data science solutions for businesses. To learn more, feel free to get in touch through our <a href="https://adsp.ai/" class="ei la" rel="noopener nofollow">website</a>.</p></div></div></section></div></article><div class="xc gd fc pf v yo fe pd ph" data-test-id="post-sidebar"><div class="n p"><div class="ao ap aq ar as at au v"><div class="pi n ah"><div class="acp"><div><div class="pj pk s"><div class="np s"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener" href="https://medium.com/@dtfoster?source=post_sidebar--------------------------post_sidebar-----------"><h2 class="ba lk dk bc hh if ge">David Foster</h2></a></div><div class="pl s"><p class="ba b bb bc du">Author of the Generative Deep Learning book :: Founding Partner of Applied Data Science Partners</p></div><div class="pm s"><span><button class="ba b bb bc dy be dz ea eb ec ed bl bm bn ee ef br bs bt bu bv bw">Follow</button></span></div></div><div class="pn po pp n"><div class="n o"><div class="s am pq pr ps pt pu"><span><a href="https://medium.com/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fvote%2Fapplied-data-science%2Ff77d5718061a&operation=register&redirect=https%3A%2F%2Fmedium.com%2Fapplied-data-science%2Fhow-to-build-your-own-muzero-in-python-f77d5718061a&source=post_sidebar-----f77d5718061a---------------------clap_sidebar-----------" class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener"><div class="ce pv pw px ev py pz qa r qb qc"><svg width="29" height="29" aria-label="clap"><g fill-rule="evenodd"><path d="M13.74 1l.76 2.97.76-2.97zM16.82 4.78l1.84-2.56-1.43-.47zM10.38 2.22l1.84 2.56-.41-3.03zM22.38 22.62a5.11 5.11 0 0 1-3.16 1.61l.49-.45c2.88-2.89 3.45-5.98 1.69-9.21l-1.1-1.94-.96-2.02c-.31-.67-.23-1.18.25-1.55a.84.84 0 0 1 .66-.16c.34.05.66.28.88.6l2.85 5.02c1.18 1.97 1.38 5.12-1.6 8.1M9.1 22.1l-5.02-5.02a1 1 0 0 1 .7-1.7 1 1 0 0 1 .72.3l2.6 2.6a.44.44 0 0 0 .63-.62L6.1 15.04l-1.75-1.75a1 1 0 1 1 1.41-1.41l4.15 4.15a.44.44 0 0 0 .63 0 .44.44 0 0 0 0-.62L6.4 11.26l-1.18-1.18a1 1 0 0 1 0-1.4 1.02 1.02 0 0 1 1.41 0l1.18 1.16L11.96 14a.44.44 0 0 0 .62 0 .44.44 0 0 0 0-.63L8.43 9.22a.99.99 0 0 1-.3-.7.99.99 0 0 1 .3-.7 1 1 0 0 1 1.41 0l7 6.98a.44.44 0 0 0 .7-.5l-1.35-2.85c-.31-.68-.23-1.19.25-1.56a.85.85 0 0 1 .66-.16c.34.06.66.28.88.6L20.63 15c1.57 2.88 1.07 5.54-1.55 8.16a5.62 5.62 0 0 1-5.06 1.65 9.35 9.35 0 0 1-4.93-2.72zM13 6.98l2.56 2.56c-.5.6-.56 1.41-.15 2.28l.26.56-4.25-4.25a.98.98 0 0 1-.12-.45 1 1 0 0 1 .29-.7 1.02 1.02 0 0 1 1.41 0zm8.89 2.06c-.38-.56-.9-.92-1.49-1.01a1.74 1.74 0 0 0-1.34.33c-.38.29-.61.65-.71 1.06a2.1 2.1 0 0 0-1.1-.56 1.78 1.78 0 0 0-.99.13l-2.64-2.64a1.88 1.88 0 0 0-2.65 0 1.86 1.86 0 0 0-.48.85 1.89 1.89 0 0 0-2.67-.01 1.87 1.87 0 0 0-.5.9c-.76-.75-2-.75-2.7-.04a1.88 1.88 0 0 0 0 2.66c-.3.12-.61.29-.87.55a1.88 1.88 0 0 0 0 2.66l.62.62a1.88 1.88 0 0 0-.9 3.16l5.01 5.02c1.6 1.6 3.52 2.64 5.4 2.96a7.16 7.16 0 0 0 1.18.1c1.03 0 2-.25 2.9-.7A5.9 5.9 0 0 0 23 23.24c3.34-3.34 3.08-6.93 1.74-9.17l-2.87-5.04z"></path></g></svg></div></a></span></div><div class="s qd qe qf qg qh qi qj"><div class="qk"><p class="ba b bb bc du"><button class="ei ej bz ca cb cc cd ce cf bl ek el cg em en">1.3K </button></p></div></div></div></div><div class="po s"><button class="ev pw ce"><div class="qn n o aw"><svg width="25" height="25" class="r" aria-label="responses"><path d="M19.07 21.12a6.33 6.33 0 0 1-3.53-1.1 7.8 7.8 0 0 1-.7-.52c-.77.21-1.57.32-2.38.32-4.67 0-8.46-3.5-8.46-7.8C4 7.7 7.79 4.2 12.46 4.2c4.66 0 8.46 3.5 8.46 7.8 0 2.06-.85 3.99-2.4 5.45a6.28 6.28 0 0 0 1.14 2.59c.15.21.17.48.06.7a.69.69 0 0 1-.62.38h-.03zm0-1v.5l.03-.5h-.03zm-3.92-1.64l.21.2a6.09 6.09 0 0 0 3.24 1.54 7.14 7.14 0 0 1-.83-1.84 5.15 5.15 0 0 1-.16-.75 2.4 2.4 0 0 1-.02-.29v-.23l.18-.15a6.6 6.6 0 0 0 2.3-4.96c0-3.82-3.4-6.93-7.6-6.93-4.19 0-7.6 3.11-7.6 6.93 0 3.83 3.41 6.94 7.6 6.94.83 0 1.64-.12 2.41-.35l.28-.08z" fill-rule="evenodd"></path></svg><div class="s am qo qp qq qr qs qt qu qv"><p class="ba b bb bc du">4<!-- --> </p></div></div></button></div><div class="jz"><span><a href="https://medium.com/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fbookmark%2Fp%2Ff77d5718061a&operation=register&redirect=https%3A%2F%2Fmedium.com%2Fapplied-data-science%2Fhow-to-build-your-own-muzero-in-python-f77d5718061a&source=post_sidebar-----f77d5718061a---------------------bookmark_sidebar-----------" class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener"><svg width="25" height="25" viewBox="0 0 25 25"><path d="M19 6a2 2 0 0 0-2-2H8a2 2 0 0 0-2 2v14.66h.01c.01.1.05.2.12.28a.5.5 0 0 0 .7.03l5.67-4.12 5.66 4.13a.5.5 0 0 0 .71-.03.5.5 0 0 0 .12-.29H19V6zm-6.84 9.97L7 19.64V6a1 1 0 0 1 1-1h9a1 1 0 0 1 1 1v13.64l-5.16-3.67a.49.49 0 0 0-.68 0z" fill-rule="evenodd"></path></svg></a></span></div></div></div></div></div></div></div><div class="xc acp oz fc pa jg pb pc pd pe"></div><div><div class="qw gp n ah p"><div class="n p"><div class="ao ap aq ar as gi au v"><div class="n ct"></div><div class="n o ct"></div><div class="qx qw s"><div class="qy n ck jw"><div class="n aw"><div class="qz s"><span class="s ra rb rc e d"><div class="n o"><div class="s am pq pr ps pt pu"><span><a href="https://medium.com/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fvote%2Fapplied-data-science%2Ff77d5718061a&operation=register&redirect=https%3A%2F%2Fmedium.com%2Fapplied-data-science%2Fhow-to-build-your-own-muzero-in-python-f77d5718061a&source=post_actions_footer-----f77d5718061a---------------------clap_footer-----------" class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener"><div class="ce pv pw px ev py pz qa r qb qc"><svg width="25" height="25" viewBox="0 0 25 25" aria-label="clap"><g fill-rule="evenodd"><path d="M11.74 0l.76 2.97.76-2.97zM14.81 3.78l1.84-2.56-1.42-.47zM8.38 1.22l1.84 2.56L9.8.75zM20.38 21.62a5.11 5.11 0 0 1-3.16 1.61l.49-.45c2.88-2.89 3.45-5.98 1.69-9.21l-1.1-1.94-.96-2.02c-.31-.67-.23-1.18.25-1.55a.84.84 0 0 1 .66-.16c.34.05.66.28.88.6l2.85 5.02c1.18 1.97 1.38 5.12-1.6 8.1M7.1 21.1l-5.02-5.02a1 1 0 0 1 .7-1.7 1 1 0 0 1 .72.3l2.6 2.6a.44.44 0 0 0 .63-.62L4.1 14.04l-1.75-1.75a1 1 0 1 1 1.41-1.41l4.15 4.15a.44.44 0 0 0 .63 0 .44.44 0 0 0 0-.62L4.4 10.26 3.22 9.08a1 1 0 0 1 0-1.4 1.02 1.02 0 0 1 1.41 0l1.18 1.16L9.96 13a.44.44 0 0 0 .62 0 .44.44 0 0 0 0-.63L6.43 8.22a.99.99 0 0 1-.3-.7.99.99 0 0 1 .3-.7 1 1 0 0 1 1.41 0l7 6.98a.44.44 0 0 0 .7-.5l-1.35-2.85c-.31-.68-.23-1.19.25-1.56a.85.85 0 0 1 .66-.16c.34.06.66.28.88.6L18.63 14c1.57 2.88 1.07 5.54-1.55 8.16a5.62 5.62 0 0 1-5.06 1.65 9.35 9.35 0 0 1-4.93-2.72zM11 5.98l2.56 2.56c-.5.6-.56 1.41-.15 2.28l.26.56-4.25-4.25a.98.98 0 0 1-.12-.45 1 1 0 0 1 .29-.7 1.02 1.02 0 0 1 1.41 0zm8.89 2.06c-.38-.56-.9-.92-1.49-1.01a1.74 1.74 0 0 0-1.34.33c-.38.29-.61.65-.71 1.06a2.1 2.1 0 0 0-1.1-.56 1.78 1.78 0 0 0-.99.13l-2.64-2.64a1.88 1.88 0 0 0-2.65 0 1.86 1.86 0 0 0-.48.85 1.89 1.89 0 0 0-2.67-.01 1.87 1.87 0 0 0-.5.9c-.76-.75-2-.75-2.7-.04a1.88 1.88 0 0 0 0 2.66c-.3.12-.61.29-.87.55a1.88 1.88 0 0 0 0 2.66l.62.62a1.88 1.88 0 0 0-.9 3.16l5.01 5.02c1.6 1.6 3.52 2.64 5.4 2.96a7.16 7.16 0 0 0 1.18.1c1.03 0 2-.25 2.9-.7A5.9 5.9 0 0 0 21 22.24c3.34-3.34 3.08-6.93 1.74-9.17l-2.87-5.04z"></path></g></svg></div></a></span></div><div class="s qd qe qf qg qh qi qj"><div class="am rd qk"><p class="ba b bb bc if"><button class="ei ej bz ca cb cc cd ce cf bl ek el cg em en">1.3K<span class="s h g f re rf"> </span></button><span class="s h g f re rf"></span></p></div></div></div></span><span class="s h g f re rf"><div class="n cq"><div class="s am pq pr"><span><a href="https://medium.com/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fvote%2Fapplied-data-science%2Ff77d5718061a&operation=register&redirect=https%3A%2F%2Fmedium.com%2Fapplied-data-science%2Fhow-to-build-your-own-muzero-in-python-f77d5718061a&source=post_actions_footer-----f77d5718061a---------------------clap_footer-----------" class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener"><div class="ce pv pw px ev py pz qa r qb qc"><svg width="33" height="33" viewBox="0 0 33 33" aria-label="clap"><path d="M28.86 17.34l-3.64-6.4c-.3-.43-.71-.73-1.16-.8a1.12 1.12 0 0 0-.9.21c-.62.5-.73 1.18-.32 2.06l1.22 2.6 1.4 2.45c2.23 4.09 1.51 8-2.15 11.66a9.6 9.6 0 0 1-.8.71 6.53 6.53 0 0 0 4.3-2.1c3.82-3.82 3.57-7.87 2.05-10.39zm-6.25 11.08c3.35-3.35 4-6.78 1.98-10.47L21.2 12c-.3-.43-.71-.72-1.16-.8a1.12 1.12 0 0 0-.9.22c-.62.49-.74 1.18-.32 2.06l1.72 3.63a.5.5 0 0 1-.81.57l-8.91-8.9a1.33 1.33 0 0 0-1.89 1.88l5.3 5.3a.5.5 0 0 1-.71.7l-5.3-5.3-1.49-1.49c-.5-.5-1.38-.5-1.88 0a1.34 1.34 0 0 0 0 1.89l1.49 1.5 5.3 5.28a.5.5 0 0 1-.36.86.5.5 0 0 1-.36-.15l-5.29-5.29a1.34 1.34 0 0 0-1.88 0 1.34 1.34 0 0 0 0 1.89l2.23 2.23L9.3 21.4a.5.5 0 0 1-.36.85.5.5 0 0 1-.35-.14l-3.32-3.33a1.33 1.33 0 0 0-1.89 0 1.32 1.32 0 0 0-.39.95c0 .35.14.69.4.94l6.39 6.4c3.53 3.53 8.86 5.3 12.82 1.35zM12.73 9.26l5.68 5.68-.49-1.04c-.52-1.1-.43-2.13.22-2.89l-3.3-3.3a1.34 1.34 0 0 0-1.88 0 1.33 1.33 0 0 0-.4.94c0 .22.07.42.17.61zm14.79 19.18a7.46 7.46 0 0 1-6.41 2.31 7.92 7.92 0 0 1-3.67.9c-3.05 0-6.12-1.63-8.36-3.88l-6.4-6.4A2.31 2.31 0 0 1 2 19.72a2.33 2.33 0 0 1 1.92-2.3l-.87-.87a2.34 2.34 0 0 1 0-3.3 2.33 2.33 0 0 1 1.24-.64l-.14-.14a2.34 2.34 0 0 1 0-3.3 2.39 2.39 0 0 1 3.3 0l.14.14a2.33 2.33 0 0 1 3.95-1.24l.09.09c.09-.42.29-.83.62-1.16a2.34 2.34 0 0 1 3.3 0l3.38 3.39a2.17 2.17 0 0 1 1.27-.17c.54.08 1.03.35 1.45.76.1-.55.41-1.03.9-1.42a2.12 2.12 0 0 1 1.67-.4 2.8 2.8 0 0 1 1.85 1.25l3.65 6.43c1.7 2.83 2.03 7.37-2.2 11.6zM13.22.48l-1.92.89 2.37 2.83-.45-3.72zm8.48.88L19.78.5l-.44 3.7 2.36-2.84zM16.5 3.3L15.48 0h2.04L16.5 3.3z" fill-rule="evenodd"></path></svg></div></a></span></div><div class="s qd qe qf qg rg rh ri rj rk rl"><div class="am rd qk"><p class="ba b bb bc if"><button class="ei ej bz ca cb cc cd ce cf bl ek el cg em en">1.3K<span class="s h g f re rf"> </span></button><span class="s h g f re rf"></span></p></div></div></div></span></div><div class="s rm rn ro rp rq"></div><button class="ev pw ce"><div class="qn n o aw"><div><div class="bv" role="tooltip" aria-hidden="false" aria-describedby="3" aria-labelledby="3"><span class="rr s h g f re rf"><svg width="33" height="33" viewBox="0 0 33 33" fill="none" class="r" aria-label="responses"><path clip-rule="evenodd" d="M24.28 25.5l.32-.29c2.11-1.94 3.4-4.61 3.4-7.56C28 11.83 22.92 7 16.5 7S5 11.83 5 17.65s5.08 10.66 11.5 10.66c1.22 0 2.4-.18 3.5-.5l.5-.15.41.33a8.86 8.86 0 0 0 4.68 2.1 7.34 7.34 0 0 1-1.3-4.15v-.43zm1 .45c0 1.5.46 2.62 1.69 4.44.22.32.01.75-.38.75a9.69 9.69 0 0 1-6.31-2.37c-1.2.35-2.46.54-3.78.54C9.6 29.3 4 24.09 4 17.65 4 11.22 9.6 6 16.5 6S29 11.22 29 17.65c0 3.25-1.42 6.18-3.72 8.3z"></path></svg></span><span class="rs s ra rb rc e d"><svg width="25" height="25" class="r" aria-label="responses"><path d="M19.07 21.12a6.33 6.33 0 0 1-3.53-1.1 7.8 7.8 0 0 1-.7-.52c-.77.21-1.57.32-2.38.32-4.67 0-8.46-3.5-8.46-7.8C4 7.7 7.79 4.2 12.46 4.2c4.66 0 8.46 3.5 8.46 7.8 0 2.06-.85 3.99-2.4 5.45a6.28 6.28 0 0 0 1.14 2.59c.15.21.17.48.06.7a.69.69 0 0 1-.62.38h-.03zm0-1v.5l.03-.5h-.03zm-3.92-1.64l.21.2a6.09 6.09 0 0 0 3.24 1.54 7.14 7.14 0 0 1-.83-1.84 5.15 5.15 0 0 1-.16-.75 2.4 2.4 0 0 1-.02-.29v-.23l.18-.15a6.6 6.6 0 0 0 2.3-4.96c0-3.82-3.4-6.93-7.6-6.93-4.19 0-7.6 3.11-7.6 6.93 0 3.83 3.41 6.94 7.6 6.94.83 0 1.64-.12 2.41-.35l.28-.08z" fill-rule="evenodd"></path></svg></span></div></div><div class="s am rt qp ru qr rv qt rw rx ry rz"><p class="ba b bb bc if">4<!-- --> </p></div></div></button></div><div class="n o"><div class="jx s"><div class="bv" aria-hidden="false" aria-describedby="postFooterSocialMenu" aria-labelledby="postFooterSocialMenu"><div><div class="bv" role="tooltip" aria-hidden="false" aria-describedby="4" aria-labelledby="4"><button class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" aria-controls="postFooterSocialMenu" aria-expanded="false" aria-label="Share Post"><svg width="25" height="25" class="r"><g fill-rule="evenodd"><path d="M15.6 5a.42.42 0 0 0 .17-.3.42.42 0 0 0-.12-.33l-2.8-2.79a.5.5 0 0 0-.7 0l-2.8 2.8a.4.4 0 0 0-.1.32c0 .12.07.23.16.3h.02a.45.45 0 0 0 .57-.04l2-2V10c0 .28.23.5.5.5s.5-.22.5-.5V2.93l2.02 2.02c.08.07.18.12.3.13.11.01.21-.02.3-.08v.01"></path><path d="M18 7h-1.5a.5.5 0 0 0 0 1h1.6c.5 0 .9.4.9.9v10.2c0 .5-.4.9-.9.9H6.9a.9.9 0 0 1-.9-.9V8.9c0-.5.4-.9.9-.9h1.6a.5.5 0 0 0 .35-.15A.5.5 0 0 0 9 7.5a.5.5 0 0 0-.15-.35A.5.5 0 0 0 8.5 7H7a2 2 0 0 0-2 2v10c0 1.1.9 2 2 2h11a2 2 0 0 0 2-2V9a2 2 0 0 0-2-2"></path></g></svg></button></div></div></div></div><div class="jx s db"><div class="jz"><span><a href="https://medium.com/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fbookmark%2Fp%2Ff77d5718061a&operation=register&redirect=https%3A%2F%2Fmedium.com%2Fapplied-data-science%2Fhow-to-build-your-own-muzero-in-python-f77d5718061a&source=post_actions_footer--------------------------bookmark_footer-----------" class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener"><svg width="25" height="25" viewBox="0 0 25 25"><path d="M19 6a2 2 0 0 0-2-2H8a2 2 0 0 0-2 2v14.66h.01c.01.1.05.2.12.28a.5.5 0 0 0 .7.03l5.67-4.12 5.66 4.13a.5.5 0 0 0 .71-.03.5.5 0 0 0 .12-.29H19V6zm-6.84 9.97L7 19.64V6a1 1 0 0 1 1-1h9a1 1 0 0 1 1 1v13.64l-5.16-3.67a.49.49 0 0 0-.68 0z" fill-rule="evenodd"></path></svg></a></span></div></div></div></div></div><div class="sa qy s"><ul class="ce cf"><li class="bv sb jy sc"><a href="https://medium.com/applied-data-science/tagged/machine-learning" class="ba b mv sd du se sf bw s sg">Machine Learning</a></li><li class="bv sb jy sc"><a href="https://medium.com/applied-data-science/tagged/deep-learning" class="ba b mv sd du se sf bw s sg">Deep Learning</a></li><li class="bv sb jy sc"><a href="https://medium.com/applied-data-science/tagged/data-science" class="ba b mv sd du se sf bw s sg">Data Science</a></li><li class="bv sb jy sc"><a href="https://medium.com/applied-data-science/tagged/artificial-intelligence" class="ba b mv sd du se sf bw s sg">Artificial Intelligence</a></li><li class="bv sb jy sc"><a href="https://medium.com/applied-data-science/tagged/ai" class="ba b mv sd du se sf bw s sg">AI</a></li></ul></div></div></div><div><div class="n p"><div class="ao ap aq ar as gi au v"></div></div><div class="s jw"><div class="sh si s sj"><div class="n p"><div class="ao ap aq ar as gi au v"><div class="n o ck"><h2 class="ba lk sk sl sm ln sn so sp lq sq sr ss lu st su sv ly sw sx sy mc gw mo mp sz mr ms if"><a href="https://medium.com/applied-data-science?source=follow_footer-------------------------------------" class="ei ej bz ca cb cc cd ce cf bl ek el cg em en" rel="noopener">More from Applied Data Science</a></h2><div class="bv" aria-hidden="false" aria-describedby="collectionFollowPopover" aria-labelledby="collectionFollowPopover"><span><button class="ba b bb bc dy be dz ea eb ec ed bl bm bn ee ef br bs bt bu bv bw"><div class="n aw">Follow</div></button></span></div></div><div class="nq ta s"><p class="ba b bb bc du">Cutting edge data science, machine learning and AI projects</p></div></div></div></div></div><div class="tb s sj jw"><div class="n p"><div class="nh nj nl tc td nc au v"><div class="te qw s"><div class="nc s ns"><a href="https://medium.com/applied-data-science?source=follow_footer-------------------------------------" class="ba b bb bc dy be dz ea eb ec ed bl bm bn ee ef br bs bt bu bv bw" rel="noopener">Read more from <!-- -->Applied Data Science</a></div></div></div></div></div><div class="s fv jw"><div class="n p"><div class="ao ap aq ar as at au v"><div class="tf tg s"><div class="th pk ti tg s tj tk"><h2 class="ba lk nx ij ln nz im lq in ip lu iq is ly it iv mc if">More From Medium</h2></div><div class="cq n aw ct tl tm tn to tp tq tr ts tt tu tv tw tx ty tz"><div class="ua ub uc ni ud ue uf nk ug uh ui uj uk ul um un uo up uq ur us"><div class="ut uu s"><div class="v gv"><div class="n ck"><div class="s uv uw ux uy"><div class="uz s"><h2 class="ba lk sk sl ln sn so lq va vb lu vc vd ly ve vf mc if"><a rel="noopener" href="https://medium.com/applied-data-science/of-emotional-machines-and-irrational-men-70ffe0a429a2?source=post_internal_links---------0----------------------------">Of Emotional Machines and Irrational Men</a></h2></div><div class="o n"><div></div><div class="v s"><div class="n"><div style="flex:1"><span class="ba b bb bc if"><div class="cr n o vg"><span class="ba b mv bc if"><a class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener" href="https://medium.com/@elennisioti?source=post_internal_links---------0----------------------------">Eleni Nisioti</a><span> <!-- -->in<!-- --> <a href="https://medium.com/applied-data-science?source=post_internal_links---------0----------------------------" class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener">Applied Data Science</a></span></span></div></span></div></div></div></div></div><div class="eo jy s vi vj"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en s" rel="noopener" href="https://medium.com/applied-data-science/of-emotional-machines-and-irrational-men-70ffe0a429a2?source=post_internal_links---------0----------------------------"><div class="gz s am ha"><div class="lh hc s"><div class="xc xy eq fe fa gv v gw gx gy"><img class="eq fe fa gv v hd he hf" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_Uw_5AXGiULKKHqU6mEqYqg.jpeg" width="70" height="70" role="presentation"></div><img class="fb gu vk vl vm vn vo vp vq vr vs vt c" width="70" height="70" role="presentation"><noscript><img class="vk vl vm vn vo vp vq vr vs vt" src="https://miro.medium.com/fit/c/140/140/1*Uw_5AXGiULKKHqU6mEqYqg.jpeg" width="70" height="70" srcSet="https://miro.medium.com/fit/c/96/140/1*Uw_5AXGiULKKHqU6mEqYqg.jpeg 48w, https://miro.medium.com/fit/c/140/140/1*Uw_5AXGiULKKHqU6mEqYqg.jpeg 70w" sizes="70px" role="presentation"/></noscript></div></div></a></div></div></div></div></div><div class="ua ub uc ni ud ue uf nk ug uh ui uj uk ul um un uo up uq ur us"><div class="ut uu s"><div class="v gv"><div class="n ck"><div class="s uv uw ux uy"><div class="uz s"><h2 class="ba lk sk sl ln sn so lq va vb lu vc vd ly ve vf mc if"><a rel="noopener" href="https://medium.com/applied-data-science/how-to-train-ai-agents-to-play-multiplayer-games-using-self-play-deep-reinforcement-learning-247d0b440717?source=post_internal_links---------1----------------------------">How To Build Your Own AI To Play Any Board Game</a></h2></div><div class="o n"><div></div><div class="v s"><div class="n"><div style="flex:1"><span class="ba b bb bc if"><div class="cr n o vg"><span class="ba b mv bc if"><a class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener" href="https://medium.com/@dtfoster?source=post_internal_links---------1----------------------------">David Foster</a><span> <!-- -->in<!-- --> <a href="https://medium.com/applied-data-science?source=post_internal_links---------1----------------------------" class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener">Applied Data Science</a></span></span></div></span></div></div></div></div></div><div class="eo jy s vi vj"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en s" rel="noopener" href="https://medium.com/applied-data-science/how-to-train-ai-agents-to-play-multiplayer-games-using-self-play-deep-reinforcement-learning-247d0b440717?source=post_internal_links---------1----------------------------"><div class="gz s am ha"><div class="lh hc s"><div class="xc xy eq fe fa gv v gw gx gy"><img class="eq fe fa gv v hd he hf" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_y93HRfJgLjZno0g7fb4wgw.png" width="70" height="70" role="presentation"></div><img class="fb gu vk vl vm vn vo vp vq vr vs vt c" width="70" height="70" role="presentation"><noscript><img class="vk vl vm vn vo vp vq vr vs vt" src="https://miro.medium.com/fit/c/140/140/1*y93HRfJgLjZno0g7fb4wgw.png" width="70" height="70" srcSet="https://miro.medium.com/fit/c/96/140/1*y93HRfJgLjZno0g7fb4wgw.png 48w, https://miro.medium.com/fit/c/140/140/1*y93HRfJgLjZno0g7fb4wgw.png 70w" sizes="70px" role="presentation"/></noscript></div></div></a></div></div></div></div></div><div class="ua ub uc ni ud ue uf nk ug uh ui uj uk ul um un uo up uq ur us"><div class="ut uu s"><div class="v gv"><div class="n ck"><div class="s uv uw ux uy"><div class="uz s"><h2 class="ba lk sk sl ln sn so lq va vb lu vc vd ly ve vf mc if"><a rel="noopener" href="https://medium.com/applied-data-science/56-070-165-facebook-ad-spend-of-us-presidential-candidates-broken-down-by-age-and-gender-2dcc32fe2c02?source=post_internal_links---------2----------------------------">We Analysed the 527,350 Facebook Ads of the US Presidential Candidates. Here Are The Results.</a></h2></div><div class="o n"><div></div><div class="v s"><div class="n"><div style="flex:1"><span class="ba b bb bc if"><div class="cr n o vg"><span class="ba b mv bc if"><a class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener" href="https://medium.com/@dtfoster?source=post_internal_links---------2----------------------------">David Foster</a><span> <!-- -->in<!-- --> <a href="https://medium.com/applied-data-science?source=post_internal_links---------2----------------------------" class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener">Applied Data Science</a></span></span></div></span></div></div></div></div></div><div class="eo jy s vi vj"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en s" rel="noopener" href="https://medium.com/applied-data-science/56-070-165-facebook-ad-spend-of-us-presidential-candidates-broken-down-by-age-and-gender-2dcc32fe2c02?source=post_internal_links---------2----------------------------"><div class="gz s am ha"><div class="lh hc s"><div class="xc xy eq fe fa gv v gw gx gy"><img class="eq fe fa gv v hd he hf" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_UMh-U2rra06VIhkVcBi8HQ.png" width="70" height="70" role="presentation"></div><img class="fb gu vk vl vm vn vo vp vq vr vs vt c" width="70" height="70" role="presentation"><noscript><img class="vk vl vm vn vo vp vq vr vs vt" src="https://miro.medium.com/fit/c/140/140/1*UMh-U2rra06VIhkVcBi8HQ.png" width="70" height="70" srcSet="https://miro.medium.com/fit/c/96/140/1*UMh-U2rra06VIhkVcBi8HQ.png 48w, https://miro.medium.com/fit/c/140/140/1*UMh-U2rra06VIhkVcBi8HQ.png 70w" sizes="70px" role="presentation"/></noscript></div></div></a></div></div></div></div></div><div class="ua ub uc ni ud ue uf nk ug uh ui uj uk ul um un uo up uq ur us"><div class="ut uu s"><div class="v gv"><div class="n ck"><div class="s uv uw ux uy"><div class="uz s"><h2 class="ba lk sk sl ln sn so lq va vb lu vc vd ly ve vf mc if"><a rel="noopener" href="https://medium.com/applied-data-science/what-can-you-do-with-the-openai-gpt-3-language-model-d95e1d4fe558?source=post_internal_links---------3----------------------------">What Can You Do with the OpenAI GPT-3 Language Model?</a></h2></div><div class="o n"><div></div><div class="v s"><div class="n"><div style="flex:1"><span class="ba b bb bc if"><div class="cr n o vg"><span class="ba b mv bc if"><a class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener" href="https://medium.com/@james.montantes.exxact?source=post_internal_links---------3----------------------------">James Montantes</a><span> <!-- -->in<!-- --> <a href="https://medium.com/applied-data-science?source=post_internal_links---------3----------------------------" class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener">Applied Data Science</a></span></span></div></span></div></div></div></div></div><div class="eo jy s vi vj"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en s" rel="noopener" href="https://medium.com/applied-data-science/what-can-you-do-with-the-openai-gpt-3-language-model-d95e1d4fe558?source=post_internal_links---------3----------------------------"><div class="gz s am ha"><div class="lh hc s"><div class="xc xy eq fe fa gv v gw gx gy"><img class="eq fe fa gv v hd he hf" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_WGic6ivmGu1dfWtmHC865w.jpeg" width="70" height="70" role="presentation"></div><img class="fb gu vk vl vm vn vo vp vq vr vs vt c" width="70" height="70" role="presentation"><noscript><img class="vk vl vm vn vo vp vq vr vs vt" src="https://miro.medium.com/fit/c/140/140/1*WGic6ivmGu1dfWtmHC865w.jpeg" width="70" height="70" srcSet="https://miro.medium.com/fit/c/96/140/1*WGic6ivmGu1dfWtmHC865w.jpeg 48w, https://miro.medium.com/fit/c/140/140/1*WGic6ivmGu1dfWtmHC865w.jpeg 70w" sizes="70px" role="presentation"/></noscript></div></div></a></div></div></div></div></div><div class="ua ub uc ni ud ue uf nk ug uh ui uj uk ul um un uo up uq ur us"><div class="ut uu s"><div class="v gv"><div class="n ck"><div class="s uv uw ux uy"><div class="uz s"><h2 class="ba lk sk sl ln sn so lq va vb lu vc vd ly ve vf mc if"><a rel="noopener" href="https://medium.com/applied-data-science/coronavirus-in-context-93e03bfdc151?source=post_internal_links---------4----------------------------">How To Put Coronavirus Into Context</a></h2></div><div class="o n"><div></div><div class="v s"><div class="n"><div style="flex:1"><span class="ba b bb bc if"><div class="cr n o vg"><span class="ba b mv bc if"><a class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener" href="https://medium.com/@dtfoster?source=post_internal_links---------4----------------------------">David Foster</a><span> <!-- -->in<!-- --> <a href="https://medium.com/applied-data-science?source=post_internal_links---------4----------------------------" class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener">Applied Data Science</a></span></span></div></span></div></div></div></div></div><div class="eo jy s vi vj"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en s" rel="noopener" href="https://medium.com/applied-data-science/coronavirus-in-context-93e03bfdc151?source=post_internal_links---------4----------------------------"><div class="gz s am ha"><div class="lh hc s"><div class="xc xy eq fe fa gv v gw gx gy"><img class="eq fe fa gv v hd he hf" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_3Zc12UKeiAI2--bi9F6xXw.png" width="70" height="70" role="presentation"></div><img class="fb gu vk vl vm vn vo vp vq vr vs vt c" width="70" height="70" role="presentation"><noscript><img class="vk vl vm vn vo vp vq vr vs vt" src="https://miro.medium.com/fit/c/140/140/1*3Zc12UKeiAI2--bi9F6xXw.png" width="70" height="70" srcSet="https://miro.medium.com/fit/c/96/140/1*3Zc12UKeiAI2--bi9F6xXw.png 48w, https://miro.medium.com/fit/c/140/140/1*3Zc12UKeiAI2--bi9F6xXw.png 70w" sizes="70px" role="presentation"/></noscript></div></div></a></div></div></div></div></div><div class="ua ub uc ni ud ue uf nk ug uh ui uj uk ul um un uo up uq ur us"><div class="ut uu s"><div class="v gv"><div class="n ck"><div class="s uv uw ux uy"><div class="uz s"><h2 class="ba lk sk sl ln sn so lq va vb lu vc vd ly ve vf mc if"><a rel="noopener" href="https://medium.com/applied-data-science/full-stack-data-scientist-5-automating-report-generation-with-jupyter-notebooks-919e32e88d18?source=post_internal_links---------5----------------------------">Full Stack Data Scientist 5: Automating report generation with Jupyter Notebooks</a></h2></div><div class="o n"><div></div><div class="v s"><div class="n"><div style="flex:1"><span class="ba b bb bc if"><div class="cr n o vg"><span class="ba b mv bc if"><a class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener" href="https://medium.com/@dsharpc?source=post_internal_links---------5----------------------------">Daniel Sharp</a><span> <!-- -->in<!-- --> <a href="https://medium.com/applied-data-science?source=post_internal_links---------5----------------------------" class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener">Applied Data Science</a></span></span></div></span></div></div></div></div></div><div class="eo jy s vi vj"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en s" rel="noopener" href="https://medium.com/applied-data-science/full-stack-data-scientist-5-automating-report-generation-with-jupyter-notebooks-919e32e88d18?source=post_internal_links---------5----------------------------"><div class="gz s am ha"><div class="lh hc s"><div class="xc xy eq fe fa gv v gw gx gy"><img class="eq fe fa gv v hd he hf" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_aQaPAEsi6kuRWScVPOsS7Q.jpeg" width="70" height="70" role="presentation"></div><img class="fb gu vk vl vm vn vo vp vq vr vs vt c" width="70" height="70" role="presentation"><noscript><img class="vk vl vm vn vo vp vq vr vs vt" src="https://miro.medium.com/fit/c/140/140/1*aQaPAEsi6kuRWScVPOsS7Q.jpeg" width="70" height="70" srcSet="https://miro.medium.com/fit/c/96/140/1*aQaPAEsi6kuRWScVPOsS7Q.jpeg 48w, https://miro.medium.com/fit/c/140/140/1*aQaPAEsi6kuRWScVPOsS7Q.jpeg 70w" sizes="70px" role="presentation"/></noscript></div></div></a></div></div></div></div></div><div class="ua ub uc ni ud ue uf nk ug uh ui uj uk ul um un uo up uq ur us"><div class="ut uu s"><div class="v gv"><div class="n ck"><div class="s uv uw ux uy"><div class="uz s"><h2 class="ba lk sk sl ln sn so lq va vb lu vc vd ly ve vf mc if"><a rel="noopener" href="https://medium.com/applied-data-science/survival-%CE%B1nalysis-ddebd92ae26c?source=post_internal_links---------6----------------------------">Survival Αnalysis</a></h2></div><div class="o n"><div></div><div class="v s"><div class="n"><div style="flex:1"><span class="ba b bb bc if"><div class="cr n o vg"><span class="ba b mv bc if"><a class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener" href="https://medium.com/@elennisioti?source=post_internal_links---------6----------------------------">Eleni Nisioti</a><span> <!-- -->in<!-- --> <a href="https://medium.com/applied-data-science?source=post_internal_links---------6----------------------------" class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener">Applied Data Science</a></span></span></div></span></div></div></div></div></div><div class="eo jy s vi vj"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en s" rel="noopener" href="https://medium.com/applied-data-science/survival-%CE%B1nalysis-ddebd92ae26c?source=post_internal_links---------6----------------------------"><div class="gz s am ha"><div class="lh hc s"><div class="xc xy eq fe fa gv v gw gx gy"><img class="eq fe fa gv v hd he hf" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_wJcVGzmj7r8ZM6XVFfdtLA.jpeg" width="70" height="70" role="presentation"></div><img class="fb gu vk vl vm vn vo vp vq vr vs vt c" width="70" height="70" role="presentation"><noscript><img class="vk vl vm vn vo vp vq vr vs vt" src="https://miro.medium.com/fit/c/140/140/1*wJcVGzmj7r8ZM6XVFfdtLA.jpeg" width="70" height="70" srcSet="https://miro.medium.com/fit/c/96/140/1*wJcVGzmj7r8ZM6XVFfdtLA.jpeg 48w, https://miro.medium.com/fit/c/140/140/1*wJcVGzmj7r8ZM6XVFfdtLA.jpeg 70w" sizes="70px" role="presentation"/></noscript></div></div></a></div></div></div></div></div><div class="ua ub uc ni ud ue uf nk ug uh ui uj uk ul um un uo up uq ur us"><div class="ut uu s"><div class="v gv"><div class="n ck"><div class="s uv uw ux uy"><div class="uz s"><h2 class="ba lk sk sl ln sn so lq va vb lu vc vd ly ve vf mc if"><a rel="noopener" href="https://medium.com/applied-data-science/the-pandepreneurs-the-rise-of-new-companies-through-lockdown-and-market-adaptability-482271dfb8fd?source=post_internal_links---------7----------------------------">The Pandepreneurs: the rise of new companies through lockdown and market adaptability</a></h2></div><div class="o n"><div></div><div class="v s"><div class="n"><div style="flex:1"><span class="ba b bb bc if"><div class="cr n o vg"><span class="ba b mv bc if"><a class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener" href="https://medium.com/@dsharpc?source=post_internal_links---------7----------------------------">Daniel Sharp</a><span> <!-- -->in<!-- --> <a href="https://medium.com/applied-data-science?source=post_internal_links---------7----------------------------" class="ei ej bz ca cb cc cd ce cf bl vh cg em en" rel="noopener">Applied Data Science</a></span></span></div></span></div></div></div></div></div><div class="eo jy s vi vj"><a class="ei ej bz ca cb cc cd ce cf bl ek el cg em en s" rel="noopener" href="https://medium.com/applied-data-science/the-pandepreneurs-the-rise-of-new-companies-through-lockdown-and-market-adaptability-482271dfb8fd?source=post_internal_links---------7----------------------------"><div class="gz s am ha"><div class="lh hc s"><div class="xc xy eq fe fa gv v gw gx gy"><img class="eq fe fa gv v hd he hf" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_vrj_U6tBMYzLADoE37jQ4g.jpeg" width="70" height="70" role="presentation"></div><img class="fb gu vk vl vm vn vo vp vq vr vs vt c" width="70" height="70" role="presentation"><noscript><img class="vk vl vm vn vo vp vq vr vs vt" src="https://miro.medium.com/fit/c/140/140/1*vrj_U6tBMYzLADoE37jQ4g.jpeg" width="70" height="70" srcSet="https://miro.medium.com/fit/c/96/140/1*vrj_U6tBMYzLADoE37jQ4g.jpeg 48w, https://miro.medium.com/fit/c/140/140/1*vrj_U6tBMYzLADoE37jQ4g.jpeg 70w" sizes="70px" role="presentation"/></noscript></div></div></a></div></div></div></div></div></div></div></div></div></div></div></div></div><div class="vu s vv vw"><div class="n p"><div class="ao ap aq ar as at au v"><div class="n ah"><div class="n o ck"><a class="ei ej bz ca cb cc cd ce cf bl vx vy cg vz wa" rel="noopener" href="https://medium.com/?source=post_page-----f77d5718061a--------------------------------"><svg viewBox="0 0 3940 610" class="dz wb"><path d="M594.79 308.2c0 163.76-131.85 296.52-294.5 296.52S5.8 472 5.8 308.2 137.65 11.69 300.29 11.69s294.5 132.75 294.5 296.51M917.86 308.2c0 154.16-65.93 279.12-147.25 279.12s-147.25-125-147.25-279.12S689.29 29.08 770.61 29.08s147.25 125 147.25 279.12M1050 308.2c0 138.12-23.19 250.08-51.79 250.08s-51.79-112-51.79-250.08 23.19-250.08 51.8-250.08S1050 170.09 1050 308.2M1862.77 37.4l.82-.18v-6.35h-167.48l-155.51 365.5-155.51-365.5h-180.48v6.35l.81.18c30.57 6.9 46.09 17.19 46.09 54.3v434.45c0 37.11-15.58 47.4-46.15 54.3l-.81.18V587H1327v-6.35l-.81-.18c-30.57-6.9-46.09-17.19-46.09-54.3V116.9L1479.87 587h11.33l205.59-483.21V536.9c-2.62 29.31-18 38.36-45.68 44.61l-.82.19v6.3h213.3v-6.3l-.82-.19c-27.71-6.25-43.46-15.3-46.08-44.61l-.14-445.2h.14c0-37.11 15.52-47.4 46.08-54.3m97.43 287.8c3.49-78.06 31.52-134.4 78.56-135.37 14.51.24 26.68 5 36.14 14.16 20.1 19.51 29.55 60.28 28.09 121.21zm-2.11 22h250v-1.05c-.71-59.69-18-106.12-51.34-138-28.82-27.55-71.49-42.71-116.31-42.71h-1c-23.26 0-51.79 5.64-72.09 15.86-23.11 10.7-43.49 26.7-60.45 47.7-27.3 33.83-43.84 79.55-47.86 130.93-.13 1.54-.24 3.08-.35 4.62s-.18 2.92-.25 4.39a332.64 332.64 0 0 0-.36 21.69C1860.79 507 1923.65 600 2035.3 600c98 0 155.07-71.64 169.3-167.8l-7.19-2.53c-25 51.68-69.9 83-121 79.18-69.76-5.22-123.2-75.95-118.35-161.63m532.69 157.68c-8.2 19.45-25.31 30.15-48.24 30.15s-43.89-15.74-58.78-44.34c-16-30.7-24.42-74.1-24.42-125.51 0-107 33.28-176.21 84.79-176.21 21.57 0 38.55 10.7 46.65 29.37zm165.84 76.28c-30.57-7.23-46.09-18-46.09-57V5.28L2424.77 60v6.7l1.14-.09c25.62-2.07 43 1.47 53.09 10.79 7.9 7.3 11.75 18.5 11.75 34.26v71.14c-18.31-11.69-40.09-17.38-66.52-17.38-53.6 0-102.59 22.57-137.92 63.56-36.83 42.72-56.3 101.1-56.3 168.81C2230 518.72 2289.53 600 2378.13 600c51.83 0 93.53-28.4 112.62-76.3V588h166.65v-6.66zm159.29-505.33c0-37.76-28.47-66.24-66.24-66.24-37.59 0-67 29.1-67 66.24s29.44 66.24 67 66.24c37.77 0 66.24-28.48 66.24-66.24m43.84 505.33c-30.57-7.23-46.09-18-46.09-57h-.13V166.65l-166.66 47.85v6.5l1 .09c36.06 3.21 45.93 15.63 45.93 57.77V588h166.8v-6.66zm427.05 0c-30.57-7.23-46.09-18-46.09-57V166.65L3082 212.92v6.52l.94.1c29.48 3.1 38 16.23 38 58.56v226c-9.83 19.45-28.27 31-50.61 31.78-36.23 0-56.18-24.47-56.18-68.9V166.66l-166.66 47.85V221l1 .09c36.06 3.2 45.94 15.62 45.94 57.77v191.27a214.48 214.48 0 0 0 3.47 39.82l3 13.05c14.11 50.56 51.08 77 109 77 49.06 0 92.06-30.37 111-77.89v66h166.66v-6.66zM3934.2 588v-6.67l-.81-.19c-33.17-7.65-46.09-22.07-46.09-51.43v-243.2c0-75.83-42.59-121.09-113.93-121.09-52 0-95.85 30.05-112.73 76.86-13.41-49.6-52-76.86-109.06-76.86-50.12 0-89.4 26.45-106.25 71.13v-69.87l-166.66 45.89v6.54l1 .09c35.63 3.16 45.93 15.94 45.93 57V588h155.5v-6.66l-.82-.2c-26.46-6.22-35-17.56-35-46.66V255.72c7-16.35 21.11-35.72 49-35.72 34.64 0 52.2 24 52.2 71.28V588h155.54v-6.66l-.82-.2c-26.46-6.22-35-17.56-35-46.66v-248a160.45 160.45 0 0 0-2.2-27.68c7.42-17.77 22.34-38.8 51.37-38.8 35.13 0 52.2 23.31 52.2 71.28V588z"></path></svg></a><div class="wc wd n ck we cm"><p class="ba b dk dl wf"><a href="https://medium.com/about?autoplay=1&source=post_page-----f77d5718061a--------------------------------" class="ei ej bz ca cb cc cd ce cf bl vh cg vz wa" rel="noopener">About</a></p><p class="ba b dk dl wf"><a href="https://help.medium.com/hc/en-us?source=post_page-----f77d5718061a--------------------------------" class="ei ej bz ca cb cc cd ce cf bl vh cg vz wa" rel="noopener">Help</a></p><p class="ba b dk dl wf"><a href="https://policy.medium.com/medium-terms-of-service-9db0094a1e0f?source=post_page-----f77d5718061a--------------------------------" class="ei ej bz ca cb cc cd ce cf bl vh cg vz wa" rel="noopener">Legal</a></p></div></div><div class="aj wg wh cm"><p class="ba b dk dl wi">Get the Medium app</p></div><div class="aj wg wj cm wk"><div class="az s"><a href="https://itunes.apple.com/app/medium-everyones-stories/id828256236?pt=698524&mt=8&ct=post_page&source=post_page-----f77d5718061a--------------------------------" class="ei ej bz ca cb cc cd ce cf bl vx vy cg vz wa" rel="noopener nofollow"><img alt="A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store" class="" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_Crl55Tm6yDNMoucPo1tvDg.png" width="135" height="41"></a></div><div class="s"><a href="https://play.google.com/store/apps/details?id=com.medium.reader&source=post_page-----f77d5718061a--------------------------------" class="ei ej bz ca cb cc cd ce cf bl vx vy cg vz wa" rel="noopener nofollow"><img alt="A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store" class="" src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1_W_RAPQ62h0em559zluJLdQ.png" width="135" height="41"></a></div></div></div></div></div></div></div></div></div><script>window.__BUILD_ID__ = "main-20210305-183757-c310e81f10"</script><script>window.__GRAPHQL_URI__ = "https://medium.com/_/graphql"</script><script>window.__PRELOADED_STATE__ = {"auroraPage":{"isAuroraPageEnabled":true},"bookReader":{"assets":{},"reader":{"currentAsset":null,"settingsPanelIsOpen":false,"settings":{"fontFamily":"FELL","fontScale":"M","publisherStyling":true,"textAlignment":"START","theme":"White","lineSpacing":0,"wordSpacing":0,"letterSpacing":0},"internalNavCounter":0}},"cache":{"experimentGroupSet":false,"group":"disabled","tags":[],"serverVariantState":""},"client":{"isUs":false,"isNativeMedium":false,"isSafariMobile":false,"isSafari":false,"routingEntity":{"type":"DEFAULT","explicit":false},"supportsWebp":true},"config":{"nodeEnv":"production","version":"main-20210305-183757-c310e81f10","isTaggedVersion":false,"target":"production","productName":"Medium","publicUrl":"https:\u002F\u002Fcdn-client.medium.com\u002Flite","authDomain":"medium.com","authGoogleClientId":"216296035834-k1k6qe060s2tp2a2jam4ljdcms00sttg.apps.googleusercontent.com","favicon":"production","glyphUrl":"https:\u002F\u002Fglyph.medium.com","branchKey":"key_live_ofxXr2qTrrU9NqURK8ZwEhknBxiI6KBm","lightStep":{"name":"lite-web","host":"lightstep.medium.systems","token":"ce5be895bef60919541332990ac9fef2","appVersion":"main-20210305-183757-c310e81f10"},"algolia":{"appId":"MQ57UUUQZ2","apiKeySearch":"394474ced050e3911ae2249ecc774921","indexPrefix":"medium_","host":"-dsn.algolia.net"},"recaptchaKey":"6Lfc37IUAAAAAKGGtC6rLS13R1Hrw_BqADfS1LRk","recaptcha3Key":"6Lf8R9wUAAAAABMI_85Wb8melS7Zj6ziuf99Yot5","datadog":{"clientToken":"pub853ea8d17ad6821d9f8f11861d23dfed","context":{"deployment":{"target":"production","tag":"main-20210305-183757-c310e81f10","commit":"c310e81f1083a2494066966a3d5a2eb55e2d87ae"}},"datacenter":"us"},"isAmp":false,"googleAnalyticsCode":"UA-24232453-2","signInWallCustomDomainCollectionIds":["3a8144eabfe3","336d898217ee","61061eb0c96b","138adf9c44c","819cc2aaeee0"],"mediumOwnedAndOperatedCollectionIds":["544c7006046e","bcc38c8f6edf","444d13b52878","8d6b8a439e32","92d2092dc598","1285ba81cada","cb8577c9149e","8ccfed20cbb2","ae2a65f35510","3f6ecf56618","7b6769f2748b","fc8964313712","ef8e90590e66","191186aaafa0","d944778ce714","bdc4052bbdba","88d9857e584e","9dc80918cc93","8a9336e5bb4","cef6983b292","54c98c43354d","193b68bd4fba","b7e45b22fec3","55760f21cdc5"],"tierOneDomains":["medium.com","thebolditalic.com","arcdigital.media","towardsdatascience.com","uxdesign.cc","codeburst.io","psiloveyou.xyz","writingcooperative.com","entrepreneurshandbook.co","prototypr.io","betterhumans.coach.me","theascent.pub"],"topicsToFollow":["d61cf867d93f","8a146bc21b28","1eca0103fff3","4d562ee63426","aef1078a3ef5","e15e46793f8d","6158eb913466","55f1c20aba7a","3d18b94f6858","4861fee224fd","63c6f1f93ee","1d98b3a9a871","decb52b64abf","ae5d4995e225","830cded25262"],"defaultImages":{"avatar":{"imageId":"1*dmbNkD5D-u45r44go_cf0g.png","height":150,"width":150},"orgLogo":{"imageId":"1*OMF3fSqH8t4xBJ9-6oZDZw.png","height":106,"width":545},"postLogo":{"imageId":"1*kFrc4tBFM_tCis-2Ic87WA.png","height":810,"width":1440},"postPreviewImage":{"imageId":"1*hn4v1tCaJy7cWMyb0bpNpQ.png","height":386,"width":579}},"performanceTags":[],"collectionStructuredData":{"8d6b8a439e32":{"name":"Elemental","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fcdn-images-1.medium.com\u002Fmax\u002F980\u002F1*9ygdqoKprhwuTVKUM0DLPA@2x.png","width":980,"height":159}}},"3f6ecf56618":{"name":"Forge","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fcdn-images-1.medium.com\u002Fmax\u002F596\u002F1*uULpIlImcO5TDuBZ6lm7Lg@2x.png","width":596,"height":183}}},"ae2a65f35510":{"name":"GEN","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fmiro.medium.com\u002Fmax\u002F264\u002F1*RdVZMdvfV3YiZTw6mX7yWA.png","width":264,"height":140}}},"88d9857e584e":{"name":"LEVEL","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fmiro.medium.com\u002Fmax\u002F540\u002F1*JqYMhNX6KNNb2UlqGqO2WQ.png","width":540,"height":108}}},"7b6769f2748b":{"name":"Marker","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fcdn-images-1.medium.com\u002Fmax\u002F383\u002F1*haCUs0wF6TgOOvfoY-jEoQ@2x.png","width":383,"height":92}}},"444d13b52878":{"name":"OneZero","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fmiro.medium.com\u002Fmax\u002F540\u002F1*cw32fIqCbRWzwJaoQw6BUg.png","width":540,"height":123}}},"8ccfed20cbb2":{"name":"Zora","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fmiro.medium.com\u002Fmax\u002F540\u002F1*tZUQqRcCCZDXjjiZ4bDvgQ.png","width":540,"height":106}}}},"embeddedPostIds":{"coronavirus":"cd3010f9d81f"},"sharedCdcMessaging":{"COVID_APPLICABLE_TAG_SLUGS":[],"COVID_APPLICABLE_TOPIC_NAMES":[],"COVID_APPLICABLE_TOPIC_NAMES_FOR_TOPIC_PAGE":[],"COVID_MESSAGES":{"tierA":{"text":"For more information on the novel coronavirus and Covid-19, visit cdc.gov.","markups":[{"start":66,"end":73,"href":"https:\u002F\u002Fwww.cdc.gov\u002Fcoronavirus\u002F2019-nCoV"}]},"tierB":{"text":"Anyone can publish on Medium per our Policies, but we don’t fact-check every story. For more info about the coronavirus, see cdc.gov.","markups":[{"start":37,"end":45,"href":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Fcategories\u002F201931128-Policies-Safety"},{"start":125,"end":132,"href":"https:\u002F\u002Fwww.cdc.gov\u002Fcoronavirus\u002F2019-nCoV"}]},"paywall":{"text":"This article has been made free for everyone, thanks to Medium Members. For more information on the novel coronavirus and Covid-19, visit cdc.gov.","markups":[{"start":56,"end":70,"href":"https:\u002F\u002Fmedium.com\u002Fmembership"},{"start":138,"end":145,"href":"https:\u002F\u002Fwww.cdc.gov\u002Fcoronavirus\u002F2019-nCoV"}]},"unbound":{"text":"This article is free for everyone, thanks to Medium Members. For more information on the novel coronavirus and Covid-19, visit cdc.gov.","markups":[{"start":45,"end":59,"href":"https:\u002F\u002Fmedium.com\u002Fmembership"},{"start":127,"end":134,"href":"https:\u002F\u002Fwww.cdc.gov\u002Fcoronavirus\u002F2019-nCoV"}]}},"COVID_BANNER_POST_ID_OVERRIDE_WHITELIST":["3b31a67bff4a"]},"sharedVoteMessaging":{"TAGS":["politics","election-2020","government","us-politics","election","2020-presidential-race","trump","donald-trump","democrats","republicans","congress","republican-party","democratic-party","biden","joe-biden","maga"],"TOPICS":["politics","election"],"MESSAGE":{"text":"Find out more about the U.S. election results here.","markups":[{"start":46,"end":50,"href":"https:\u002F\u002Fcookpolitical.com\u002F2020-national-popular-vote-tracker"}]},"EXCLUDE_POSTS":["397ef29e3ca5"]},"embedPostRules":[],"recircOptions":{"v1":{"limit":3},"v2":{"limit":8}},"braintreeClientKey":"production_zjkj96jm_m56f8fqpf7ngnrd4","paypalClientId":"AXj1G4fotC2GE8KzWX9mSxCH1wmPE3nJglf4Z2ig_amnhvlMVX87otaq58niAg9iuLktVNF_1WCMnN7v","stripePublishableKey":"pk_live_7FReX44VnNIInZwrIIx6ghjl","errorTracking":"none"},"debug":{"requestId":"4d62f9b1-f768-4ba8-9660-aa2b2c7cd1db","branchDeployConfig":null,"originalSpanCarrier":{"ot-tracer-spanid":"465fa6393bf3b0c3","ot-tracer-traceid":"27b7697d37480c9a","ot-tracer-sampled":"true"}},"multiVote":{"clapsPerPost":{}},"navigation":{"branch":{"show":null,"hasRendered":null,"blockedByCTA":false},"hideGoogleOneTap":false,"hasRenderedGoogleOneTap":null,"hasRenderedAlternateUserBanner":null,"currentLocation":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fhow-to-build-your-own-muzero-in-python-f77d5718061a","host":"medium.com","hostname":"medium.com","referrer":"https:\u002F\u002Fwww.google.com\u002F","susiModal":{"step":null,"operation":"register"},"postRead":false,"responsesOpen":false},"session":{"user":{"id":"lo_a9acde113469"},"xsrf":"","isSpoofed":false},"tracing":{}}</script><script>window.__APOLLO_STATE__ = {"ROOT_QUERY":{"__typename":"Query","viewer":null,"variantFlags":[{"__typename":"VariantFlag","name":"allow_access","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"allow_signup","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"allow_test_auth","valueType":{"__typename":"VariantFlagString","value":"disallow"}},{"__typename":"VariantFlag","name":"android_enable_lock_responses","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"assign_default_topic_to_posts","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"available_annual_plan","valueType":{"__typename":"VariantFlagString","value":"2c754bcc2995"}},{"__typename":"VariantFlag","name":"available_monthly_plan","valueType":{"__typename":"VariantFlagString","value":"60e220181034"}},{"__typename":"VariantFlag","name":"bane_add_user","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"bane_verify_domain","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"branch_seo_metadata","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"browsable_stream_config_bucket","valueType":{"__typename":"VariantFlagString","value":"curated-topics"}},{"__typename":"VariantFlag","name":"coronavirus_topic_recirc","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"covid_19_cdc_banner","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"default_seo_post_titles","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"disable_android_subscription_activity_carousel","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"disable_ios_resume_reading_toast","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"disable_ios_subscription_activity_carousel","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"disable_mobile_featured_chunk","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"disable_post_recommended_from_friends_provider","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_android_local_currency","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_annual_renewal_reminder_email","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_app_flirty_thirty","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_apple_parse_expires_at","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_apple_sign_in","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_apple_webhook","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_apple_webhook_renewal_failure","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_aurora_about_page_routing","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_aurora_general_admission","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_aurora_nav","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_aurora_profile_page","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_aurora_pub_follower_page","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_aurora_recirc","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_aurora_sticky_nav","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_aurora_tag_page_routing","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_author_autotier","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_author_cards","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_author_cards_byline","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_automated_mission_control_triggers","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_braintree_apple_pay","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_braintree_client","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_braintree_integration","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_braintree_paypal","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_braintree_trial_membership","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_braintree_webhook","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_branch_io","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_branch_text_me_the_app","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_branding","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_branding_fonts","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_cleansweep_double_writes","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_client_error_tracking","valueType":{"__typename":"VariantFlagString","value":"none"}},{"__typename":"VariantFlag","name":"enable_confirm_sign_in","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_cta_meter","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_curation_priority_queue_experiment","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_custom_domain_v2_settings","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_dedicated_series_tab_api_ios","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_digest_feature_logging","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_digest_tagline","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_earn_redirect","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_edit_alt_text","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_email_sign_in_captcha","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_embedding_based_diversification","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_end_of_post_cleanup","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_evhead_com_to_ev_medium_com_redirect","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_expanded_feature_chunk_pool","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_filter_by_resend_rules","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_filter_expire_processor","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_footer_app_buttons","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_fyf_authors_and_collections","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_fyf_rex_sourcing","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_global_susi_modal","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_google_one_tap","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_google_webhook","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_google_webhook_subscription_cancelled","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_highlander_member_digest","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_hightower_user_minimum_guarantee","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_homepage_who_to_follow_module","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_homepage_write_button","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_ios_post_stats","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_json_logs_trained_ranker","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_kbfd_rex","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_kbfd_rex_app_highlights","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_homepage","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_homepage_feed","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_notifications","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_pay_page","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_post","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_post_cd","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_pub_homepage_for_selected_domains","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_publish_to_profile","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_server_upstream_deadlines","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_stories","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_threaded_responses","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_topics","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lite_unread_notification_count_mutation","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_lock_responses","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_login_code_flow","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_marketing_emails","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_media_resource_try_catch","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_medium2_kbfd","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_membership_remove_section_a","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_miro_on_kubernetes","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_mission_control","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_ml_rank_modules","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_ml_rank_rex_anno","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_mute","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_new_checkout_flow","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_new_collaborative_filtering_data","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_new_login_flow","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_new_three_dot_menu","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_parsely","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_patronus_on_kubernetes","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_popularity_feature","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_post_import","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_post_page_nav_stickiness_removal","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_post_settings_screen","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_primary_topic_for_mobile","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_profile_design_reminder","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_profile_page_seo_titles","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_publish_to_email_for_publication_posts","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_receipt_notes","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_responses_2","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_responses_all","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_responses_edit_and_delete","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_responses_moderation","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_rex_follow_feed_cache","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_rito_upstream_deadlines","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_rtr_channel","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_s3_sites","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_save_to_medium","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_signup_friction","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_starspace","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_starspace_ranker_starspace","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_stripegate","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_tick_landing_page","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_tipalti_onboarding","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_trending_posts_diversification","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_tribute_landing_page","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_triton_predictions","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_trumpland_landing_page","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_twitter_auth_suggestions","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"enable_untruncated_author_post_as_email","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"glyph_font_set","valueType":{"__typename":"VariantFlagString","value":"m2-unbound"}},{"__typename":"VariantFlag","name":"google_sign_in_android","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"ios_enable_generic_home_modules","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"ios_enable_home_post_menu","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"ios_enable_iceland_paywall","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"ios_enable_lock_responses","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"ios_iceland_nux","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"ios_pub_follow_email_opt_in","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"is_not_medium_subscriber","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"kill_fastrak","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"kill_stripe_express","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"limit_post_referrers","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"limit_user_follows","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"make_nav_sticky","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"new_transition_page","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"provider_for_credit_card_form","valueType":{"__typename":"VariantFlagString","value":"BRAINTREE"}},{"__typename":"VariantFlag","name":"pub_sidebar","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"redefine_average_post_reading_time","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"remove_post_post_similarity","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"retrained_ranker","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"sign_up_with_email_button","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"signin_services","valueType":{"__typename":"VariantFlagString","value":"twitter,facebook,google,email,google-fastidv,google-one-tap,apple"}},{"__typename":"VariantFlag","name":"signup_services","valueType":{"__typename":"VariantFlagString","value":"twitter,facebook,google,email,google-fastidv,google-one-tap,apple"}},{"__typename":"VariantFlag","name":"skip_sign_in_recaptcha","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"suppress_apple_missing_expires_date_alert","valueType":{"__typename":"VariantFlagBoolean","value":true}},{"__typename":"VariantFlag","name":"use_new_admin_topic_backend","valueType":{"__typename":"VariantFlagBoolean","value":true}}],"meterPost({\"postId\":\"f77d5718061a\",\"postMeteringOptions\":{\"referrer\":\"https:\u002F\u002Fwww.google.com\u002F\"}})":{"__ref":"MeteringInfo:{}"},"postResult({\"id\":\"f77d5718061a\"})":{"__ref":"Post:f77d5718061a"}},"MeteringInfo:{}":{"__typename":"MeteringInfo","postIds":["f77d5718061a"],"maxUnlockCount":3,"unlocksRemaining":2},"ImageMetadata:":{"id":"","__typename":"ImageMetadata"},"CustomStyleSheet:f87c127d73df":{"id":"f87c127d73df","__typename":"CustomStyleSheet","global":{"__typename":"GlobalStyles","colorPalette":{"__typename":"StyleSheetColorPalette","primary":{"__typename":"ColorValue","colorPalette":{"__typename":"ColorPalette","highlightSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FFFFFFFF","colorPoints":[{"__typename":"ColorPoint","color":"#FFF4F2F5","point":0},{"__typename":"ColorPoint","color":"#FFF2F0F4","point":0.1},{"__typename":"ColorPoint","color":"#FFF0EEF2","point":0.2},{"__typename":"ColorPoint","color":"#FFEDECF1","point":0.3},{"__typename":"ColorPoint","color":"#FFEBEAF0","point":0.4},{"__typename":"ColorPoint","color":"#FFE9E8EF","point":0.5},{"__typename":"ColorPoint","color":"#FFE7E6EE","point":0.6},{"__typename":"ColorPoint","color":"#FFE5E5EC","point":0.7},{"__typename":"ColorPoint","color":"#FFE3E3EB","point":0.8},{"__typename":"ColorPoint","color":"#FFE1E1EA","point":0.9},{"__typename":"ColorPoint","color":"#FFDFDFE9","point":1}]},"defaultBackgroundSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FFFFFFFF","colorPoints":[{"__typename":"ColorPoint","color":"#FF82848F","point":0},{"__typename":"ColorPoint","color":"#FF797B85","point":0.1},{"__typename":"ColorPoint","color":"#FF70717B","point":0.2},{"__typename":"ColorPoint","color":"#FF676870","point":0.3},{"__typename":"ColorPoint","color":"#FF5D5E65","point":0.4},{"__typename":"ColorPoint","color":"#FF53545A","point":0.5},{"__typename":"ColorPoint","color":"#FF49494E","point":0.6},{"__typename":"ColorPoint","color":"#FF3E3E43","point":0.7},{"__typename":"ColorPoint","color":"#FF333336","point":0.8},{"__typename":"ColorPoint","color":"#FF272729","point":0.9},{"__typename":"ColorPoint","color":"#FF1A1A1B","point":1}]},"tintBackgroundSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FF1B1D26","colorPoints":[{"__typename":"ColorPoint","color":"#FF1B1D26","point":0},{"__typename":"ColorPoint","color":"#FF393B44","point":0.1},{"__typename":"ColorPoint","color":"#FF53555E","point":0.2},{"__typename":"ColorPoint","color":"#FF6C6D75","point":0.3},{"__typename":"ColorPoint","color":"#FF83848B","point":0.4},{"__typename":"ColorPoint","color":"#FF9999A0","point":0.5},{"__typename":"ColorPoint","color":"#FFAFAEB4","point":0.6},{"__typename":"ColorPoint","color":"#FFC3C2C8","point":0.7},{"__typename":"ColorPoint","color":"#FFD7D6DA","point":0.8},{"__typename":"ColorPoint","color":"#FFEBE9EC","point":0.9},{"__typename":"ColorPoint","color":"#FFFEFCFE","point":1}]}}},"background":null},"fonts":{"__typename":"StyleSheetFonts","font1":{"__typename":"StyleSheetFont","name":"SANS_SERIF_1"},"font2":{"__typename":"StyleSheetFont","name":"SANS_SERIF_1"},"font3":{"__typename":"StyleSheetFont","name":"SERIF_2"}}},"header":{"__typename":"HeaderStyles","backgroundColor":{"__typename":"ColorValue","colorPalette":{"__typename":"ColorPalette","tintBackgroundSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FF292a3a","colorPoints":[{"__typename":"ColorPoint","color":"#FF292A3A","point":0},{"__typename":"ColorPoint","color":"#FF444555","point":0.1},{"__typename":"ColorPoint","color":"#FF5D5E6C","point":0.2},{"__typename":"ColorPoint","color":"#FF747482","point":0.3},{"__typename":"ColorPoint","color":"#FF8A8A97","point":0.4},{"__typename":"ColorPoint","color":"#FF9F9FAA","point":0.5},{"__typename":"ColorPoint","color":"#FFB4B3BD","point":0.6},{"__typename":"ColorPoint","color":"#FFC8C6CF","point":0.7},{"__typename":"ColorPoint","color":"#FFDBD9E1","point":0.8},{"__typename":"ColorPoint","color":"#FFEEECF2","point":0.9},{"__typename":"ColorPoint","color":"#FFFFFEFF","point":1}]}},"rgb":"292a3a","alpha":"ff"},"postBackgroundColor":{"__typename":"ColorValue","colorPalette":{"__typename":"ColorPalette","tintBackgroundSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FF292a3a","colorPoints":[{"__typename":"ColorPoint","color":"#FF292A3A","point":0},{"__typename":"ColorPoint","color":"#FF444555","point":0.1},{"__typename":"ColorPoint","color":"#FF5D5E6C","point":0.2},{"__typename":"ColorPoint","color":"#FF747482","point":0.3},{"__typename":"ColorPoint","color":"#FF8A8A97","point":0.4},{"__typename":"ColorPoint","color":"#FF9F9FAA","point":0.5},{"__typename":"ColorPoint","color":"#FFB4B3BD","point":0.6},{"__typename":"ColorPoint","color":"#FFC8C6CF","point":0.7},{"__typename":"ColorPoint","color":"#FFDBD9E1","point":0.8},{"__typename":"ColorPoint","color":"#FFEEECF2","point":0.9},{"__typename":"ColorPoint","color":"#FFFFFEFF","point":1}]}},"rgb":"292a3a","alpha":"ff"},"backgroundImage":null,"headerScale":"HEADER_SCALE_MEDIUM","horizontalAlignment":"CENTER","backgroundImageDisplayMode":"IMAGE_DISPLAY_MODE_FILL","backgroundImageVerticalAlignment":"START","backgroundColorDisplayMode":"COLOR_DISPLAY_MODE_SOLID","secondaryBackgroundColor":null,"nameColor":null,"nameTreatment":"NAME_TREATMENT_LOGO","postNameTreatment":"NAME_TREATMENT_LOGO","logoImage":{"__ref":"ImageMetadata:1*2A6OcWvQBoboXH2AW0LQMw.png"},"logoScale":"HEADER_SCALE_LARGE","taglineColor":{"__typename":"ColorValue","rgb":"ffffff","alpha":"ff"},"taglineTreatment":"TAGLINE_TREATMENT_HEADER"},"navigation":{"__typename":"HeaderNavigation","navItems":[{"__typename":"HeaderNavigationItem","name":"Write For Us","href":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fwrite-for-applied-data-science-3615cca98e91","tagSlugs":[],"type":"NAV_TYPE_LINK"},{"__typename":"HeaderNavigationItem","name":"Website","href":"https:\u002F\u002Fadsp.ai","tagSlugs":[],"type":"NAV_TYPE_LINK"}]},"postBody":null},"ImageMetadata:1*2A6OcWvQBoboXH2AW0LQMw.png":{"id":"1*2A6OcWvQBoboXH2AW0LQMw.png","__typename":"ImageMetadata","originalWidth":1378,"originalHeight":410},"User:2d3c4b088dc6":{"id":"2d3c4b088dc6","__typename":"User","isFollowing":null,"viewerIsUser":false,"isSuspended":false,"name":"David Foster","hasCompletedProfile":false,"bio":"Author of the Generative Deep Learning book :: Founding Partner of Applied Data Science Partners","imageId":"1*QqMD1rth7ahcS7Gc415R9w.jpeg","username":"dtfoster","customStyleSheet":{"__ref":"CustomStyleSheet:ee479ebc1b9a"},"customDomainState":null,"isAuroraVisible":true,"createdAt":0,"mediumMemberAt":1569164263034,"lastPostCreatedAt":0,"socialStats":{"__typename":"SocialStats","followerCount":4856},"hasSubdomain":false,"isBlocking":null,"isMuting":null,"allowNotes":true,"newsletterV3":{"__ref":"NewsletterV3:6e906e22e5e7"},"twitterScreenName":"","isPartnerProgramEnrolled":false},"ImageMetadata:1*JxkX0uKc5as8yksGa20vow.png":{"id":"1*JxkX0uKc5as8yksGa20vow.png","__typename":"ImageMetadata"},"Collection:70cd67c5d0e":{"id":"70cd67c5d0e","__typename":"Collection","domain":null,"googleAnalyticsId":null,"slug":"applied-data-science","colorBehavior":"ACCENT_COLOR_AND_FILL_BACKGROUND","isAuroraVisible":true,"favicon":{"__ref":"ImageMetadata:"},"name":"Applied Data Science","colorPalette":{"__typename":"ColorPalette","highlightSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FFFFFFFF","colorPoints":[{"__typename":"ColorPoint","color":"#FFF4F2F5","point":0},{"__typename":"ColorPoint","color":"#FFF2F0F4","point":0.1},{"__typename":"ColorPoint","color":"#FFF0EEF2","point":0.2},{"__typename":"ColorPoint","color":"#FFEDECF1","point":0.3},{"__typename":"ColorPoint","color":"#FFEBEAF0","point":0.4},{"__typename":"ColorPoint","color":"#FFE9E8EF","point":0.5},{"__typename":"ColorPoint","color":"#FFE7E6EE","point":0.6},{"__typename":"ColorPoint","color":"#FFE5E5EC","point":0.7},{"__typename":"ColorPoint","color":"#FFE3E3EB","point":0.8},{"__typename":"ColorPoint","color":"#FFE1E1EA","point":0.9},{"__typename":"ColorPoint","color":"#FFDFDFE9","point":1}]},"defaultBackgroundSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FFFFFFFF","colorPoints":[{"__typename":"ColorPoint","color":"#FF82848F","point":0},{"__typename":"ColorPoint","color":"#FF797B85","point":0.1},{"__typename":"ColorPoint","color":"#FF70717B","point":0.2},{"__typename":"ColorPoint","color":"#FF676870","point":0.3},{"__typename":"ColorPoint","color":"#FF5D5E65","point":0.4},{"__typename":"ColorPoint","color":"#FF53545A","point":0.5},{"__typename":"ColorPoint","color":"#FF49494E","point":0.6},{"__typename":"ColorPoint","color":"#FF3E3E43","point":0.7},{"__typename":"ColorPoint","color":"#FF333336","point":0.8},{"__typename":"ColorPoint","color":"#FF272729","point":0.9},{"__typename":"ColorPoint","color":"#FF1A1A1B","point":1}]},"tintBackgroundSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FF1B1D26","colorPoints":[{"__typename":"ColorPoint","color":"#FF1B1D26","point":0},{"__typename":"ColorPoint","color":"#FF393B44","point":0.1},{"__typename":"ColorPoint","color":"#FF53555E","point":0.2},{"__typename":"ColorPoint","color":"#FF6C6D75","point":0.3},{"__typename":"ColorPoint","color":"#FF83848B","point":0.4},{"__typename":"ColorPoint","color":"#FF9999A0","point":0.5},{"__typename":"ColorPoint","color":"#FFAFAEB4","point":0.6},{"__typename":"ColorPoint","color":"#FFC3C2C8","point":0.7},{"__typename":"ColorPoint","color":"#FFD7D6DA","point":0.8},{"__typename":"ColorPoint","color":"#FFEBE9EC","point":0.9},{"__typename":"ColorPoint","color":"#FFFEFCFE","point":1}]}},"customStyleSheet":{"__ref":"CustomStyleSheet:f87c127d73df"},"tagline":"Cutting edge data science, machine learning and AI projects","isAuroraEligible":true,"viewerIsEditor":false,"logo":{"__ref":"ImageMetadata:1*2A6OcWvQBoboXH2AW0LQMw.png"},"navItems":[{"__typename":"NavItem","title":"The Full Stack Data Scientist","url":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fthe-full-stack-data-scientist\u002Fhome","type":"TOPIC_PAGE"},{"__typename":"NavItem","title":"Write For Us","url":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fwrite-for-applied-data-science-3615cca98e91","type":"POST_NAV_ITEM"},{"__typename":"NavItem","title":"MAIN SITE","url":"https:\u002F\u002Fadsp.ai","type":"EXTERNAL_LINK_NAV_ITEM"}],"creator":{"__ref":"User:2d3c4b088dc6"},"subscriberCount":7622,"avatar":{"__ref":"ImageMetadata:1*JxkX0uKc5as8yksGa20vow.png"},"isEnrolledInHightower":false,"newsletterV3":null,"viewerIsFollowing":false,"viewerIsSubscribedToLetters":false,"canToggleEmail":false,"isUserSubscribedToCollectionEmails":false,"viewerIsMuting":false,"viewerCanEditOwnPosts":false,"viewerCanEditPosts":false,"description":"Cutting edge data science, machine learning and AI projects","ampEnabled":false,"twitterUsername":"applied_data","facebookPageId":null},"ImageMetadata:1*2Rq-dOKy7zJ2lin_sQYyuw.png":{"id":"1*2Rq-dOKy7zJ2lin_sQYyuw.png","__typename":"ImageMetadata"},"CustomStyleSheet:ee479ebc1b9a":{"id":"ee479ebc1b9a","__typename":"CustomStyleSheet","global":{"__typename":"GlobalStyles","colorPalette":{"__typename":"StyleSheetColorPalette","primary":{"__typename":"ColorValue","colorPalette":{"__typename":"ColorPalette","tintBackgroundSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FF0bbfff","colorPoints":[{"__typename":"ColorPoint","color":"#FF0BBFFF","point":0},{"__typename":"ColorPoint","color":"#FF46C6FF","point":0.1},{"__typename":"ColorPoint","color":"#FF63CEFF","point":0.2},{"__typename":"ColorPoint","color":"#FF7BD5FF","point":0.3},{"__typename":"ColorPoint","color":"#FF8FDCFF","point":0.4},{"__typename":"ColorPoint","color":"#FFA1E3FF","point":0.5},{"__typename":"ColorPoint","color":"#FFB2E9FF","point":0.6},{"__typename":"ColorPoint","color":"#FFC3F0FF","point":0.7},{"__typename":"ColorPoint","color":"#FFD2F6FF","point":0.8},{"__typename":"ColorPoint","color":"#FFE1FDFF","point":0.9},{"__typename":"ColorPoint","color":"#FFF0FFFF","point":1}]},"highlightSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FFFFFFFF","colorPoints":[{"__typename":"ColorPoint","color":"#FFDFF8FF","point":0},{"__typename":"ColorPoint","color":"#FFD8F7FF","point":0.1},{"__typename":"ColorPoint","color":"#FFD2F6FF","point":0.2},{"__typename":"ColorPoint","color":"#FFCBF5FF","point":0.3},{"__typename":"ColorPoint","color":"#FFC4F4FF","point":0.4},{"__typename":"ColorPoint","color":"#FFBCF3FF","point":0.5},{"__typename":"ColorPoint","color":"#FFB5F3FF","point":0.6},{"__typename":"ColorPoint","color":"#FFACF2FF","point":0.7},{"__typename":"ColorPoint","color":"#FFA4F1FF","point":0.8},{"__typename":"ColorPoint","color":"#FF9AF0FF","point":0.9},{"__typename":"ColorPoint","color":"#FF91EFFF","point":1}]},"defaultBackgroundSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FFFFFFFF","colorPoints":[{"__typename":"ColorPoint","color":"#FF0096D4","point":0},{"__typename":"ColorPoint","color":"#FF008BC2","point":0.1},{"__typename":"ColorPoint","color":"#FF0080B1","point":0.2},{"__typename":"ColorPoint","color":"#FF00749F","point":0.3},{"__typename":"ColorPoint","color":"#FF02698E","point":0.4},{"__typename":"ColorPoint","color":"#FF115D7D","point":0.5},{"__typename":"ColorPoint","color":"#FF16516C","point":0.6},{"__typename":"ColorPoint","color":"#FF17455B","point":0.7},{"__typename":"ColorPoint","color":"#FF163849","point":0.8},{"__typename":"ColorPoint","color":"#FF122B38","point":0.9},{"__typename":"ColorPoint","color":"#FF0C1D25","point":1}]}}},"background":null}},"header":{"__typename":"HeaderStyles","backgroundColor":{"__typename":"ColorValue","colorPalette":{"__typename":"ColorPalette","tintBackgroundSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FFffffff","colorPoints":[{"__typename":"ColorPoint","color":"#FFFFFFFF","point":0},{"__typename":"ColorPoint","color":"#FFECECEC","point":0.1},{"__typename":"ColorPoint","color":"#FFD9D9D9","point":0.2},{"__typename":"ColorPoint","color":"#FFC5C6C6","point":0.3},{"__typename":"ColorPoint","color":"#FFB1B1B1","point":0.4},{"__typename":"ColorPoint","color":"#FF9C9D9D","point":0.5},{"__typename":"ColorPoint","color":"#FF868787","point":0.6},{"__typename":"ColorPoint","color":"#FF6F7071","point":0.7},{"__typename":"ColorPoint","color":"#FF575959","point":0.8},{"__typename":"ColorPoint","color":"#FF3D3F3F","point":0.9},{"__typename":"ColorPoint","color":"#FF202122","point":1}]}}},"postBackgroundColor":null,"backgroundImage":{"__ref":"ImageMetadata:1*2Rq-dOKy7zJ2lin_sQYyuw.png"}}},"Paragraph:43c983f0d936_0":{"id":"43c983f0d936_0","__typename":"Paragraph","name":"26cb","text":"","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__ref":"ImageMetadata:1*y-FnMby1uCDxw6Br-BeZng.png"},"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_1":{"id":"43c983f0d936_1","__typename":"Paragraph","name":"5321","text":"MuZero: The Walkthrough (Part 1\u002F3)","type":"H3","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_2":{"id":"43c983f0d936_2","__typename":"Paragraph","name":"0289","text":"Teaching a machine to play games using self-play and deep learning…without telling it the rules 🤯","type":"H4","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_3":{"id":"43c983f0d936_3","__typename":"Paragraph","name":"f918","text":"If you want to learn how one of the most sophisticated AI systems ever built works, you’ve come to the right place.","type":"BQ","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_4":{"id":"43c983f0d936_4","__typename":"Paragraph","name":"2bfb","text":"In this three part series, we’ll explore the inner workings of the DeepMind MuZero model — the younger (and even more impressive) brother of AlphaZero.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_5":{"id":"43c983f0d936_5","__typename":"Paragraph","name":"4340","text":"👉 Part 2","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":3,"end":9,"type":"A","href":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fhow-to-build-your-own-deepmind-muzero-in-python-part-2-3-f99dad7a7ad","anchorType":"LINK","userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_6":{"id":"43c983f0d936_6","__typename":"Paragraph","name":"1a04","text":"👉 Part 3","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":3,"end":9,"type":"A","href":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fhow-to-build-your-own-deepmind-muzero-in-python-part-3-3-ccea6b03538b","anchorType":"LINK","userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_7":{"id":"43c983f0d936_7","__typename":"Paragraph","name":"ab29","text":"Also check out my latest post, about how to train reinforcement learning agents for multi-player board games, using self-play!","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_8":{"id":"43c983f0d936_8","__typename":"Paragraph","name":"efd8","text":"👉 Self-Play in Multiplayer Environments","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":3,"end":40,"type":"A","href":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fhow-to-train-ai-agents-to-play-multiplayer-games-using-self-play-deep-reinforcement-learning-247d0b440717","anchorType":"LINK","userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_9":{"id":"43c983f0d936_9","__typename":"Paragraph","name":"4295","text":"We’ll be walking through the pseudocode that accompanies the MuZero paper — so grab yourself a cup of tea and a comfy chair and let’s begin.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":29,"end":39,"type":"A","href":"https:\u002F\u002Farxiv.org\u002Fsrc\u002F1911.08265v1\u002Fanc\u002Fpseudocode.py","anchorType":"LINK","userId":null,"linkMetadata":null},{"__typename":"Markup","start":68,"end":73,"type":"A","href":"https:\u002F\u002Farxiv.org\u002Fabs\u002F1911.08265","anchorType":"LINK","userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_10":{"id":"43c983f0d936_10","__typename":"Paragraph","name":"ef7e","text":"","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__ref":"ImageMetadata:1*62G0bnqJoqujSIO4hizzKw.png"},"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_11":{"id":"43c983f0d936_11","__typename":"Paragraph","name":"e82d","text":"The story so far…","type":"H3","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_12":{"id":"43c983f0d936_12","__typename":"Paragraph","name":"0bb5","text":"On 19th November 2019 DeepMind released their latest model-based reinforcement learning algorithm to the world — MuZero.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":113,"end":119,"type":"A","href":"https:\u002F\u002Farxiv.org\u002Fabs\u002F1911.08265","anchorType":"LINK","userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_13":{"id":"43c983f0d936_13","__typename":"Paragraph","name":"225e","text":"This is the fourth in a line of DeepMind reinforcement learning papers that have continually smashed through the barriers of possibility, starting with AlphaGo in 2016.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_14":{"id":"43c983f0d936_14","__typename":"Paragraph","name":"b5bc","text":"To read about the full history from AlphaGo through to AlphaZero — check out my previous blog 👇","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_15":{"id":"43c983f0d936_15","__typename":"Paragraph","name":"a17a","text":"How to build your own AlphaZero AI using Python and Keras\nTeach a machine to learn Connect4 strategy through self-play and deep learning.medium.com","type":"MIXTAPE_EMBED","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":{"__typename":"MixtapeMetadata","href":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fhow-to-build-your-own-alphazero-ai-using-python-and-keras-7f664945c188","thumbnailImageId":"1*PanQe3XaV_tpFLddTzpz3g.png"},"markups":[{"__typename":"Markup","start":0,"end":147,"type":"A","href":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fhow-to-build-your-own-alphazero-ai-using-python-and-keras-7f664945c188","anchorType":"LINK","userId":null,"linkMetadata":null},{"__typename":"Markup","start":0,"end":57,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":58,"end":137,"type":"EM","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_16":{"id":"43c983f0d936_16","__typename":"Paragraph","name":"f065","text":"AlphaZero was hailed as the general algorithm for getting good at something, quickly, without any prior knowledge of human expert strategy.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_17":{"id":"43c983f0d936_17","__typename":"Paragraph","name":"c320","text":"So…what now?","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_18":{"id":"43c983f0d936_18","__typename":"Paragraph","name":"d951","text":"","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__ref":"ImageMetadata:1*VVGmi0SSqHVJPtpuOw-aVw.png"},"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_19":{"id":"43c983f0d936_19","__typename":"Paragraph","name":"9fe1","text":"MuZero","type":"H3","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_20":{"id":"43c983f0d936_20","__typename":"Paragraph","name":"5249","text":"Mastering Atari, Go, Chess and Shogi by Planning with a Learned Model","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":0,"end":69,"type":"A","href":"https:\u002F\u002Farxiv.org\u002Fabs\u002F1911.08265","anchorType":"LINK","userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_21":{"id":"43c983f0d936_21","__typename":"Paragraph","name":"63d2","text":"MuZero takes the ultimate next step. Not only does MuZero deny itself human strategy to learn from. It isn’t even shown the rules of the game.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_22":{"id":"43c983f0d936_22","__typename":"Paragraph","name":"766f","text":"In other words, for chess, AlphaZero is set the following challenge:","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_23":{"id":"43c983f0d936_23","__typename":"Paragraph","name":"80a1","text":"Learn how to play this game on your own — here’s the rulebook that explains how each piece moves and which moves are legal. Also it tells you how to tell if a position is checkmate (or a draw).","type":"BQ","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_24":{"id":"43c983f0d936_24","__typename":"Paragraph","name":"4f85","text":"MuZero on the other hand, is set this challenge:","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_25":{"id":"43c983f0d936_25","__typename":"Paragraph","name":"e932","text":"Learn how to play this game on your own — I’ll tell you what moves are legal in the current position and when one side has won (or it’s a draw), but I won’t tell you the overall rules of the game.","type":"BQ","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_26":{"id":"43c983f0d936_26","__typename":"Paragraph","name":"3b5e","text":"Alongside developing winning strategies, MuZero must therefore also develop its own dynamic model of the environment so that it can understand the implications of its choices and plan ahead.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_27":{"id":"43c983f0d936_27","__typename":"Paragraph","name":"bf72","text":"Imagine trying to become better than the world champion at a game where you are never told the rules. MuZero achieves precisely this.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_28":{"id":"43c983f0d936_28","__typename":"Paragraph","name":"f3b8","text":"In the next section we will explore how MuZero achieves this amazing feat, by walking through the codebase in detail.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_29":{"id":"43c983f0d936_29","__typename":"Paragraph","name":"a6e8","text":"","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__ref":"ImageMetadata:1*zZvaEuY5-OhKB2-GYzuUdw.png"},"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_30":{"id":"43c983f0d936_30","__typename":"Paragraph","name":"b162","text":"The MuZero pseudocode","type":"H3","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_31":{"id":"43c983f0d936_31","__typename":"Paragraph","name":"8a62","text":"Alongside the MuZero preprint paper, DeepMind have released Python pseudocode detailing the interactions between each part of the algorithm.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":30,"end":35,"type":"A","href":"https:\u002F\u002Farxiv.org\u002Fabs\u002F1911.08265","anchorType":"LINK","userId":null,"linkMetadata":null},{"__typename":"Markup","start":67,"end":77,"type":"A","href":"https:\u002F\u002Farxiv.org\u002Fsrc\u002F1911.08265v1\u002Fanc\u002Fpseudocode.py","anchorType":"LINK","userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_32":{"id":"43c983f0d936_32","__typename":"Paragraph","name":"fd51","text":"In this section, we’ll pick apart each function and class in a logical order, and I’ll explain what each part is doing and why. We’ll assume MuZero is learning to play chess, but the process is the same for any game, just with different parameters. All code is from the open-sourced DeepMind pseudocode.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":292,"end":302,"type":"A","href":"https:\u002F\u002Farxiv.org\u002Fsrc\u002F1911.08265v1\u002Fanc\u002Fpseudocode.py","anchorType":"LINK","userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_33":{"id":"43c983f0d936_33","__typename":"Paragraph","name":"7ec3","text":"Let’s start with an overview of the entire process, starting with the entrypoint function, muzero.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":91,"end":97,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_34":{"id":"43c983f0d936_34","__typename":"Paragraph","name":"cbcd","text":"Overview of the MuZero self-play and training process","type":"IMG","href":null,"layout":"OUTSET_CENTER","metadata":{"__ref":"ImageMetadata:1*ajFyjeF-1hVbmtlAsSoT2Q.png"},"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_35":{"id":"43c983f0d936_35","__typename":"Paragraph","name":"36f1","text":"","type":"IFRAME","href":null,"layout":"INSET_CENTER","metadata":null,"hasDropCap":null,"iframe":{"__typename":"Iframe","mediaResource":{"__ref":"MediaResource:37989df7e0c024c98a85ab9154886677"}},"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_36":{"id":"43c983f0d936_36","__typename":"Paragraph","name":"05af","text":"The entrypoint function muzero is passed a MuZeroConfig object, which stores important information about the parameterisation of the run, such as the action_space_size (number of possible actions) and num_actors (the number of parallel game simulations to spin up). We’ll go through these parameters in more detail as we encounter them in other functions.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":24,"end":30,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":43,"end":55,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":150,"end":167,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":201,"end":211,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_37":{"id":"43c983f0d936_37","__typename":"Paragraph","name":"1e7b","text":"At a high level, there are two independent parts to the MuZero algorithm — self-play (creating game data) and training (producing improved versions of the neural network). TheSharedStorage and ReplayBuffer objects can be accessed by both halves of the algorithm and store neural network versions and game data respectively.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":175,"end":188,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":193,"end":205,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":75,"end":84,"type":"EM","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":110,"end":118,"type":"EM","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_38":{"id":"43c983f0d936_38","__typename":"Paragraph","name":"b4d4","text":"","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__ref":"ImageMetadata:1*sWVAnhQvj2EmapE1UqIZDg.png"},"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_39":{"id":"43c983f0d936_39","__typename":"Paragraph","name":"8e1a","text":"Shared Storage and the Replay Buffer","type":"H4","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_40":{"id":"43c983f0d936_40","__typename":"Paragraph","name":"4e18","text":"The SharedStorage object contains methods for saving a version of the neural network and retrieving the latest neural network from the store.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":4,"end":17,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_41":{"id":"43c983f0d936_41","__typename":"Paragraph","name":"5f8b","text":"","type":"IFRAME","href":null,"layout":"INSET_CENTER","metadata":null,"hasDropCap":null,"iframe":{"__typename":"Iframe","mediaResource":{"__ref":"MediaResource:57eaa5c07fa0d54806a3c01079a6365c"}},"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_42":{"id":"43c983f0d936_42","__typename":"Paragraph","name":"bedb","text":"We also need a ReplayBuffer to store data from previous games. This takes the following form:","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":15,"end":27,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_43":{"id":"43c983f0d936_43","__typename":"Paragraph","name":"0b79","text":"","type":"IFRAME","href":null,"layout":"INSET_CENTER","metadata":null,"hasDropCap":null,"iframe":{"__typename":"Iframe","mediaResource":{"__ref":"MediaResource:83100f618299e154eaa27d3b5cfef9e4"}},"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_44":{"id":"43c983f0d936_44","__typename":"Paragraph","name":"44ce","text":"Notice how the window_size parameter limits the maximum number of games stored in the buffer. In MuZero, this is set to the latest 1,000,000 games.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":15,"end":26,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_45":{"id":"43c983f0d936_45","__typename":"Paragraph","name":"b57b","text":"","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__ref":"ImageMetadata:1*WNkAYNV92eJNZqxc9EfcBw.png"},"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_46":{"id":"43c983f0d936_46","__typename":"Paragraph","name":"b3ce","text":"Self-play (run_selfplay)","type":"H4","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_47":{"id":"43c983f0d936_47","__typename":"Paragraph","name":"05b3","text":"After creating the shared storage and replay buffer, MuZero launches num_actors parallel game environments, that run independently. For chess, num_actors is set to 3000. Each is running a function run_selfplay that grabs the latest version of the network from the store, plays a game with it (play_game) and saves the game data to the shared buffer.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":69,"end":79,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":143,"end":153,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":197,"end":209,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":293,"end":302,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_48":{"id":"43c983f0d936_48","__typename":"Paragraph","name":"d8e6","text":"","type":"IFRAME","href":null,"layout":"INSET_CENTER","metadata":null,"hasDropCap":null,"iframe":{"__typename":"Iframe","mediaResource":{"__ref":"MediaResource:77780ee132a794f4bc8d6470d61b3896"}},"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_49":{"id":"43c983f0d936_49","__typename":"Paragraph","name":"b6fe","text":"So in summary, MuZero is playing thousands of games against itself, saving these to a buffer and then training itself on data from those games. So far, this is no different to AlphaZero.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_50":{"id":"43c983f0d936_50","__typename":"Paragraph","name":"5970","text":"To end Part 1, we will cover one of the key differences between AlphaZero and MuZero — why does MuZero have three neural networks, whereas AlphaZero only has one?","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_51":{"id":"43c983f0d936_51","__typename":"Paragraph","name":"170e","text":"","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__ref":"ImageMetadata:1*XgPxsnGohRYsJZftwNNUJQ.png"},"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_52":{"id":"43c983f0d936_52","__typename":"Paragraph","name":"0aad","text":"The 3 Neural Networks of MuZero","type":"H4","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":0,"end":31,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_53":{"id":"43c983f0d936_53","__typename":"Paragraph","name":"859c","text":"Both AlphaZero and MuZero utilise a technique known as Monte Carlo Tree Search (MCTS) to select the next best move.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":55,"end":85,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_54":{"id":"43c983f0d936_54","__typename":"Paragraph","name":"3224","text":"The idea is that in order to select the next best move, it makes sense to ‘play out’ likely future scenarios from the current position, evaluate their value using a neural network and choose the action that maximises the future expected value. This seems to be what we humans are doing in our head when playing chess, and the AI is also designed to make use of this technique.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_55":{"id":"43c983f0d936_55","__typename":"Paragraph","name":"84ee","text":"However, MuZero has a problem. As it doesn’t know the rules of the game, it has no idea how a given action will affect the game state, so it cannot imagine future scenarios in the MCTS. It doesn’t even know how to work out what moves are legal from a given position, or whether one side has won.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_56":{"id":"43c983f0d936_56","__typename":"Paragraph","name":"ac7b","text":"The stunning development in the MuZero paper is to show that this doesn’t matter. MuZero learns how to play the game by creating a dynamic model of the environment within its own imagination and optimising within this model.","type":"BQ","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_57":{"id":"43c983f0d936_57","__typename":"Paragraph","name":"5154","text":"The diagram below shows a comparison between the MCTS processes in AlphaZero and MuZero:","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_58":{"id":"43c983f0d936_58","__typename":"Paragraph","name":"e7ed","text":"","type":"IMG","href":null,"layout":"OUTSET_CENTER","metadata":{"__ref":"ImageMetadata:1*NowOwxV5SQ9aLKbjdz41lQ.png"},"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_59":{"id":"43c983f0d936_59","__typename":"Paragraph","name":"e0a3","text":"Whereas AlphaZero only has only one neural network (prediction), MuZero needs three (prediction, dynamics, representation)","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":52,"end":62,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":85,"end":95,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":97,"end":105,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":107,"end":121,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_60":{"id":"43c983f0d936_60","__typename":"Paragraph","name":"f763","text":"The job of the AlphaZero prediction neural network f is to predict the policy p and value v of a given game state. The policy is a probability distribution over all moves and the value is just a single number that estimates the future rewards. This prediction is made every time the MCTS hits an unexplored leaf node, so that it can immediately assign an estimated value to the new position and also assign a probability to each subsequent action. The values are backfilled up the tree, back to the root node, so that after many simulations, the root node has a good idea of the future value of the current state, having explored lots of different possible futures.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":51,"end":52,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":78,"end":79,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":90,"end":91,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":25,"end":35,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_61":{"id":"43c983f0d936_61","__typename":"Paragraph","name":"36ac","text":"MuZero also has a prediction neural network f, but now the ‘game state’ that it operates on is a hidden representation that MuZero learns how to evolve through a dynamics neural network g. The dynamics network takes the current hidden state s and chosen action a and outputs a reward r and new state. Notice how in AlphaZero, moving between states in the MCTS tree is simply a case of asking the environment. MuZero doesn’t have this luxury, so needs to build its own dynamic model!","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":44,"end":45,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":186,"end":187,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":241,"end":242,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":261,"end":262,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":284,"end":285,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":18,"end":28,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":162,"end":170,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_62":{"id":"43c983f0d936_62","__typename":"Paragraph","name":"0942","text":"Lastly, in order to map from the current observed game state to the initial representation, MuZero uses a third representation neural network, h.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":143,"end":144,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":112,"end":126,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_63":{"id":"43c983f0d936_63","__typename":"Paragraph","name":"7939","text":"There are therefore two inference functions MuZero needs, in order to move through the MCTS tree making predictions:","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_64":{"id":"43c983f0d936_64","__typename":"Paragraph","name":"aad9","text":"initial_inference for the current state. h followed by f (representation followed by prediction) .","type":"ULI","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":0,"end":17,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":41,"end":42,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":55,"end":56,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_65":{"id":"43c983f0d936_65","__typename":"Paragraph","name":"7f8d","text":"recurrent_inference for moving between states inside the MCTS tree.g followed by f (representation followed by dynamics).","type":"ULI","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":0,"end":19,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":67,"end":68,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":81,"end":82,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_66":{"id":"43c983f0d936_66","__typename":"Paragraph","name":"dd55","text":"The two types of inference in MuZero","type":"IMG","href":null,"layout":"OUTSET_CENTER","metadata":{"__ref":"ImageMetadata:1*GA72IpY7ZciGshmVvtl8kQ.png"},"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_67":{"id":"43c983f0d936_67","__typename":"Paragraph","name":"101a","text":"The exact models aren’t provided in the pseudocode, but detailed descriptions are given in the accompanying paper.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_68":{"id":"43c983f0d936_68","__typename":"Paragraph","name":"d86a","text":"","type":"IFRAME","href":null,"layout":"INSET_CENTER","metadata":null,"hasDropCap":null,"iframe":{"__typename":"Iframe","mediaResource":{"__ref":"MediaResource:73b49aa3a96a32f405650a287b0ae512"}},"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_69":{"id":"43c983f0d936_69","__typename":"Paragraph","name":"4df5","text":"In summary, in the absence of the actual rules of chess, MuZero creates a new game inside its mind that it can control and uses this to plan into the future. The three networks (prediction, dynamics and representation) are optimised together so that strategies that perform well inside the imagined environment, also perform well in the real environment.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":178,"end":188,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":190,"end":198,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":203,"end":217,"type":"STRONG","href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_70":{"id":"43c983f0d936_70","__typename":"Paragraph","name":"376f","text":"Amazing stuff.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_71":{"id":"43c983f0d936_71","__typename":"Paragraph","name":"ae0f","text":"This is the end of Part 1 — in Part 2, we’ll start by walking through the play_game function and see how MuZero makes a decision about the next best move at each turn.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":74,"end":83,"type":"CODE","href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","start":31,"end":37,"type":"A","href":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fhow-to-build-your-own-deepmind-muzero-in-python-part-2-3-f99dad7a7ad","anchorType":"LINK","userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_72":{"id":"43c983f0d936_72","__typename":"Paragraph","name":"f1a4","text":"Please clap if you’ve enjoyed this post and I’ll see you in Part 2!","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":60,"end":66,"type":"A","href":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fhow-to-build-your-own-deepmind-muzero-in-python-part-2-3-f99dad7a7ad","anchorType":"LINK","userId":null,"linkMetadata":null}],"dropCapImage":null},"Paragraph:43c983f0d936_73":{"id":"43c983f0d936_73","__typename":"Paragraph","name":"bfea","text":"","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__ref":"ImageMetadata:0*x1sJqcMlPO9p-R-0.png"},"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[],"dropCapImage":null},"Paragraph:43c983f0d936_74":{"id":"43c983f0d936_74","__typename":"Paragraph","name":"0737","text":"This is the blog of Applied Data Science Partners, a consultancy that develops innovative data science solutions for businesses. To learn more, feel free to get in touch through our website.","type":"P","href":null,"layout":null,"metadata":null,"hasDropCap":null,"iframe":null,"mixtapeMetadata":null,"markups":[{"__typename":"Markup","start":182,"end":189,"type":"A","href":"https:\u002F\u002Fadsp.ai\u002F","anchorType":"LINK","userId":null,"linkMetadata":null}],"dropCapImage":null},"ImageMetadata:1*y-FnMby1uCDxw6Br-BeZng.png":{"id":"1*y-FnMby1uCDxw6Br-BeZng.png","__typename":"ImageMetadata","originalHeight":659,"originalWidth":1211,"focusPercentX":null,"focusPercentY":null,"alt":null},"ImageMetadata:1*62G0bnqJoqujSIO4hizzKw.png":{"id":"1*62G0bnqJoqujSIO4hizzKw.png","__typename":"ImageMetadata","originalHeight":75,"originalWidth":75,"focusPercentX":null,"focusPercentY":null,"alt":null},"ImageMetadata:1*VVGmi0SSqHVJPtpuOw-aVw.png":{"id":"1*VVGmi0SSqHVJPtpuOw-aVw.png","__typename":"ImageMetadata","originalHeight":75,"originalWidth":75,"focusPercentX":null,"focusPercentY":null,"alt":null},"ImageMetadata:1*zZvaEuY5-OhKB2-GYzuUdw.png":{"id":"1*zZvaEuY5-OhKB2-GYzuUdw.png","__typename":"ImageMetadata","originalHeight":75,"originalWidth":75,"focusPercentX":null,"focusPercentY":null,"alt":null},"ImageMetadata:1*ajFyjeF-1hVbmtlAsSoT2Q.png":{"id":"1*ajFyjeF-1hVbmtlAsSoT2Q.png","__typename":"ImageMetadata","originalHeight":659,"originalWidth":1211,"focusPercentX":null,"focusPercentY":null,"alt":null},"MediaResource:37989df7e0c024c98a85ab9154886677":{"id":"37989df7e0c024c98a85ab9154886677","__typename":"MediaResource","iframeSrc":"","iframeHeight":0,"iframeWidth":0,"title":"MuZero pseudocode (https:\u002F\u002Farxiv.org\u002Fsrc\u002F1911.08265v1\u002Fanc\u002Fpseudocode.py)"},"ImageMetadata:1*sWVAnhQvj2EmapE1UqIZDg.png":{"id":"1*sWVAnhQvj2EmapE1UqIZDg.png","__typename":"ImageMetadata","originalHeight":75,"originalWidth":75,"focusPercentX":null,"focusPercentY":null,"alt":null},"MediaResource:57eaa5c07fa0d54806a3c01079a6365c":{"id":"57eaa5c07fa0d54806a3c01079a6365c","__typename":"MediaResource","iframeSrc":"","iframeHeight":0,"iframeWidth":0,"title":"SharedStorage (https:\u002F\u002Farxiv.org\u002Fsrc\u002F1911.08265v1\u002Fanc\u002Fpseudocode.py)"},"MediaResource:83100f618299e154eaa27d3b5cfef9e4":{"id":"83100f618299e154eaa27d3b5cfef9e4","__typename":"MediaResource","iframeSrc":"","iframeHeight":0,"iframeWidth":0,"title":"ReplayBuffer (https:\u002F\u002Farxiv.org\u002Fsrc\u002F1911.08265v1\u002Fanc\u002Fpseudocode.py)"},"ImageMetadata:1*WNkAYNV92eJNZqxc9EfcBw.png":{"id":"1*WNkAYNV92eJNZqxc9EfcBw.png","__typename":"ImageMetadata","originalHeight":75,"originalWidth":75,"focusPercentX":null,"focusPercentY":null,"alt":null},"MediaResource:77780ee132a794f4bc8d6470d61b3896":{"id":"77780ee132a794f4bc8d6470d61b3896","__typename":"MediaResource","iframeSrc":"","iframeHeight":0,"iframeWidth":0,"title":"run_selfplay (https:\u002F\u002Farxiv.org\u002Fsrc\u002F1911.08265v1\u002Fanc\u002Fpseudocode.py)"},"ImageMetadata:1*XgPxsnGohRYsJZftwNNUJQ.png":{"id":"1*XgPxsnGohRYsJZftwNNUJQ.png","__typename":"ImageMetadata","originalHeight":75,"originalWidth":75,"focusPercentX":null,"focusPercentY":null,"alt":null},"ImageMetadata:1*NowOwxV5SQ9aLKbjdz41lQ.png":{"id":"1*NowOwxV5SQ9aLKbjdz41lQ.png","__typename":"ImageMetadata","originalHeight":703,"originalWidth":1212,"focusPercentX":null,"focusPercentY":null,"alt":null},"ImageMetadata:1*GA72IpY7ZciGshmVvtl8kQ.png":{"id":"1*GA72IpY7ZciGshmVvtl8kQ.png","__typename":"ImageMetadata","originalHeight":555,"originalWidth":1211,"focusPercentX":null,"focusPercentY":null,"alt":null},"MediaResource:73b49aa3a96a32f405650a287b0ae512":{"id":"73b49aa3a96a32f405650a287b0ae512","__typename":"MediaResource","iframeSrc":"","iframeHeight":0,"iframeWidth":0,"title":"Network (https:\u002F\u002Farxiv.org\u002Fsrc\u002F1911.08265v1\u002Fanc\u002Fpseudocode.py)"},"ImageMetadata:0*x1sJqcMlPO9p-R-0.png":{"id":"0*x1sJqcMlPO9p-R-0.png","__typename":"ImageMetadata","originalHeight":194,"originalWidth":700,"focusPercentX":null,"focusPercentY":null,"alt":null},"NewsletterV3:6e906e22e5e7":{"id":"6e906e22e5e7","__typename":"NewsletterV3","slug":"2d3c4b088dc6","name":"2d3c4b088dc6","description":"","isSubscribed":false,"type":"NEWSLETTER_TYPE_AUTHOR","user":{"__ref":"User:2d3c4b088dc6"},"collection":null,"subscribersCount":2},"Tag:machine-learning":{"id":"machine-learning","__typename":"Tag","displayTitle":"Machine Learning"},"Tag:deep-learning":{"id":"deep-learning","__typename":"Tag","displayTitle":"Deep Learning"},"Tag:data-science":{"id":"data-science","__typename":"Tag","displayTitle":"Data Science"},"Tag:artificial-intelligence":{"id":"artificial-intelligence","__typename":"Tag","displayTitle":"Artificial Intelligence"},"Tag:ai":{"id":"ai","__typename":"Tag","displayTitle":"AI"},"ImageMetadata:1*Uw_5AXGiULKKHqU6mEqYqg.jpeg":{"id":"1*Uw_5AXGiULKKHqU6mEqYqg.jpeg","__typename":"ImageMetadata","focusPercentX":null,"focusPercentY":null},"User:98ae8c5c242e":{"id":"98ae8c5c242e","__typename":"User","name":"Eleni Nisioti","username":"elennisioti","bio":"PhD student in AI. Deep learning is not just for machines. I like my coffee like I like my code. Without bugs.","isFollowing":null,"imageId":"2*aauJgt7VhVttnGsxpH14xg.jpeg","mediumMemberAt":0,"customDomainState":null,"hasSubdomain":false},"Post:70ffe0a429a2":{"id":"70ffe0a429a2","__typename":"Post","title":"Of Emotional Machines and Irrational Men","mediumUrl":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fof-emotional-machines-and-irrational-men-70ffe0a429a2","previewImage":{"__ref":"ImageMetadata:1*Uw_5AXGiULKKHqU6mEqYqg.jpeg"},"isPublished":true,"firstPublishedAt":1613669433562,"readingTime":8.222641509433963,"statusForCollection":"APPROVED","isLocked":false,"isShortform":false,"visibility":"PUBLIC","collection":{"__ref":"Collection:70cd67c5d0e"},"creator":{"__ref":"User:98ae8c5c242e"},"previewContent":{"__typename":"PreviewContent","isFullContent":false}},"ImageMetadata:1*y93HRfJgLjZno0g7fb4wgw.png":{"id":"1*y93HRfJgLjZno0g7fb4wgw.png","__typename":"ImageMetadata","focusPercentX":null,"focusPercentY":null},"Post:247d0b440717":{"id":"247d0b440717","__typename":"Post","title":"How To Build Your Own AI To Play Any Board Game","mediumUrl":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fhow-to-train-ai-agents-to-play-multiplayer-games-using-self-play-deep-reinforcement-learning-247d0b440717","previewImage":{"__ref":"ImageMetadata:1*y93HRfJgLjZno0g7fb4wgw.png"},"isPublished":true,"firstPublishedAt":1611537846008,"readingTime":9.419811320754716,"statusForCollection":"APPROVED","isLocked":false,"isShortform":false,"visibility":"PUBLIC","collection":{"__ref":"Collection:70cd67c5d0e"},"creator":{"__ref":"User:2d3c4b088dc6"},"previewContent":{"__typename":"PreviewContent","isFullContent":false}},"ImageMetadata:1*UMh-U2rra06VIhkVcBi8HQ.png":{"id":"1*UMh-U2rra06VIhkVcBi8HQ.png","__typename":"ImageMetadata","focusPercentX":null,"focusPercentY":null},"Post:2dcc32fe2c02":{"id":"2dcc32fe2c02","__typename":"Post","title":"We Analysed the 527,350 Facebook Ads of the US Presidential Candidates. Here Are The Results.","mediumUrl":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002F56-070-165-facebook-ad-spend-of-us-presidential-candidates-broken-down-by-age-and-gender-2dcc32fe2c02","previewImage":{"__ref":"ImageMetadata:1*UMh-U2rra06VIhkVcBi8HQ.png"},"isPublished":true,"firstPublishedAt":1565442085086,"readingTime":5.011320754716981,"statusForCollection":"APPROVED","isLocked":false,"isShortform":false,"visibility":"PUBLIC","collection":{"__ref":"Collection:70cd67c5d0e"},"creator":{"__ref":"User:2d3c4b088dc6"},"previewContent":{"__typename":"PreviewContent","isFullContent":false}},"ImageMetadata:1*WGic6ivmGu1dfWtmHC865w.jpeg":{"id":"1*WGic6ivmGu1dfWtmHC865w.jpeg","__typename":"ImageMetadata","focusPercentX":null,"focusPercentY":null},"User:3d8a2ff208c3":{"id":"3d8a2ff208c3","__typename":"User","name":"James Montantes","username":"james.montantes.exxact","bio":"Interested in HMI, AI, and decentralized systems and applications. I like to tinker with GPU systems for deep learning. Currently at Exxact Corporation.","isFollowing":null,"imageId":"1*T0iXGuhJgqm9zHWlRnXdgg.jpeg","mediumMemberAt":0,"customDomainState":null,"hasSubdomain":false},"Post:d95e1d4fe558":{"id":"d95e1d4fe558","__typename":"Post","title":"What Can You Do with the OpenAI GPT-3 Language Model?","mediumUrl":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fwhat-can-you-do-with-the-openai-gpt-3-language-model-d95e1d4fe558","previewImage":{"__ref":"ImageMetadata:1*WGic6ivmGu1dfWtmHC865w.jpeg"},"isPublished":true,"firstPublishedAt":1597154857162,"readingTime":12.68993710691824,"statusForCollection":"APPROVED","isLocked":true,"isShortform":false,"visibility":"LOCKED","collection":{"__ref":"Collection:70cd67c5d0e"},"creator":{"__ref":"User:3d8a2ff208c3"},"previewContent":{"__typename":"PreviewContent","isFullContent":false}},"ImageMetadata:1*3Zc12UKeiAI2--bi9F6xXw.png":{"id":"1*3Zc12UKeiAI2--bi9F6xXw.png","__typename":"ImageMetadata","focusPercentX":null,"focusPercentY":null},"Post:93e03bfdc151":{"id":"93e03bfdc151","__typename":"Post","title":"How To Put Coronavirus Into Context","mediumUrl":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fcoronavirus-in-context-93e03bfdc151","previewImage":{"__ref":"ImageMetadata:1*3Zc12UKeiAI2--bi9F6xXw.png"},"isPublished":true,"firstPublishedAt":1590714488934,"readingTime":10.667924528301887,"statusForCollection":"APPROVED","isLocked":true,"isShortform":false,"visibility":"LOCKED","collection":{"__ref":"Collection:70cd67c5d0e"},"creator":{"__ref":"User:2d3c4b088dc6"},"previewContent":{"__typename":"PreviewContent","isFullContent":false}},"ImageMetadata:1*aQaPAEsi6kuRWScVPOsS7Q.jpeg":{"id":"1*aQaPAEsi6kuRWScVPOsS7Q.jpeg","__typename":"ImageMetadata","focusPercentX":null,"focusPercentY":null},"User:ae8b4923c38d":{"id":"ae8b4923c38d","__typename":"User","name":"Daniel Sharp","username":"dsharpc","bio":"","isFollowing":null,"imageId":"2*UZKdOeULb1wNmn9ud2hE_A.png","mediumMemberAt":1575646765000,"customDomainState":null,"hasSubdomain":false},"Post:919e32e88d18":{"id":"919e32e88d18","__typename":"Post","title":"Full Stack Data Scientist 5: Automating report generation with Jupyter Notebooks","mediumUrl":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Ffull-stack-data-scientist-5-automating-report-generation-with-jupyter-notebooks-919e32e88d18","previewImage":{"__ref":"ImageMetadata:1*aQaPAEsi6kuRWScVPOsS7Q.jpeg"},"isPublished":true,"firstPublishedAt":1600682512119,"readingTime":2.422012578616352,"statusForCollection":"APPROVED","isLocked":false,"isShortform":false,"visibility":"PUBLIC","collection":{"__ref":"Collection:70cd67c5d0e"},"creator":{"__ref":"User:ae8b4923c38d"},"previewContent":{"__typename":"PreviewContent","isFullContent":false}},"ImageMetadata:1*wJcVGzmj7r8ZM6XVFfdtLA.jpeg":{"id":"1*wJcVGzmj7r8ZM6XVFfdtLA.jpeg","__typename":"ImageMetadata","focusPercentX":null,"focusPercentY":null},"Post:ddebd92ae26c":{"id":"ddebd92ae26c","__typename":"Post","title":"Survival Αnalysis","mediumUrl":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fsurvival-%CE%B1nalysis-ddebd92ae26c","previewImage":{"__ref":"ImageMetadata:1*wJcVGzmj7r8ZM6XVFfdtLA.jpeg"},"isPublished":true,"firstPublishedAt":1580827063925,"readingTime":6.455974842767295,"statusForCollection":"APPROVED","isLocked":false,"isShortform":false,"visibility":"PUBLIC","collection":{"__ref":"Collection:70cd67c5d0e"},"creator":{"__ref":"User:98ae8c5c242e"},"previewContent":{"__typename":"PreviewContent","isFullContent":false}},"ImageMetadata:1*vrj_U6tBMYzLADoE37jQ4g.jpeg":{"id":"1*vrj_U6tBMYzLADoE37jQ4g.jpeg","__typename":"ImageMetadata","focusPercentX":null,"focusPercentY":null},"Post:482271dfb8fd":{"id":"482271dfb8fd","__typename":"Post","title":"The Pandepreneurs: the rise of new companies through lockdown and market adaptability","mediumUrl":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fthe-pandepreneurs-the-rise-of-new-companies-through-lockdown-and-market-adaptability-482271dfb8fd","previewImage":{"__ref":"ImageMetadata:1*vrj_U6tBMYzLADoE37jQ4g.jpeg"},"isPublished":true,"firstPublishedAt":1599483368915,"readingTime":5.276415094339622,"statusForCollection":"APPROVED","isLocked":true,"isShortform":false,"visibility":"LOCKED","collection":{"__ref":"Collection:70cd67c5d0e"},"creator":{"__ref":"User:ae8b4923c38d"},"previewContent":{"__typename":"PreviewContent","isFullContent":false}},"Post:f77d5718061a":{"id":"f77d5718061a","__typename":"Post","canonicalUrl":"","collection":{"__ref":"Collection:70cd67c5d0e"},"content({})":{"__typename":"PostContent","isLockedPreviewOnly":false,"validatedShareKey":"","isCacheableContent":false,"bodyModel":{"__typename":"RichText","paragraphs":[{"__ref":"Paragraph:43c983f0d936_0"},{"__ref":"Paragraph:43c983f0d936_1"},{"__ref":"Paragraph:43c983f0d936_2"},{"__ref":"Paragraph:43c983f0d936_3"},{"__ref":"Paragraph:43c983f0d936_4"},{"__ref":"Paragraph:43c983f0d936_5"},{"__ref":"Paragraph:43c983f0d936_6"},{"__ref":"Paragraph:43c983f0d936_7"},{"__ref":"Paragraph:43c983f0d936_8"},{"__ref":"Paragraph:43c983f0d936_9"},{"__ref":"Paragraph:43c983f0d936_10"},{"__ref":"Paragraph:43c983f0d936_11"},{"__ref":"Paragraph:43c983f0d936_12"},{"__ref":"Paragraph:43c983f0d936_13"},{"__ref":"Paragraph:43c983f0d936_14"},{"__ref":"Paragraph:43c983f0d936_15"},{"__ref":"Paragraph:43c983f0d936_16"},{"__ref":"Paragraph:43c983f0d936_17"},{"__ref":"Paragraph:43c983f0d936_18"},{"__ref":"Paragraph:43c983f0d936_19"},{"__ref":"Paragraph:43c983f0d936_20"},{"__ref":"Paragraph:43c983f0d936_21"},{"__ref":"Paragraph:43c983f0d936_22"},{"__ref":"Paragraph:43c983f0d936_23"},{"__ref":"Paragraph:43c983f0d936_24"},{"__ref":"Paragraph:43c983f0d936_25"},{"__ref":"Paragraph:43c983f0d936_26"},{"__ref":"Paragraph:43c983f0d936_27"},{"__ref":"Paragraph:43c983f0d936_28"},{"__ref":"Paragraph:43c983f0d936_29"},{"__ref":"Paragraph:43c983f0d936_30"},{"__ref":"Paragraph:43c983f0d936_31"},{"__ref":"Paragraph:43c983f0d936_32"},{"__ref":"Paragraph:43c983f0d936_33"},{"__ref":"Paragraph:43c983f0d936_34"},{"__ref":"Paragraph:43c983f0d936_35"},{"__ref":"Paragraph:43c983f0d936_36"},{"__ref":"Paragraph:43c983f0d936_37"},{"__ref":"Paragraph:43c983f0d936_38"},{"__ref":"Paragraph:43c983f0d936_39"},{"__ref":"Paragraph:43c983f0d936_40"},{"__ref":"Paragraph:43c983f0d936_41"},{"__ref":"Paragraph:43c983f0d936_42"},{"__ref":"Paragraph:43c983f0d936_43"},{"__ref":"Paragraph:43c983f0d936_44"},{"__ref":"Paragraph:43c983f0d936_45"},{"__ref":"Paragraph:43c983f0d936_46"},{"__ref":"Paragraph:43c983f0d936_47"},{"__ref":"Paragraph:43c983f0d936_48"},{"__ref":"Paragraph:43c983f0d936_49"},{"__ref":"Paragraph:43c983f0d936_50"},{"__ref":"Paragraph:43c983f0d936_51"},{"__ref":"Paragraph:43c983f0d936_52"},{"__ref":"Paragraph:43c983f0d936_53"},{"__ref":"Paragraph:43c983f0d936_54"},{"__ref":"Paragraph:43c983f0d936_55"},{"__ref":"Paragraph:43c983f0d936_56"},{"__ref":"Paragraph:43c983f0d936_57"},{"__ref":"Paragraph:43c983f0d936_58"},{"__ref":"Paragraph:43c983f0d936_59"},{"__ref":"Paragraph:43c983f0d936_60"},{"__ref":"Paragraph:43c983f0d936_61"},{"__ref":"Paragraph:43c983f0d936_62"},{"__ref":"Paragraph:43c983f0d936_63"},{"__ref":"Paragraph:43c983f0d936_64"},{"__ref":"Paragraph:43c983f0d936_65"},{"__ref":"Paragraph:43c983f0d936_66"},{"__ref":"Paragraph:43c983f0d936_67"},{"__ref":"Paragraph:43c983f0d936_68"},{"__ref":"Paragraph:43c983f0d936_69"},{"__ref":"Paragraph:43c983f0d936_70"},{"__ref":"Paragraph:43c983f0d936_71"},{"__ref":"Paragraph:43c983f0d936_72"},{"__ref":"Paragraph:43c983f0d936_73"},{"__ref":"Paragraph:43c983f0d936_74"}],"sections":[{"__typename":"Section","name":"00a3","startIndex":0,"textLayout":null,"imageLayout":null,"backgroundImage":null,"videoLayout":null,"backgroundVideo":null},{"__typename":"Section","name":"b021","startIndex":71,"textLayout":null,"imageLayout":null,"backgroundImage":null,"videoLayout":null,"backgroundVideo":null},{"__typename":"Section","name":"983b","startIndex":73,"textLayout":null,"imageLayout":null,"backgroundImage":null,"videoLayout":null,"backgroundVideo":null}]}},"creator":{"__ref":"User:2d3c4b088dc6"},"customStyleSheet":{"__ref":"CustomStyleSheet:f87c127d73df"},"firstPublishedAt":1575262880163,"isLocked":true,"isPublished":true,"isShortform":false,"layerCake":4,"primaryTopic":null,"title":"How To Build Your Own MuZero AI Using Python (Part 1\u002F3)","readCreatorPostsCount":0,"mediumUrl":"https:\u002F\u002Fmedium.com\u002Fapplied-data-science\u002Fhow-to-build-your-own-muzero-in-python-f77d5718061a","isLimitedState":false,"visibility":"LOCKED","license":"ALL_RIGHTS_RESERVED","allowResponses":true,"newsletterId":"","sequence":null,"tags":[{"__ref":"Tag:machine-learning"},{"__ref":"Tag:deep-learning"},{"__ref":"Tag:data-science"},{"__ref":"Tag:artificial-intelligence"},{"__ref":"Tag:ai"}],"topics":[],"viewerClapCount":0,"showSubscribeToProfilePromo":false,"showSubscribeToCollectionNewsletterV3Promo":false,"inResponseToPostResult":null,"isNewsletter":false,"socialTitle":"","socialDek":"","metaDescription":"A Walkthrough Of The DeepMind MuZero Pseudocode","latestPublishedAt":1614071505969,"readingTime":6.918867924528302,"previewContent":{"__typename":"PreviewContent","subtitle":"Teaching A Machine To Play Games Using Self-Play And Deep Learning…Without Telling It The Rules 🤯"},"previewImage":{"__ref":"ImageMetadata:1*y-FnMby1uCDxw6Br-BeZng.png"},"creatorPartnerProgramEnrollmentStatus":"PERMISSION_DENIED","clapCount":1338,"lockedSource":"LOCKED_POST_SOURCE_UGC_UNENROLLED","isSuspended":false,"pendingCollection":null,"statusForCollection":"APPROVED","pinnedAt":0,"pinnedByCreatorAt":0,"curationEligibleAt":1575262877480,"responseDistribution":"NOT_DISTRIBUTED","shareKey":null,"internalLinks({\"paging\":{\"limit\":8}})":{"__typename":"InternalLinksConnection","items":[{"__ref":"Post:70ffe0a429a2"},{"__ref":"Post:247d0b440717"},{"__ref":"Post:2dcc32fe2c02"},{"__ref":"Post:d95e1d4fe558"},{"__ref":"Post:93e03bfdc151"},{"__ref":"Post:919e32e88d18"},{"__ref":"Post:ddebd92ae26c"},{"__ref":"Post:482271dfb8fd"}]},"collaborators":[],"translationSourcePost":null,"inResponseToMediaResource":null,"isDistributionAlertDismissed":true,"audioVersionUrl":"","seoTitle":"","updatedAt":1614071509158,"shortformType":"SHORTFORM_TYPE_LINK","structuredData":"","seoDescription":"","postResponses":{"__typename":"PostResponses","count":4},"latestPublishedVersion":"43c983f0d936","isPublishToEmail":false,"readingList":"READING_LIST_NONE","voterCount":296,"recommenders":[]}}</script><script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/manifest.1fd15351.js"></script><script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/5566.011c6c68.js"></script><script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/main.4f933583.js"></script><script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/5573.159bf40f.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/instrumentation.2774f137.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/reporting.0e714607.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1752.a348f767.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/1491.c08ce3ca.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/4964.fb36722e.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/8342.6aa0b45e.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/9692.3fc1c18a.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/4586.7c07e0df.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/5064.ee9cac75.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/9355.c6c38ae6.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/2846.bbadae04.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/7012.f1ee6871.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/6230.b596e4e9.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/9972.a4eec407.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/5127.810d9eac.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/8580.84adae83.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/8751.8bd4ce88.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/2955.e3a25a19.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/7131.4059cf64.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/8127.58f0a64c.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/6371.4672dd1f.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/304.7fb69e54.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/2874.661d5cbc.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/2514.2c8bf092.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/3874.19af3a09.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/2558.1ac5c4fd.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/8286.962b84f6.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/5025.250e5ada.chunk.js"></script>
|
||
<script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/Post.4a47de6d.chunk.js"></script><script>window.main();</script><script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/p.js" async="" id="parsely-cf"></script><iframe src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/a16180790160.html" hidden="" tabindex="-1" title="Optimizely Internal Frame" height="0" width="0" style="display: none;"></iframe><script src="./MuZero_ The Walkthrough (Part 1_3) _ by David Foster _ Applied Data Science _ Medium_files/client" async=""></script></body></html> |