import{S as sv,i as nv,s as iv,e as o,k as c,w as m,t as s,M as pv,c as a,d as l,m as f,a as r,x as u,h as n,b as h,N as xe,G as t,g as p,y as d,L as hv,q as y,o as v,B as w,v as cv}from"../../chunks/vendor-hf-doc-builder.js";import{I as T}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as _}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as fv}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function mv(Am){let R,_r,C,he,mo,Oe,li,uo,oi,Ur,Ke,jr,et,ai,tt,Zl,Nm,Ir,V,ce,yo,lt,ri,vo,si,Er,ot,Rm,kr,fe,ni,at,ii,pi,Zr,Gl,hi,Gr,k,wo,me,rt,ci,fi,Mo,mi,ui,di,go,E,yi,bo,vi,wi,Jo,Mi,gi,To,bi,Ji,Ti,_o,st,_i,nt,Ui,ji,$r,it,Br,ue,Ii,pt,Ei,ki,Sr,$l,Bl,Uo,Zi,Gi,Wr,X,$i,ht,Bi,Si,jo,Wi,Ar,ct,Ai,Io,Ni,Nr,ft,Ri,mt,Ci,Rr,z,de,Eo,ut,Vi,ko,Xi,Cr,Sl,ye,zi,Zo,qi,Hi,Go,Fi,Vr,dt,Cm,Xr,Wl,$o,Bo,Yi,zr,yt,Vm,qr,ve,Li,So,Pi,Qi,Hr,q,we,Wo,vt,Di,Ao,xi,Fr,H,wt,Oi,Ki,No,ep,tp,Yr,Z,lp,Ro,op,ap,Co,rp,sp,Lr,Mt,Xm,Pr,F,Me,Vo,gt,np,Xo,ip,Qr,M,bt,pp,Jt,hp,cp,fp,Al,Tt,mp,up,dp,Nl,_t,yp,vp,wp,Rl,Mp,Ut,gp,bp,jt,Jp,It,Tp,_p,Up,Et,jp,kt,Ip,Ep,kp,zo,Zp,Gp,qo,$p,Bp,Cl,Sp,Zt,Wp,Ap,Gt,Np,$t,Rp,Cp,Vp,ge,Xp,Bt,zp,qp,St,Hp,Fp,Vl,Wt,Yp,Lp,Pp,U,At,Qp,Dp,Nt,xp,Op,Rt,Kp,eh,Ct,th,lh,Dr,Vt,oh,Xt,ah,xr,Y,be,Ho,zt,rh,Fo,sh,Or,Je,nh,Yo,ih,ph,Kr,Te,hh,Lo,ch,fh,es,G,mh,Po,uh,dh,Qo,yh,vh,ts,$,wh,Do,Mh,gh,xo,bh,Jh,ls,B,Th,Oo,_h,Uh,Ko,jh,Ih,os,S,Eh,ea,kh,Zh,ta,Gh,$h,as,qt,zm,rs,L,_e,la,Ht,Bh,oa,Sh,ss,Xl,Wh,ns,W,zl,aa,Ah,Nh,Rh,ql,ra,Ch,Vh,Xh,Hl,sa,zh,qh,is,Ue,Hh,Ft,Fh,Yh,ps,P,je,na,Yt,Lh,ia,Ph,hs,Q,Lt,Qh,Dh,pa,xh,Oh,cs,Fl,Kh,fs,A,ec,ha,tc,lc,ca,oc,ac,ms,Yl,rc,us,g,fa,sc,nc,ma,ic,pc,Ll,Pt,hc,cc,fc,ua,mc,uc,da,dc,yc,ya,vc,wc,va,Mc,gc,wa,bc,Jc,Ma,Tc,_c,ga,Uc,jc,ba,Ic,Ec,Ja,kc,Zc,Ta,Gc,ds,D,Ie,_a,Qt,$c,Ua,Bc,ys,Pl,Sc,vs,Ee,Wc,Dt,Ac,Nc,ws,xt,Ms,x,ke,ja,Ot,Rc,Ia,Cc,gs,Ql,Ea,Vc,bs,Kt,Js,O,Ze,ka,el,Xc,Za,zc,Ts,tl,_s,Ge,qc,Ga,Hc,Fc,Us,K,$e,$a,ll,Yc,Ba,Lc,js,ol,qm,Is,j,Pc,Sa,Qc,Dc,Wa,xc,Oc,Aa,Kc,ef,Es,N,tf,Na,lf,of,Ra,af,rf,ks,Dl,sf,Zs,I,Ca,nf,pf,Va,hf,cf,Xa,ff,mf,za,uf,Gs,Be,df,al,yf,vf,$s,Se,wf,rl,Mf,gf,Bs,ee,We,qa,sl,bf,Ha,Jf,Ss,xl,Fa,Tf,Ws,nl,As,te,Ae,Ya,il,_f,La,Uf,Ns,pl,Rs,le,Ne,Pa,hl,jf,Qa,If,Cs,cl,Vs,Ol,Ef,Xs,oe,Re,Da,fl,kf,xa,Zf,zs,Kl,Gf,qs,ml,$f,ul,Bf,Hs,eo,Sf,Fs,to,Ce,Wf,dl,Af,Nf,Oa,Rf,Ys,yl,Hm,Ls,Ve,Ka,Cf,Vf,er,Xf,Ps,vl,zf,tr,qf,Qs,wl,Ds,Ml,xs,ae,Xe,lr,gl,Hf,or,Ff,Os,lo,Yf,Ks,bl,en,Jl,tn,Tl,ln,_l,on,re,ze,ar,Ul,Lf,rr,Pf,an,qe,Qf,sr,Df,xf,rn,He,Of,nr,Kf,em,sn,jl,nn,Fe,tm,ir,lm,om,pn,Il,hn,se,am,pr,rm,sm,hr,nm,cn,El,im,ne,pm,cr,hm,cm,fr,fm,mm,fn,mn,un,Ye,um,mr,dm,ym,dn,ie,Le,ur,kl,vm,dr,wm,yn;return Oe=new T({}),Ke=new fv({props:{classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/deep-rl-class/blob/main/notebooks/unit8/unit8_part2.ipynb"}],askForHelpUrl:"http://hf.co/join/discord"}}),lt=new T({}),it=new _({props:{code:"ZnJvbSUyMElQeXRob24uZGlzcGxheSUyMGltcG9ydCUyMEhUTUwlMEElMEFIVE1MKCUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMiUzQ3ZpZGVvJTIwd2lkdGglM0QlMjI2NDAlMjIlMjBoZWlnaHQlM0QlMjI0ODAlMjIlMjBjb250cm9scyUzRSUwQSUyMCUyMCUzQ3NvdXJjZSUyMHNyYyUzRCUyMmh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZlZGJlZWNoaW5nJTJGZG9vbV9oZWFsdGhfZ2F0aGVyaW5nX3N1cHJlbWVfMzMzMyUyRnJlc29sdmUlMkZtYWluJTJGcmVwbGF5Lm1wNCUyMiUwQSUyMCUyMHR5cGUlM0QlMjJ2aWRlbyUyRm1wNCUyMiUzRVlvdXIlMjBicm93c2VyJTIwZG9lcyUyMG5vdCUyMHN1cHBvcnQlMjB0aGUlMjB2aWRlbyUyMHRhZy4lM0MlMkZ2aWRlbyUzRSUyMiUyMiUyMiUwQSk=",highlighted:`<span class="hljs-keyword">from</span> IPython.display <span class="hljs-keyword">import</span> HTML

HTML(
    <span class="hljs-string">&quot;&quot;&quot;&lt;video width=&quot;640&quot; height=&quot;480&quot; controls&gt;
  &lt;source src=&quot;https://huggingface.co/edbeeching/doom_health_gathering_supreme_3333/resolve/main/replay.mp4&quot;
  type=&quot;video/mp4&quot;&gt;Your browser does not support the video tag.&lt;/video&gt;&quot;&quot;&quot;</span>
)`}}),ut=new T({}),vt=new T({}),gt=new T({}),zt=new T({}),Ht=new T({}),Yt=new T({}),Qt=new T({}),xt=new _({props:{code:"JTIzJTIwSW5zdGFsbCUyMFZpWkRvb20lMjBkZXBzJTIwZnJvbSUwQSUyMyUyMGh0dHBzJTNBJTJGJTJGZ2l0aHViLmNvbSUyRm13eWRtdWNoJTJGVmlaRG9vbSUyRmJsb2IlMkZtYXN0ZXIlMkZkb2MlMkZCdWlsZGluZy5tZCUyMy1saW51eCUwQSUwQWFwdC1nZXQlMjBpbnN0YWxsJTIwYnVpbGQtZXNzZW50aWFsJTIwemxpYjFnLWRldiUyMGxpYnNkbDItZGV2JTIwbGlianBlZy1kZXYlMjAlNUMlMEFuYXNtJTIwdGFyJTIwbGliYnoyLWRldiUyMGxpYmd0azIuMC1kZXYlMjBjbWFrZSUyMGdpdCUyMGxpYmZsdWlkc3ludGgtZGV2JTIwbGliZ21lLWRldiUyMCU1QyUwQWxpYm9wZW5hbC1kZXYlMjB0aW1pZGl0eSUyMGxpYndpbGRtaWRpLWRldiUyMHVuemlwJTIwZmZtcGVnJTBBJTBBJTIzJTIwQm9vc3QlMjBsaWJyYXJpZXMlMEFhcHQtZ2V0JTIwaW5zdGFsbCUyMGxpYmJvb3N0LWFsbC1kZXYlMEElMEElMjMlMjBMdWElMjBiaW5kaW5nJTIwZGVwZW5kZW5jaWVzJTBBYXB0LWdldCUyMGluc3RhbGwlMjBsaWJsdWE1LjEtZGV2",highlighted:`<span class="hljs-comment"># Install ViZDoom deps from</span>
<span class="hljs-comment"># https://github.com/mwydmuch/ViZDoom/blob/master/doc/Building.md#-linux</span>

apt-get install build-essential zlib1g-dev libsdl2-dev libjpeg-dev \\
nasm tar libbz2-dev libgtk2<span class="hljs-number">.0</span>-dev cmake git libfluidsynth-dev libgme-dev \\
libopenal-dev timidity libwildmidi-dev unzip ffmpeg

<span class="hljs-comment"># Boost libraries</span>
apt-get install libboost-<span class="hljs-built_in">all</span>-dev

<span class="hljs-comment"># Lua binding dependencies</span>
apt-get install liblua5<span class="hljs-number">.1</span>-dev`}}),Ot=new T({}),Kt=new _({props:{code:"cGlwJTIwaW5zdGFsbCUyMHNhbXBsZS1mYWN0b3J5JTBBcGlwJTIwaW5zdGFsbCUyMHZpemRvb20=",highlighted:`pip install sample-factory
pip install vizdoom`}}),el=new T({}),tl=new _({props:{code:"aW1wb3J0JTIwZnVuY3Rvb2xzJTBBJTBBZnJvbSUyMHNhbXBsZV9mYWN0b3J5LmFsZ28udXRpbHMuY29udGV4dCUyMGltcG9ydCUyMGdsb2JhbF9tb2RlbF9mYWN0b3J5JTBBZnJvbSUyMHNhbXBsZV9mYWN0b3J5LmNmZy5hcmd1bWVudHMlMjBpbXBvcnQlMjBwYXJzZV9mdWxsX2NmZyUyQyUyMHBhcnNlX3NmX2FyZ3MlMEFmcm9tJTIwc2FtcGxlX2ZhY3RvcnkuZW52cy5lbnZfdXRpbHMlMjBpbXBvcnQlMjByZWdpc3Rlcl9lbnYlMEFmcm9tJTIwc2FtcGxlX2ZhY3RvcnkudHJhaW4lMjBpbXBvcnQlMjBydW5fcmwlMEElMEFmcm9tJTIwc2ZfZXhhbXBsZXMudml6ZG9vbS5kb29tLmRvb21fbW9kZWwlMjBpbXBvcnQlMjBtYWtlX3ZpemRvb21fZW5jb2RlciUwQWZyb20lMjBzZl9leGFtcGxlcy52aXpkb29tLmRvb20uZG9vbV9wYXJhbXMlMjBpbXBvcnQlMjBhZGRfZG9vbV9lbnZfYXJncyUyQyUyMGRvb21fb3ZlcnJpZGVfZGVmYXVsdHMlMEFmcm9tJTIwc2ZfZXhhbXBsZXMudml6ZG9vbS5kb29tLmRvb21fdXRpbHMlMjBpbXBvcnQlMjBET09NX0VOVlMlMkMlMjBtYWtlX2Rvb21fZW52X2Zyb21fc3BlYyUwQSUwQSUwQSUyMyUyMFJlZ2lzdGVycyUyMGFsbCUyMHRoZSUyMFZpWkRvb20lMjBlbnZpcm9ubWVudHMlMEFkZWYlMjByZWdpc3Rlcl92aXpkb29tX2VudnMoKSUzQSUwQSUyMCUyMCUyMCUyMGZvciUyMGVudl9zcGVjJTIwaW4lMjBET09NX0VOVlMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtYWtlX2Vudl9mdW5jJTIwJTNEJTIwZnVuY3Rvb2xzLnBhcnRpYWwobWFrZV9kb29tX2Vudl9mcm9tX3NwZWMlMkMlMjBlbnZfc3BlYyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZWdpc3Rlcl9lbnYoZW52X3NwZWMubmFtZSUyQyUyMG1ha2VfZW52X2Z1bmMpJTBBJTBBJTBBJTIzJTIwU2FtcGxlJTIwRmFjdG9yeSUyMGFsbG93cyUyMHRoZSUyMHJlZ2lzdHJhdGlvbiUyMG9mJTIwYSUyMGN1c3RvbSUyME5ldXJhbCUyME5ldHdvcmslMjBhcmNoaXRlY3R1cmUlMEElMjMlMjBTZWUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZhbGV4LXBldHJlbmtvJTJGc2FtcGxlLWZhY3RvcnklMkZibG9iJTJGbWFzdGVyJTJGc2ZfZXhhbXBsZXMlMkZ2aXpkb29tJTJGZG9vbSUyRmRvb21fbW9kZWwucHklMjBmb3IlMjBtb3JlJTIwZGV0YWlscyUwQWRlZiUyMHJlZ2lzdGVyX3ZpemRvb21fbW9kZWxzKCklM0ElMEElMjAlMjAlMjAlMjBnbG9iYWxfbW9kZWxfZmFjdG9yeSgpLnJlZ2lzdGVyX2VuY29kZXJfZmFjdG9yeShtYWtlX3ZpemRvb21fZW5jb2RlciklMEElMEElMEFkZWYlMjByZWdpc3Rlcl92aXpkb29tX2NvbXBvbmVudHMoKSUzQSUwQSUyMCUyMCUyMCUyMHJlZ2lzdGVyX3ZpemRvb21fZW52cygpJTBBJTIwJTIwJTIwJTIwcmVnaXN0ZXJfdml6ZG9vbV9tb2RlbHMoKSUwQSUwQSUwQSUyMyUyMHBhcnNlJTIwdGhlJTIwY29tbWFuZCUyMGxpbmUlMjBhcmdzJTIwYW5kJTIwY3JlYXRlJTIwYSUyMGNvbmZpZyUwQWRlZiUyMHBhcnNlX3ZpemRvb21fY2ZnKGFyZ3YlM0ROb25lJTJDJTIwZXZhbHVhdGlvbiUzREZhbHNlKSUzQSUwQSUyMCUyMCUyMCUyMHBhcnNlciUyQyUyMF8lMjAlM0QlMjBwYXJzZV9zZl9hcmdzKGFyZ3YlM0Rhcmd2JTJDJTIwZXZhbHVhdGlvbiUzRGV2YWx1YXRpb24pJTBBJTIwJTIwJTIwJTIwJTIzJTIwcGFyYW1ldGVycyUyMHNwZWNpZmljJTIwdG8lMjBEb29tJTIwZW52cyUwQSUyMCUyMCUyMCUyMGFkZF9kb29tX2Vudl9hcmdzKHBhcnNlciklMEElMjAlMjAlMjAlMjAlMjMlMjBvdmVycmlkZSUyMERvb20lMjBkZWZhdWx0JTIwdmFsdWVzJTIwZm9yJTIwYWxnbyUyMHBhcmFtZXRlcnMlMEElMjAlMjAlMjAlMjBkb29tX292ZXJyaWRlX2RlZmF1bHRzKHBhcnNlciklMEElMjAlMjAlMjAlMjAlMjMlMjBzZWNvbmQlMjBwYXJzaW5nJTIwcGFzcyUyMHlpZWxkcyUyMHRoZSUyMGZpbmFsJTIwY29uZmlndXJhdGlvbiUwQSUyMCUyMCUyMCUyMGZpbmFsX2NmZyUyMCUzRCUyMHBhcnNlX2Z1bGxfY2ZnKHBhcnNlciUyQyUyMGFyZ3YpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwZmluYWxfY2Zn",highlighted:`<span class="hljs-keyword">import</span> functools

<span class="hljs-keyword">from</span> sample_factory.algo.utils.context <span class="hljs-keyword">import</span> global_model_factory
<span class="hljs-keyword">from</span> sample_factory.cfg.arguments <span class="hljs-keyword">import</span> parse_full_cfg, parse_sf_args
<span class="hljs-keyword">from</span> sample_factory.envs.env_utils <span class="hljs-keyword">import</span> register_env
<span class="hljs-keyword">from</span> sample_factory.train <span class="hljs-keyword">import</span> run_rl

<span class="hljs-keyword">from</span> sf_examples.vizdoom.doom.doom_model <span class="hljs-keyword">import</span> make_vizdoom_encoder
<span class="hljs-keyword">from</span> sf_examples.vizdoom.doom.doom_params <span class="hljs-keyword">import</span> add_doom_env_args, doom_override_defaults
<span class="hljs-keyword">from</span> sf_examples.vizdoom.doom.doom_utils <span class="hljs-keyword">import</span> DOOM_ENVS, make_doom_env_from_spec


<span class="hljs-comment"># Registers all the ViZDoom environments</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">register_vizdoom_envs</span>():
    <span class="hljs-keyword">for</span> env_spec <span class="hljs-keyword">in</span> DOOM_ENVS:
        make_env_func = functools.partial(make_doom_env_from_spec, env_spec)
        register_env(env_spec.name, make_env_func)


<span class="hljs-comment"># Sample Factory allows the registration of a custom Neural Network architecture</span>
<span class="hljs-comment"># See https://github.com/alex-petrenko/sample-factory/blob/master/sf_examples/vizdoom/doom/doom_model.py for more details</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">register_vizdoom_models</span>():
    global_model_factory().register_encoder_factory(make_vizdoom_encoder)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">register_vizdoom_components</span>():
    register_vizdoom_envs()
    register_vizdoom_models()


<span class="hljs-comment"># parse the command line args and create a config</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">parse_vizdoom_cfg</span>(<span class="hljs-params">argv=<span class="hljs-literal">None</span>, evaluation=<span class="hljs-literal">False</span></span>):
    parser, _ = parse_sf_args(argv=argv, evaluation=evaluation)
    <span class="hljs-comment"># parameters specific to Doom envs</span>
    add_doom_env_args(parser)
    <span class="hljs-comment"># override Doom default values for algo parameters</span>
    doom_override_defaults(parser)
    <span class="hljs-comment"># second parsing pass yields the final configuration</span>
    final_cfg = parse_full_cfg(parser, argv)
    <span class="hljs-keyword">return</span> final_cfg`}}),ll=new T({}),sl=new T({}),nl=new _({props:{code:"JTIzJTIzJTIwU3RhcnQlMjB0aGUlMjB0cmFpbmluZyUyQyUyMHRoaXMlMjBzaG91bGQlMjB0YWtlJTIwYXJvdW5kJTIwMTUlMjBtaW51dGVzJTBBcmVnaXN0ZXJfdml6ZG9vbV9jb21wb25lbnRzKCklMEElMEElMjMlMjBUaGUlMjBzY2VuYXJpbyUyMHdlJTIwdHJhaW4lMjBvbiUyMHRvZGF5JTIwaXMlMjBoZWFsdGglMjBnYXRoZXJpbmclMEElMjMlMjBvdGhlciUyMHNjZW5hcmlvcyUyMGluY2x1ZGUlMjAlMjJkb29tX2Jhc2ljJTIyJTJDJTIwJTIyZG9vbV90d29fY29sb3JzX2Vhc3klMjIlMkMlMjAlMjJkb29tX2RtJTIyJTJDJTIwJTIyZG9vbV9kd2FuZ281JTIyJTJDJTIwJTIyZG9vbV9teV93YXlfaG9tZSUyMiUyQyUyMCUyMmRvb21fZGVhZGx5X2NvcnJpZG9yJTIyJTJDJTIwJTIyZG9vbV9kZWZlbmRfdGhlX2NlbnRlciUyMiUyQyUyMCUyMmRvb21fZGVmZW5kX3RoZV9saW5lJTIyJTBBZW52JTIwJTNEJTIwJTIyZG9vbV9oZWFsdGhfZ2F0aGVyaW5nX3N1cHJlbWUlMjIlMEFjZmclMjAlM0QlMjBwYXJzZV92aXpkb29tX2NmZyglMEElMjAlMjAlMjAlMjBhcmd2JTNEJTVCZiUyMi0tZW52JTNEJTdCZW52JTdEJTIyJTJDJTIwJTIyLS1udW1fd29ya2VycyUzRDglMjIlMkMlMjAlMjItLW51bV9lbnZzX3Blcl93b3JrZXIlM0Q0JTIyJTJDJTIwJTIyLS10cmFpbl9mb3JfZW52X3N0ZXBzJTNENDAwMDAwMCUyMiU1RCUwQSklMEElMEFzdGF0dXMlMjAlM0QlMjBydW5fcmwoY2ZnKQ==",highlighted:`<span class="hljs-comment">## Start the training, this should take around 15 minutes</span>
register_vizdoom_components()

<span class="hljs-comment"># The scenario we train on today is health gathering</span>
<span class="hljs-comment"># other scenarios include &quot;doom_basic&quot;, &quot;doom_two_colors_easy&quot;, &quot;doom_dm&quot;, &quot;doom_dwango5&quot;, &quot;doom_my_way_home&quot;, &quot;doom_deadly_corridor&quot;, &quot;doom_defend_the_center&quot;, &quot;doom_defend_the_line&quot;</span>
env = <span class="hljs-string">&quot;doom_health_gathering_supreme&quot;</span>
cfg = parse_vizdoom_cfg(
    argv=[<span class="hljs-string">f&quot;--env=<span class="hljs-subst">{env}</span>&quot;</span>, <span class="hljs-string">&quot;--num_workers=8&quot;</span>, <span class="hljs-string">&quot;--num_envs_per_worker=4&quot;</span>, <span class="hljs-string">&quot;--train_for_env_steps=4000000&quot;</span>]
)

status = run_rl(cfg)`}}),il=new T({}),pl=new _({props:{code:"ZnJvbSUyMHNhbXBsZV9mYWN0b3J5LmVuam95JTIwaW1wb3J0JTIwZW5qb3klMEElMEFjZmclMjAlM0QlMjBwYXJzZV92aXpkb29tX2NmZyglMEElMjAlMjAlMjAlMjBhcmd2JTNEJTVCZiUyMi0tZW52JTNEJTdCZW52JTdEJTIyJTJDJTIwJTIyLS1udW1fd29ya2VycyUzRDElMjIlMkMlMjAlMjItLXNhdmVfdmlkZW8lMjIlMkMlMjAlMjItLW5vX3JlbmRlciUyMiUyQyUyMCUyMi0tbWF4X251bV9lcGlzb2RlcyUzRDEwJTIyJTVEJTJDJTIwZXZhbHVhdGlvbiUzRFRydWUlMEEpJTBBc3RhdHVzJTIwJTNEJTIwZW5qb3koY2ZnKQ==",highlighted:`<span class="hljs-keyword">from</span> sample_factory.enjoy <span class="hljs-keyword">import</span> enjoy

cfg = parse_vizdoom_cfg(
    argv=[<span class="hljs-string">f&quot;--env=<span class="hljs-subst">{env}</span>&quot;</span>, <span class="hljs-string">&quot;--num_workers=1&quot;</span>, <span class="hljs-string">&quot;--save_video&quot;</span>, <span class="hljs-string">&quot;--no_render&quot;</span>, <span class="hljs-string">&quot;--max_num_episodes=10&quot;</span>], evaluation=<span class="hljs-literal">True</span>
)
status = enjoy(cfg)`}}),hl=new T({}),cl=new _({props:{code:"ZnJvbSUyMGJhc2U2NCUyMGltcG9ydCUyMGI2NGVuY29kZSUwQWZyb20lMjBJUHl0aG9uLmRpc3BsYXklMjBpbXBvcnQlMjBIVE1MJTBBJTBBbXA0JTIwJTNEJTIwb3BlbiglMjIlMkZjb250ZW50JTJGdHJhaW5fZGlyJTJGZGVmYXVsdF9leHBlcmltZW50JTJGcmVwbGF5Lm1wNCUyMiUyQyUyMCUyMnJiJTIyKS5yZWFkKCklMEFkYXRhX3VybCUyMCUzRCUyMCUyMmRhdGElM0F2aWRlbyUyRm1wNCUzQmJhc2U2NCUyQyUyMiUyMCUyQiUyMGI2NGVuY29kZShtcDQpLmRlY29kZSgpJTBBSFRNTCglMEElMjAlMjAlMjAlMjAlMjIlMjIlMjIlMEElM0N2aWRlbyUyMHdpZHRoJTNENjQwJTIwY29udHJvbHMlM0UlMEElMjAlMjAlMjAlMjAlMjAlMjAlM0Nzb3VyY2UlMjBzcmMlM0QlMjIlMjVzJTIyJTIwdHlwZSUzRCUyMnZpZGVvJTJGbXA0JTIyJTNFJTBBJTNDJTJGdmlkZW8lM0UlMEElMjIlMjIlMjIlMEElMjAlMjAlMjAlMjAlMjUlMjBkYXRhX3VybCUwQSk=",highlighted:`<span class="hljs-keyword">from</span> base64 <span class="hljs-keyword">import</span> b64encode
<span class="hljs-keyword">from</span> IPython.display <span class="hljs-keyword">import</span> HTML

mp4 = <span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;/content/train_dir/default_experiment/replay.mp4&quot;</span>, <span class="hljs-string">&quot;rb&quot;</span>).read()
data_url = <span class="hljs-string">&quot;data:video/mp4;base64,&quot;</span> + b64encode(mp4).decode()
HTML(
    <span class="hljs-string">&quot;&quot;&quot;
&lt;video width=640 controls&gt;
      &lt;source src=&quot;%s&quot; type=&quot;video/mp4&quot;&gt;
&lt;/video&gt;
&quot;&quot;&quot;</span>
    % data_url
)`}}),fl=new T({}),wl=new _({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBbm90ZWJvb2tfbG9naW4oKSUwQSFnaXQlMjBjb25maWclMjAtLWdsb2JhbCUyMGNyZWRlbnRpYWwuaGVscGVyJTIwc3RvcmU=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login
notebook_login()
!git config --<span class="hljs-keyword">global</span> credential.helper store`}}),Ml=new _({props:{code:"ZnJvbSUyMHNhbXBsZV9mYWN0b3J5LmVuam95JTIwaW1wb3J0JTIwZW5qb3klMEElMEFoZl91c2VybmFtZSUyMCUzRCUyMCUyMlRob21hc1NpbW9uaW5pJTIyJTIwJTIwJTIzJTIwaW5zZXJ0JTIweW91ciUyMEh1Z2dpbmdGYWNlJTIwdXNlcm5hbWUlMjBoZXJlJTBBJTBBY2ZnJTIwJTNEJTIwcGFyc2Vfdml6ZG9vbV9jZmcoJTBBJTIwJTIwJTIwJTIwYXJndiUzRCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGYlMjItLWVudiUzRCU3QmVudiU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMi0tbnVtX3dvcmtlcnMlM0QxJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyLS1zYXZlX3ZpZGVvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyLS1ub19yZW5kZXIlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjItLW1heF9udW1fZXBpc29kZXMlM0QxMCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMi0tbWF4X251bV9mcmFtZXMlM0QxMDAwMDAlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjItLXB1c2hfdG9faHViJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZiUyMi0taGZfcmVwb3NpdG9yeSUzRCU3QmhmX3VzZXJuYW1lJTdEJTJGcmxfY291cnNlX3ZpemRvb21faGVhbHRoX2dhdGhlcmluZ19zdXByZW1lJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbHVhdGlvbiUzRFRydWUlMkMlMEEpJTBBc3RhdHVzJTIwJTNEJTIwZW5qb3koY2ZnKQ==",highlighted:`<span class="hljs-keyword">from</span> sample_factory.enjoy <span class="hljs-keyword">import</span> enjoy

hf_username = <span class="hljs-string">&quot;ThomasSimonini&quot;</span>  <span class="hljs-comment"># insert your HuggingFace username here</span>

cfg = parse_vizdoom_cfg(
    argv=[
        <span class="hljs-string">f&quot;--env=<span class="hljs-subst">{env}</span>&quot;</span>,
        <span class="hljs-string">&quot;--num_workers=1&quot;</span>,
        <span class="hljs-string">&quot;--save_video&quot;</span>,
        <span class="hljs-string">&quot;--no_render&quot;</span>,
        <span class="hljs-string">&quot;--max_num_episodes=10&quot;</span>,
        <span class="hljs-string">&quot;--max_num_frames=100000&quot;</span>,
        <span class="hljs-string">&quot;--push_to_hub&quot;</span>,
        <span class="hljs-string">f&quot;--hf_repository=<span class="hljs-subst">{hf_username}</span>/rl_course_vizdoom_health_gathering_supreme&quot;</span>,
    ],
    evaluation=<span class="hljs-literal">True</span>,
)
status = enjoy(cfg)`}}),gl=new T({}),bl=new _({props:{code:"JTIzZG93bmxvYWQlMjB0aGUlMjBhZ2VudCUyMGZyb20lMjB0aGUlMjBodWIlMEFweXRob24lMjAtbSUyMHNhbXBsZV9mYWN0b3J5Lmh1Z2dpbmdmYWNlLmxvYWRfZnJvbV9odWIlMjAtciUyMGVkYmVlY2hpbmclMkZkb29tX2hlYWx0aF9nYXRoZXJpbmdfc3VwcmVtZV8yMjIyJTIwLWQlMjAuJTJGdHJhaW5fZGly",highlighted:`<span class="hljs-comment">#download the agent from the hub</span>
python -m sample_factory.huggingface.load_from_hub -r edbeeching/doom_health_gathering_supreme_2222 -d ./train_dir`}}),Jl=new _({props:{code:"bHMlMjB0cmFpbl9kaXIlMkZkb29tX2hlYWx0aF9nYXRoZXJpbmdfc3VwcmVtZV8yMjIy",highlighted:'<span class="hljs-built_in">ls</span> train_dir/doom_health_gathering_supreme_2222'}}),Tl=new _({props:{code:"ZW52JTIwJTNEJTIwJTIyZG9vbV9oZWFsdGhfZ2F0aGVyaW5nX3N1cHJlbWUlMjIlMEFjZmclMjAlM0QlMjBwYXJzZV92aXpkb29tX2NmZyglMEElMjAlMjAlMjAlMjBhcmd2JTNEJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZiUyMi0tZW52JTNEJTdCZW52JTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyLS1udW1fd29ya2VycyUzRDElMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjItLXNhdmVfdmlkZW8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjItLW5vX3JlbmRlciUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMi0tbWF4X251bV9lcGlzb2RlcyUzRDEwJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyLS1leHBlcmltZW50JTNEZG9vbV9oZWFsdGhfZ2F0aGVyaW5nX3N1cHJlbWVfMjIyMiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMi0tdHJhaW5fZGlyJTNEdHJhaW5fZGlyJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbHVhdGlvbiUzRFRydWUlMkMlMEEpJTBBc3RhdHVzJTIwJTNEJTIwZW5qb3koY2ZnKQ==",highlighted:`env = <span class="hljs-string">&quot;doom_health_gathering_supreme&quot;</span>
cfg = parse_vizdoom_cfg(
    argv=[
        <span class="hljs-string">f&quot;--env=<span class="hljs-subst">{env}</span>&quot;</span>,
        <span class="hljs-string">&quot;--num_workers=1&quot;</span>,
        <span class="hljs-string">&quot;--save_video&quot;</span>,
        <span class="hljs-string">&quot;--no_render&quot;</span>,
        <span class="hljs-string">&quot;--max_num_episodes=10&quot;</span>,
        <span class="hljs-string">&quot;--experiment=doom_health_gathering_supreme_2222&quot;</span>,
        <span class="hljs-string">&quot;--train_dir=train_dir&quot;</span>,
    ],
    evaluation=<span class="hljs-literal">True</span>,
)
status = enjoy(cfg)`}}),_l=new _({props:{code:"bXA0JTIwJTNEJTIwb3BlbiglMjIlMkZjb250ZW50JTJGdHJhaW5fZGlyJTJGZG9vbV9oZWFsdGhfZ2F0aGVyaW5nX3N1cHJlbWVfMjIyMiUyRnJlcGxheS5tcDQlMjIlMkMlMjAlMjJyYiUyMikucmVhZCgpJTBBZGF0YV91cmwlMjAlM0QlMjAlMjJkYXRhJTNBdmlkZW8lMkZtcDQlM0JiYXNlNjQlMkMlMjIlMjAlMkIlMjBiNjRlbmNvZGUobXA0KS5kZWNvZGUoKSUwQUhUTUwoJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTBBJTNDdmlkZW8lMjB3aWR0aCUzRDY0MCUyMGNvbnRyb2xzJTNFJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTNDc291cmNlJTIwc3JjJTNEJTIyJTI1cyUyMiUyMHR5cGUlM0QlMjJ2aWRlbyUyRm1wNCUyMiUzRSUwQSUzQyUyRnZpZGVvJTNFJTBBJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwJTI1JTIwZGF0YV91cmwlMEEp",highlighted:`mp4 = <span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;/content/train_dir/doom_health_gathering_supreme_2222/replay.mp4&quot;</span>, <span class="hljs-string">&quot;rb&quot;</span>).read()
data_url = <span class="hljs-string">&quot;data:video/mp4;base64,&quot;</span> + b64encode(mp4).decode()
HTML(
    <span class="hljs-string">&quot;&quot;&quot;
&lt;video width=640 controls&gt;
      &lt;source src=&quot;%s&quot; type=&quot;video/mp4&quot;&gt;
&lt;/video&gt;
&quot;&quot;&quot;</span>
    % data_url
)`}}),Ul=new T({}),jl=new _({props:{code:"JTIzJTIwRG93bmxvYWQlMjB0aGUlMjBhZ2VudCUyMGZyb20lMjB0aGUlMjBodWIlMEFweXRob24lMjAtbSUyMHNhbXBsZV9mYWN0b3J5Lmh1Z2dpbmdmYWNlLmxvYWRfZnJvbV9odWIlMjAtciUyMGVkYmVlY2hpbmclMkZkb29tX2RlYXRobWF0Y2hfYm90c18yMjIyJTIwLWQlMjAuJTJGdHJhaW5fZGly",highlighted:`<span class="hljs-comment"># Download the agent from the hub</span>
python -m sample_factory.huggingface.load_from_hub -r edbeeching/doom_deathmatch_bots_2222 -d ./train_dir`}}),Il=new _({props:{code:"ZnJvbSUyMHNhbXBsZV9mYWN0b3J5LmVuam95JTIwaW1wb3J0JTIwZW5qb3klMEElMEFyZWdpc3Rlcl92aXpkb29tX2NvbXBvbmVudHMoKSUwQWVudiUyMCUzRCUyMCUyMmRvb21fZGVhdGhtYXRjaF9ib3RzJTIyJTBBY2ZnJTIwJTNEJTIwcGFyc2Vfdml6ZG9vbV9jZmcoJTBBJTIwJTIwJTIwJTIwYXJndiUzRCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGYlMjItLWVudiUzRCU3QmVudiU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMi0tbnVtX3dvcmtlcnMlM0QxJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyLS1zYXZlX3ZpZGVvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyLS1ub19yZW5kZXIlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjItLW1heF9udW1fZXBpc29kZXMlM0QxJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyLS1leHBlcmltZW50JTNEZG9vbV9kZWF0aG1hdGNoX2JvdHNfMjIyMiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMi0tdHJhaW5fZGlyJTNEdHJhaW5fZGlyJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbHVhdGlvbiUzRFRydWUlMkMlMEEpJTBBc3RhdHVzJTIwJTNEJTIwZW5qb3koY2ZnKSUwQW1wNCUyMCUzRCUyMG9wZW4oJTIyJTJGY29udGVudCUyRnRyYWluX2RpciUyRmRvb21fZGVhdGhtYXRjaF9ib3RzXzIyMjIlMkZyZXBsYXkubXA0JTIyJTJDJTIwJTIycmIlMjIpLnJlYWQoKSUwQWRhdGFfdXJsJTIwJTNEJTIwJTIyZGF0YSUzQXZpZGVvJTJGbXA0JTNCYmFzZTY0JTJDJTIyJTIwJTJCJTIwYjY0ZW5jb2RlKG1wNCkuZGVjb2RlKCklMEFIVE1MKCUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMiUwQSUzQ3ZpZGVvJTIwd2lkdGglM0Q2NDAlMjBjb250cm9scyUzRSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUzQ3NvdXJjZSUyMHNyYyUzRCUyMiUyNXMlMjIlMjB0eXBlJTNEJTIydmlkZW8lMkZtcDQlMjIlM0UlMEElM0MlMkZ2aWRlbyUzRSUwQSUyMiUyMiUyMiUwQSUyMCUyMCUyMCUyMCUyNSUyMGRhdGFfdXJsJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> sample_factory.enjoy <span class="hljs-keyword">import</span> enjoy

register_vizdoom_components()
env = <span class="hljs-string">&quot;doom_deathmatch_bots&quot;</span>
cfg = parse_vizdoom_cfg(
    argv=[
        <span class="hljs-string">f&quot;--env=<span class="hljs-subst">{env}</span>&quot;</span>,
        <span class="hljs-string">&quot;--num_workers=1&quot;</span>,
        <span class="hljs-string">&quot;--save_video&quot;</span>,
        <span class="hljs-string">&quot;--no_render&quot;</span>,
        <span class="hljs-string">&quot;--max_num_episodes=1&quot;</span>,
        <span class="hljs-string">&quot;--experiment=doom_deathmatch_bots_2222&quot;</span>,
        <span class="hljs-string">&quot;--train_dir=train_dir&quot;</span>,
    ],
    evaluation=<span class="hljs-literal">True</span>,
)
status = enjoy(cfg)
mp4 = <span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;/content/train_dir/doom_deathmatch_bots_2222/replay.mp4&quot;</span>, <span class="hljs-string">&quot;rb&quot;</span>).read()
data_url = <span class="hljs-string">&quot;data:video/mp4;base64,&quot;</span> + b64encode(mp4).decode()
HTML(
    <span class="hljs-string">&quot;&quot;&quot;
&lt;video width=640 controls&gt;
      &lt;source src=&quot;%s&quot; type=&quot;video/mp4&quot;&gt;
&lt;/video&gt;
&quot;&quot;&quot;</span>
    % data_url
)`}}),kl=new T({}),{c(){R=o("meta"),_r=c(),C=o("h1"),he=o("a"),mo=o("span"),m(Oe.$$.fragment),li=c(),uo=o("span"),oi=s("Hands-on: advanced Deep Reinforcement Learning. Using Sample Factory to play Doom from pixels"),Ur=c(),m(Ke.$$.fragment),jr=c(),et=o("p"),ai=s(`The colab notebook:
`),tt=o("a"),Zl=o("img"),Ir=c(),V=o("h1"),ce=o("a"),yo=o("span"),m(lt.$$.fragment),ri=c(),vo=o("span"),si=s("Unit 8 Part 2: Advanced Deep Reinforcement Learning. Using Sample Factory to play Doom from pixels"),Er=c(),ot=o("img"),kr=c(),fe=o("p"),ni=s("In this notebook, we will learn how to train a Deep Neural Network to collect objects in a 3D environment based on the game of Doom, a video of the resulting policy is shown below. We train this policy using "),at=o("a"),ii=s("Sample Factory"),pi=s(", an asynchronous implementation of the PPO algorithm."),Zr=c(),Gl=o("p"),hi=s("Please note the following points:"),Gr=c(),k=o("ul"),wo=o("li"),me=o("p"),rt=o("a"),ci=s("Sample Factory"),fi=s(" is an advanced RL framework and "),Mo=o("strong"),mi=s("only functions on Linux and Mac"),ui=s(" (not Windows)."),di=c(),go=o("li"),E=o("p"),yi=s("The framework performs best on a "),bo=o("strong"),vi=s("GPU machine with many CPU cores"),wi=s(", where it can achieve speeds of 100k interactions per second. The resources available on a standard Colab notebook "),Jo=o("strong"),Mi=s("limit the performance of this library"),gi=s(". So the speed in this setting "),To=o("strong"),bi=s("does not reflect the real-world performance"),Ji=s("."),Ti=c(),_o=o("li"),st=o("p"),_i=s("Benchmarks for Sample Factory are available in a number of settings, check out the "),nt=o("a"),Ui=s("examples"),ji=s(" if you want to find out more."),$r=c(),m(it.$$.fragment),Br=c(),ue=o("p"),Ii=s("To validate this hands-on for the "),pt=o("a"),Ei=s("certification process"),ki=s(", you need to push one model:"),Sr=c(),$l=o("ul"),Bl=o("li"),Uo=o("code"),Zi=s("doom_health_gathering_supreme"),Gi=s(" get a result of >= 5."),Wr=c(),X=o("p"),$i=s("To find your result, go to the "),ht=o("a"),Bi=s("leaderboard"),Si=s(" and find your model, "),jo=o("strong"),Wi=s("the result = mean_reward - std of reward"),Ar=c(),ct=o("p"),Ai=s("If you don\u2019t find your model, "),Io=o("strong"),Ni=s("go to the bottom of the page and click on the refresh button"),Nr=c(),ft=o("p"),Ri=s("For more information about the certification process, check this section \u{1F449} "),mt=o("a"),Ci=s("https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),Rr=c(),z=o("h2"),de=o("a"),Eo=o("span"),m(ut.$$.fragment),Vi=c(),ko=o("span"),Xi=s("Set the GPU \u{1F4AA}"),Cr=c(),Sl=o("ul"),ye=o("li"),zi=s("To "),Zo=o("strong"),qi=s("accelerate the agent\u2019s training, we\u2019ll use a GPU"),Hi=s(". To do that, go to "),Go=o("code"),Fi=s("Runtime > Change Runtime type"),Vr=c(),dt=o("img"),Xr=c(),Wl=o("ul"),$o=o("li"),Bo=o("code"),Yi=s("Hardware Accelerator > GPU"),zr=c(),yt=o("img"),qr=c(),ve=o("p"),Li=s("Before starting to train our agent, let\u2019s "),So=o("strong"),Pi=s("study the library and environments we\u2019re going to use"),Qi=s("."),Hr=c(),q=o("h2"),we=o("a"),Wo=o("span"),m(vt.$$.fragment),Di=c(),Ao=o("span"),xi=s("Sample Factory"),Fr=c(),H=o("p"),wt=o("a"),Oi=s("Sample Factory"),Ki=s(" is one of the "),No=o("strong"),ep=s("fastest RL libraries focused on very efficient synchronous and asynchronous implementations of policy gradients (PPO)"),tp=s("."),Yr=c(),Z=o("p"),lp=s("Sample Factory is thoroughly "),Ro=o("strong"),op=s("tested, used by many researchers and practitioners"),ap=s(", and is actively maintained. Our implementation is known to "),Co=o("strong"),rp=s("reach SOTA performance in a variety of domains while minimizing RL experiment training time and hardware requirements"),sp=s("."),Lr=c(),Mt=o("img"),Pr=c(),F=o("h3"),Me=o("a"),Vo=o("span"),m(gt.$$.fragment),np=c(),Xo=o("span"),ip=s("Key features"),Qr=c(),M=o("ul"),bt=o("li"),pp=s("Highly optimized algorithm\xA0"),Jt=o("a"),hp=s("architecture"),cp=s("\xA0for maximum learning throughput"),fp=c(),Al=o("li"),Tt=o("a"),mp=s("Synchronous and asynchronous"),up=s("\xA0training regimes"),dp=c(),Nl=o("li"),_t=o("a"),yp=s("Serial (single-process) mode"),vp=s("\xA0for easy debugging"),wp=c(),Rl=o("li"),Mp=s("Optimal performance in both CPU-based and\xA0"),Ut=o("a"),gp=s("GPU-accelerated environments"),bp=c(),jt=o("li"),Jp=s("Single- & multi-agent training, self-play, supports\xA0"),It=o("a"),Tp=s("training multiple policies"),_p=s("\xA0at once on one or many GPUs"),Up=c(),Et=o("li"),jp=s("Population-Based Training ("),kt=o("a"),Ip=s("PBT"),Ep=s(")"),kp=c(),zo=o("li"),Zp=s("Discrete, continuous, hybrid action spaces"),Gp=c(),qo=o("li"),$p=s("Vector-based, image-based, dictionary observation spaces"),Bp=c(),Cl=o("li"),Sp=s("Automatically creates a model architecture by parsing action/observation space specification. Supports\xA0"),Zt=o("a"),Wp=s("custom model architectures"),Ap=c(),Gt=o("li"),Np=s("Designed to be imported into other projects,\xA0"),$t=o("a"),Rp=s("custom environments"),Cp=s("\xA0are first-class citizens"),Vp=c(),ge=o("li"),Xp=s("Detailed\xA0"),Bt=o("a"),zp=s("WandB and Tensorboard summaries"),qp=s(",\xA0"),St=o("a"),Hp=s("custom metrics"),Fp=c(),Vl=o("li"),Wt=o("a"),Yp=s("HuggingFace \u{1F917} integration"),Lp=s("\xA0(upload trained models and metrics to the Hub)"),Pp=c(),U=o("li"),At=o("a"),Qp=s("Multiple"),Dp=s("\xA0"),Nt=o("a"),xp=s("example"),Op=s("\xA0"),Rt=o("a"),Kp=s("environment"),eh=s("\xA0"),Ct=o("a"),th=s("integrations"),lh=s("\xA0with tuned parameters and trained models"),Dr=c(),Vt=o("p"),oh=s("All of the above policies are available on the \u{1F917} hub. Search for the tag "),Xt=o("a"),ah=s("sample-factory"),xr=c(),Y=o("h3"),be=o("a"),Ho=o("span"),m(zt.$$.fragment),rh=c(),Fo=o("span"),sh=s("How sample-factory works"),Or=c(),Je=o("p"),nh=s("Sample-factory is one of the "),Yo=o("strong"),ih=s("most highly optimized RL implementations available to the community"),ph=s("."),Kr=c(),Te=o("p"),hh=s("It works by "),Lo=o("strong"),ch=s("spawning multiple processes that run rollout workers, inference workers and a learner worker"),fh=s("."),es=c(),G=o("p"),mh=s("The "),Po=o("em"),uh=s("workers"),dh=c(),Qo=o("strong"),yh=s("communicate through shared memory, which lowers the communication cost between processes"),vh=s("."),ts=c(),$=o("p"),wh=s("The "),Do=o("em"),Mh=s("rollout workers"),gh=s(" interact with the environment and send observations to the "),xo=o("em"),bh=s("inference workers"),Jh=s("."),ls=c(),B=o("p"),Th=s("The "),Oo=o("em"),_h=s("inferences workers"),Uh=s(" query a fixed version of the policy and "),Ko=o("strong"),jh=s("send actions back to the rollout worker"),Ih=s("."),os=c(),S=o("p"),Eh=s("After "),ea=o("em"),kh=s("k"),Zh=s(" steps the rollout works send a trajectory of experience to the learner worker, "),ta=o("strong"),Gh=s("which it uses to update the agent\u2019s policy network"),$h=s("."),as=c(),qt=o("img"),rs=c(),L=o("h3"),_e=o("a"),la=o("span"),m(Ht.$$.fragment),Bh=c(),oa=o("span"),Sh=s("Actor Critic models in Sample-factory"),ss=c(),Xl=o("p"),Wh=s("Actor Critic models in Sample Factory are composed of three components:"),ns=c(),W=o("ul"),zl=o("li"),aa=o("strong"),Ah=s("Encoder"),Nh=s(" - Process input observations (images, vectors) and map them to a vector. This is the part of the model you will most likely want to customize."),Rh=c(),ql=o("li"),ra=o("strong"),Ch=s("Core"),Vh=s(" - Intergrate vectors from one or more encoders, can optionally include a single- or multi-layer LSTM/GRU in a memory-based agent."),Xh=c(),Hl=o("li"),sa=o("strong"),zh=s("Decoder"),qh=s(" - Apply additional layers to the output of the model core before computing the policy and value outputs."),is=c(),Ue=o("p"),Hh=s("The library has been designed to automatically support any observation and action spaces. Users can easily add their custom models. You can find out more in the "),Ft=o("a"),Fh=s("documentation"),Yh=s("."),ps=c(),P=o("h2"),je=o("a"),na=o("span"),m(Yt.$$.fragment),Lh=c(),ia=o("span"),Ph=s("ViZDoom"),hs=c(),Q=o("p"),Lt=o("a"),Qh=s("ViZDoom"),Dh=s(" is an "),pa=o("strong"),xh=s("open-source python interface for the Doom Engine"),Oh=s("."),cs=c(),Fl=o("p"),Kh=s("The library was created in 2016 by Marek Wydmuch, Michal Kempka  at the\xA0Institute of Computing Science, Poznan University of Technology, Poland."),fs=c(),A=o("p"),ec=s("The library enables the "),ha=o("strong"),tc=s("training of agents directly from the screen pixels in a number of scenarios"),lc=s(", including team deathmatch, shown in the video below. Because the ViZDoom environment is based on a game the was created in the 90s, it can be run on modern hardware at accelerated speeds, "),ca=o("strong"),oc=s("allowing us to learn complex AI behaviors fairly quickly"),ac=s("."),ms=c(),Yl=o("p"),rc=s("The library includes feature such as:"),us=c(),g=o("ul"),fa=o("li"),sc=s("Multi-platform (Linux, macOS, Windows),"),nc=c(),ma=o("li"),ic=s("API for Python and C++,"),pc=c(),Ll=o("li"),Pt=o("a"),hc=s("OpenAI Gym"),cc=s("\xA0environment wrappers"),fc=c(),ua=o("li"),mc=s("Easy-to-create custom scenarios (visual editors, scripting language, and examples available),"),uc=c(),da=o("li"),dc=s("Async and sync single-player and multiplayer modes,"),yc=c(),ya=o("li"),vc=s("Lightweight (few MBs) and fast (up to 7000 fps in sync mode, single-threaded),"),wc=c(),va=o("li"),Mc=s("Customizable resolution and rendering parameters,"),gc=c(),wa=o("li"),bc=s("Access to the depth buffer (3D vision),"),Jc=c(),Ma=o("li"),Tc=s("Automatic labeling of game objects visible in the frame,"),_c=c(),ga=o("li"),Uc=s("Access to the audio buffer"),jc=c(),ba=o("li"),Ic=s("Access to the list of actors/objects and map geometry,"),Ec=c(),Ja=o("li"),kc=s("Off-screen rendering and episode recording,"),Zc=c(),Ta=o("li"),Gc=s("Time scaling in async mode."),ds=c(),D=o("h2"),Ie=o("a"),_a=o("span"),m(Qt.$$.fragment),$c=c(),Ua=o("span"),Bc=s("We first need to install some dependencies that are required for the ViZDoom environment"),ys=c(),Pl=o("p"),Sc=s("Now that our Colab runtime is set up, we can start by installing the dependencies required to run ViZDoom on linux."),vs=c(),Ee=o("p"),Wc=s("If you are following on your machine on Mac, you will want to follow the installation instructions on the "),Dt=o("a"),Ac=s("github page"),Nc=s("."),ws=c(),m(xt.$$.fragment),Ms=c(),x=o("h2"),ke=o("a"),ja=o("span"),m(Ot.$$.fragment),Rc=c(),Ia=o("span"),Cc=s("Then we can install Sample Factory and ViZDoom"),gs=c(),Ql=o("ul"),Ea=o("li"),Vc=s("This can take 7min"),bs=c(),m(Kt.$$.fragment),Js=c(),O=o("h2"),Ze=o("a"),ka=o("span"),m(el.$$.fragment),Xc=c(),Za=o("span"),zc=s("Setting up the Doom Environment in sample-factory"),Ts=c(),m(tl.$$.fragment),_s=c(),Ge=o("p"),qc=s("Now that the setup if complete, we can train the agent. We have chosen here to learn a ViZDoom task called "),Ga=o("code"),Hc=s("Health Gathering Supreme"),Fc=s("."),Us=c(),K=o("h3"),$e=o("a"),$a=o("span"),m(ll.$$.fragment),Yc=c(),Ba=o("span"),Lc=s("The scenario: Health Gathering Supreme"),js=c(),ol=o("img"),Is=c(),j=o("p"),Pc=s("The objective of this scenario is to "),Sa=o("strong"),Qc=s("teach the agent how to survive without knowing what makes it survive"),Dc=s(". The Agent know only that "),Wa=o("strong"),xc=s("life is precious"),Oc=s(" and death is bad so "),Aa=o("strong"),Kc=s("it must learn what prolongs its existence and that its health is connected with survival"),ef=s("."),Es=c(),N=o("p"),tf=s("The map is a rectangle containing walls and with a green, acidic floor which "),Na=o("strong"),lf=s("hurts the player periodically"),of=s(". Initially there are some medkits spread uniformly over the map. A new medkit falls from the skies every now and then. "),Ra=o("strong"),af=s("Medkits heal some portions of player\u2019s health"),rf=s(" - to survive, the agent needs to pick them up. The episode finishes after the player\u2019s death or on timeout."),ks=c(),Dl=o("p"),sf=s("Further configuration:"),Zs=c(),I=o("ul"),Ca=o("li"),nf=s("Living_reward = 1"),pf=c(),Va=o("li"),hf=s("3 available buttons: turn left, turn right, move forward"),cf=c(),Xa=o("li"),ff=s("1 available game variable: HEALTH"),mf=c(),za=o("li"),uf=s("death penalty = 100"),Gs=c(),Be=o("p"),df=s("You can find out more about the scenarios available in ViZDoom "),al=o("a"),yf=s("here"),vf=s("."),$s=c(),Se=o("p"),wf=s("There are also a number of more complex scenarios that have been create for ViZDoom, such as the ones detailed on "),rl=o("a"),Mf=s("this github page"),gf=s("."),Bs=c(),ee=o("h2"),We=o("a"),qa=o("span"),m(sl.$$.fragment),bf=c(),Ha=o("span"),Jf=s("Training the agent"),Ss=c(),xl=o("ul"),Fa=o("li"),Tf=s("We\u2019re going to train the agent for 4000000 steps. It will take approximately 20min"),Ws=c(),m(nl.$$.fragment),As=c(),te=o("h2"),Ae=o("a"),Ya=o("span"),m(il.$$.fragment),_f=c(),La=o("span"),Uf=s("Let's take a look at the performance of the trained policy and output a video of the agent."),Ns=c(),m(pl.$$.fragment),Rs=c(),le=o("h2"),Ne=o("a"),Pa=o("span"),m(hl.$$.fragment),jf=c(),Qa=o("span"),If=s("Now lets visualize the performance of the agent"),Cs=c(),m(cl.$$.fragment),Vs=c(),Ol=o("p"),Ef=s("The agent has learned something, but its performance could be better. We would clearly need to train for longer. But let\u2019s upload this model to the Hub."),Xs=c(),oe=o("h2"),Re=o("a"),Da=o("span"),m(fl.$$.fragment),kf=c(),xa=o("span"),Zf=s("Now lets upload your checkpoint and video to the Hugging Face Hub"),zs=c(),Kl=o("p"),Gf=s("To be able to share your model with the community there are three more steps to follow:"),qs=c(),ml=o("p"),$f=s("1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),ul=o("a"),Bf=s("https://huggingface.co/join"),Hs=c(),eo=o("p"),Sf=s("2\uFE0F\u20E3 Sign in and get your authentication token from the Hugging Face website."),Fs=c(),to=o("ul"),Ce=o("li"),Wf=s("Create a new token ("),dl=o("a"),Af=s("https://huggingface.co/settings/tokens"),Nf=s(") "),Oa=o("strong"),Rf=s("with write role"),Ys=c(),yl=o("img"),Ls=c(),Ve=o("ul"),Ka=o("li"),Cf=s("Copy the token"),Vf=c(),er=o("li"),Xf=s("Run the cell below and paste the token"),Ps=c(),vl=o("p"),zf=s("If you don\u2019t want to use Google Colab or a Jupyter Notebook, you need to use this command instead: "),tr=o("code"),qf=s("huggingface-cli login"),Qs=c(),m(wl.$$.fragment),Ds=c(),m(Ml.$$.fragment),xs=c(),ae=o("h2"),Xe=o("a"),lr=o("span"),m(gl.$$.fragment),Hf=c(),or=o("span"),Ff=s("Let's load another model"),Os=c(),lo=o("p"),Yf=s("This agent\u2019s performance was good, but we can do better! Let\u2019s download and visualize an agent trained for 10B timesteps from the hub."),Ks=c(),m(bl.$$.fragment),en=c(),m(Jl.$$.fragment),tn=c(),m(Tl.$$.fragment),ln=c(),m(_l.$$.fragment),on=c(),re=o("h2"),ze=o("a"),ar=o("span"),m(Ul.$$.fragment),Lf=c(),rr=o("span"),Pf=s("Some additional challenges \u{1F3C6}: Doom Deathmatch"),an=c(),qe=o("p"),Qf=s("Training an agent to play a Doom deathmatch "),sr=o("strong"),Df=s("takes many hours on a more beefy machine than is available in Colab"),xf=s("."),rn=c(),He=o("p"),Of=s("Fortunately, we have have "),nr=o("strong"),Kf=s("already trained an agent in this scenario and it is available in the \u{1F917} Hub!"),em=s(" Let\u2019s download the model and visualize the agent\u2019s performance."),sn=c(),m(jl.$$.fragment),nn=c(),Fe=o("p"),tm=s("Given the agent plays for a long time the video generation can take "),ir=o("strong"),lm=s("10 minutes"),om=s("."),pn=c(),m(Il.$$.fragment),hn=c(),se=o("p"),am=s("You "),pr=o("strong"),rm=s("can try to train your agent in this environment"),sm=s(` using the code above, but not on colab.
`),hr=o("strong"),nm=s("Good luck \u{1F91E}"),cn=c(),El=o("p"),im=s("If you prefer an easier scenario, "),ne=o("strong"),pm=s("why not try training in another ViZDoom scenario such as "),cr=o("code"),hm=s("doom_deadly_corridor"),cm=s(" or "),fr=o("code"),fm=s("doom_defend_the_center"),mm=s("."),fn=c(),mn=o("hr"),un=c(),Ye=o("p"),um=s("This concludes the last unit. But we are not finished yet! \u{1F917} The following "),mr=o("strong"),dm=s("bonus section include some of the most interesting, advanced, and cutting edge work in Deep Reinforcement Learning"),ym=s("."),dn=c(),ie=o("h2"),Le=o("a"),ur=o("span"),m(kl.$$.fragment),vm=c(),dr=o("span"),wm=s("Keep learning, stay awesome \u{1F917}"),this.h()},l(e){const i=pv('[data-svelte="svelte-1phssyn"]',document.head);R=a(i,"META",{name:!0,content:!0}),i.forEach(l),_r=f(e),C=a(e,"H1",{class:!0});var vn=r(C);he=a(vn,"A",{id:!0,class:!0,href:!0});var Fm=r(he);mo=a(Fm,"SPAN",{});var Ym=r(mo);u(Oe.$$.fragment,Ym),Ym.forEach(l),Fm.forEach(l),li=f(vn),uo=a(vn,"SPAN",{});var Lm=r(uo);oi=n(Lm,"Hands-on: advanced Deep Reinforcement Learning. Using Sample Factory to play Doom from pixels"),Lm.forEach(l),vn.forEach(l),Ur=f(e),u(Ke.$$.fragment,e),jr=f(e),et=a(e,"P",{});var Mm=r(et);ai=n(Mm,`The colab notebook:
`),tt=a(Mm,"A",{href:!0,rel:!0});var Pm=r(tt);Zl=a(Pm,"IMG",{src:!0,alt:!0}),Pm.forEach(l),Mm.forEach(l),Ir=f(e),V=a(e,"H1",{class:!0});var wn=r(V);ce=a(wn,"A",{id:!0,class:!0,href:!0});var Qm=r(ce);yo=a(Qm,"SPAN",{});var Dm=r(yo);u(lt.$$.fragment,Dm),Dm.forEach(l),Qm.forEach(l),ri=f(wn),vo=a(wn,"SPAN",{});var xm=r(vo);si=n(xm,"Unit 8 Part 2: Advanced Deep Reinforcement Learning. Using Sample Factory to play Doom from pixels"),xm.forEach(l),wn.forEach(l),Er=f(e),ot=a(e,"IMG",{src:!0,alt:!0}),kr=f(e),fe=a(e,"P",{});var Mn=r(fe);ni=n(Mn,"In this notebook, we will learn how to train a Deep Neural Network to collect objects in a 3D environment based on the game of Doom, a video of the resulting policy is shown below. We train this policy using "),at=a(Mn,"A",{href:!0,rel:!0});var Om=r(at);ii=n(Om,"Sample Factory"),Om.forEach(l),pi=n(Mn,", an asynchronous implementation of the PPO algorithm."),Mn.forEach(l),Zr=f(e),Gl=a(e,"P",{});var Km=r(Gl);hi=n(Km,"Please note the following points:"),Km.forEach(l),Gr=f(e),k=a(e,"UL",{});var oo=r(k);wo=a(oo,"LI",{});var eu=r(wo);me=a(eu,"P",{});var yr=r(me);rt=a(yr,"A",{href:!0,rel:!0});var tu=r(rt);ci=n(tu,"Sample Factory"),tu.forEach(l),fi=n(yr," is an advanced RL framework and "),Mo=a(yr,"STRONG",{});var lu=r(Mo);mi=n(lu,"only functions on Linux and Mac"),lu.forEach(l),ui=n(yr," (not Windows)."),yr.forEach(l),eu.forEach(l),di=f(oo),go=a(oo,"LI",{});var ou=r(go);E=a(ou,"P",{});var Pe=r(E);yi=n(Pe,"The framework performs best on a "),bo=a(Pe,"STRONG",{});var au=r(bo);vi=n(au,"GPU machine with many CPU cores"),au.forEach(l),wi=n(Pe,", where it can achieve speeds of 100k interactions per second. The resources available on a standard Colab notebook "),Jo=a(Pe,"STRONG",{});var ru=r(Jo);Mi=n(ru,"limit the performance of this library"),ru.forEach(l),gi=n(Pe,". So the speed in this setting "),To=a(Pe,"STRONG",{});var su=r(To);bi=n(su,"does not reflect the real-world performance"),su.forEach(l),Ji=n(Pe,"."),Pe.forEach(l),ou.forEach(l),Ti=f(oo),_o=a(oo,"LI",{});var nu=r(_o);st=a(nu,"P",{});var gn=r(st);_i=n(gn,"Benchmarks for Sample Factory are available in a number of settings, check out the "),nt=a(gn,"A",{href:!0,rel:!0});var iu=r(nt);Ui=n(iu,"examples"),iu.forEach(l),ji=n(gn," if you want to find out more."),gn.forEach(l),nu.forEach(l),oo.forEach(l),$r=f(e),u(it.$$.fragment,e),Br=f(e),ue=a(e,"P",{});var bn=r(ue);Ii=n(bn,"To validate this hands-on for the "),pt=a(bn,"A",{href:!0,rel:!0});var pu=r(pt);Ei=n(pu,"certification process"),pu.forEach(l),ki=n(bn,", you need to push one model:"),bn.forEach(l),Sr=f(e),$l=a(e,"UL",{});var hu=r($l);Bl=a(hu,"LI",{});var gm=r(Bl);Uo=a(gm,"CODE",{});var cu=r(Uo);Zi=n(cu,"doom_health_gathering_supreme"),cu.forEach(l),Gi=n(gm," get a result of >= 5."),gm.forEach(l),hu.forEach(l),Wr=f(e),X=a(e,"P",{});var vr=r(X);$i=n(vr,"To find your result, go to the "),ht=a(vr,"A",{href:!0,rel:!0});var fu=r(ht);Bi=n(fu,"leaderboard"),fu.forEach(l),Si=n(vr," and find your model, "),jo=a(vr,"STRONG",{});var mu=r(jo);Wi=n(mu,"the result = mean_reward - std of reward"),mu.forEach(l),vr.forEach(l),Ar=f(e),ct=a(e,"P",{});var bm=r(ct);Ai=n(bm,"If you don\u2019t find your model, "),Io=a(bm,"STRONG",{});var uu=r(Io);Ni=n(uu,"go to the bottom of the page and click on the refresh button"),uu.forEach(l),bm.forEach(l),Nr=f(e),ft=a(e,"P",{});var Jm=r(ft);Ri=n(Jm,"For more information about the certification process, check this section \u{1F449} "),mt=a(Jm,"A",{href:!0,rel:!0});var du=r(mt);Ci=n(du,"https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),du.forEach(l),Jm.forEach(l),Rr=f(e),z=a(e,"H2",{class:!0});var Jn=r(z);de=a(Jn,"A",{id:!0,class:!0,href:!0});var yu=r(de);Eo=a(yu,"SPAN",{});var vu=r(Eo);u(ut.$$.fragment,vu),vu.forEach(l),yu.forEach(l),Vi=f(Jn),ko=a(Jn,"SPAN",{});var wu=r(ko);Xi=n(wu,"Set the GPU \u{1F4AA}"),wu.forEach(l),Jn.forEach(l),Cr=f(e),Sl=a(e,"UL",{});var Mu=r(Sl);ye=a(Mu,"LI",{});var wr=r(ye);zi=n(wr,"To "),Zo=a(wr,"STRONG",{});var gu=r(Zo);qi=n(gu,"accelerate the agent\u2019s training, we\u2019ll use a GPU"),gu.forEach(l),Hi=n(wr,". To do that, go to "),Go=a(wr,"CODE",{});var bu=r(Go);Fi=n(bu,"Runtime > Change Runtime type"),bu.forEach(l),wr.forEach(l),Mu.forEach(l),Vr=f(e),dt=a(e,"IMG",{src:!0,alt:!0}),Xr=f(e),Wl=a(e,"UL",{});var Ju=r(Wl);$o=a(Ju,"LI",{});var Tu=r($o);Bo=a(Tu,"CODE",{});var _u=r(Bo);Yi=n(_u,"Hardware Accelerator > GPU"),_u.forEach(l),Tu.forEach(l),Ju.forEach(l),zr=f(e),yt=a(e,"IMG",{src:!0,alt:!0}),qr=f(e),ve=a(e,"P",{});var Tn=r(ve);Li=n(Tn,"Before starting to train our agent, let\u2019s "),So=a(Tn,"STRONG",{});var Uu=r(So);Pi=n(Uu,"study the library and environments we\u2019re going to use"),Uu.forEach(l),Qi=n(Tn,"."),Tn.forEach(l),Hr=f(e),q=a(e,"H2",{class:!0});var _n=r(q);we=a(_n,"A",{id:!0,class:!0,href:!0});var ju=r(we);Wo=a(ju,"SPAN",{});var Iu=r(Wo);u(vt.$$.fragment,Iu),Iu.forEach(l),ju.forEach(l),Di=f(_n),Ao=a(_n,"SPAN",{});var Eu=r(Ao);xi=n(Eu,"Sample Factory"),Eu.forEach(l),_n.forEach(l),Fr=f(e),H=a(e,"P",{});var Mr=r(H);wt=a(Mr,"A",{href:!0,rel:!0});var ku=r(wt);Oi=n(ku,"Sample Factory"),ku.forEach(l),Ki=n(Mr," is one of the "),No=a(Mr,"STRONG",{});var Zu=r(No);ep=n(Zu,"fastest RL libraries focused on very efficient synchronous and asynchronous implementations of policy gradients (PPO)"),Zu.forEach(l),tp=n(Mr,"."),Mr.forEach(l),Yr=f(e),Z=a(e,"P",{});var ao=r(Z);lp=n(ao,"Sample Factory is thoroughly "),Ro=a(ao,"STRONG",{});var Gu=r(Ro);op=n(Gu,"tested, used by many researchers and practitioners"),Gu.forEach(l),ap=n(ao,", and is actively maintained. Our implementation is known to "),Co=a(ao,"STRONG",{});var $u=r(Co);rp=n($u,"reach SOTA performance in a variety of domains while minimizing RL experiment training time and hardware requirements"),$u.forEach(l),sp=n(ao,"."),ao.forEach(l),Lr=f(e),Mt=a(e,"IMG",{src:!0,alt:!0}),Pr=f(e),F=a(e,"H3",{class:!0});var Un=r(F);Me=a(Un,"A",{id:!0,class:!0,href:!0});var Bu=r(Me);Vo=a(Bu,"SPAN",{});var Su=r(Vo);u(gt.$$.fragment,Su),Su.forEach(l),Bu.forEach(l),np=f(Un),Xo=a(Un,"SPAN",{});var Wu=r(Xo);ip=n(Wu,"Key features"),Wu.forEach(l),Un.forEach(l),Qr=f(e),M=a(e,"UL",{});var b=r(M);bt=a(b,"LI",{});var jn=r(bt);pp=n(jn,"Highly optimized algorithm\xA0"),Jt=a(jn,"A",{href:!0,rel:!0});var Au=r(Jt);hp=n(Au,"architecture"),Au.forEach(l),cp=n(jn,"\xA0for maximum learning throughput"),jn.forEach(l),fp=f(b),Al=a(b,"LI",{});var Tm=r(Al);Tt=a(Tm,"A",{href:!0,rel:!0});var Nu=r(Tt);mp=n(Nu,"Synchronous and asynchronous"),Nu.forEach(l),up=n(Tm,"\xA0training regimes"),Tm.forEach(l),dp=f(b),Nl=a(b,"LI",{});var _m=r(Nl);_t=a(_m,"A",{href:!0,rel:!0});var Ru=r(_t);yp=n(Ru,"Serial (single-process) mode"),Ru.forEach(l),vp=n(_m,"\xA0for easy debugging"),_m.forEach(l),wp=f(b),Rl=a(b,"LI",{});var Um=r(Rl);Mp=n(Um,"Optimal performance in both CPU-based and\xA0"),Ut=a(Um,"A",{href:!0,rel:!0});var Cu=r(Ut);gp=n(Cu,"GPU-accelerated environments"),Cu.forEach(l),Um.forEach(l),bp=f(b),jt=a(b,"LI",{});var In=r(jt);Jp=n(In,"Single- & multi-agent training, self-play, supports\xA0"),It=a(In,"A",{href:!0,rel:!0});var Vu=r(It);Tp=n(Vu,"training multiple policies"),Vu.forEach(l),_p=n(In,"\xA0at once on one or many GPUs"),In.forEach(l),Up=f(b),Et=a(b,"LI",{});var En=r(Et);jp=n(En,"Population-Based Training ("),kt=a(En,"A",{href:!0,rel:!0});var Xu=r(kt);Ip=n(Xu,"PBT"),Xu.forEach(l),Ep=n(En,")"),En.forEach(l),kp=f(b),zo=a(b,"LI",{});var zu=r(zo);Zp=n(zu,"Discrete, continuous, hybrid action spaces"),zu.forEach(l),Gp=f(b),qo=a(b,"LI",{});var qu=r(qo);$p=n(qu,"Vector-based, image-based, dictionary observation spaces"),qu.forEach(l),Bp=f(b),Cl=a(b,"LI",{});var jm=r(Cl);Sp=n(jm,"Automatically creates a model architecture by parsing action/observation space specification. Supports\xA0"),Zt=a(jm,"A",{href:!0,rel:!0});var Hu=r(Zt);Wp=n(Hu,"custom model architectures"),Hu.forEach(l),jm.forEach(l),Ap=f(b),Gt=a(b,"LI",{});var kn=r(Gt);Np=n(kn,"Designed to be imported into other projects,\xA0"),$t=a(kn,"A",{href:!0,rel:!0});var Fu=r($t);Rp=n(Fu,"custom environments"),Fu.forEach(l),Cp=n(kn,"\xA0are first-class citizens"),kn.forEach(l),Vp=f(b),ge=a(b,"LI",{});var gr=r(ge);Xp=n(gr,"Detailed\xA0"),Bt=a(gr,"A",{href:!0,rel:!0});var Yu=r(Bt);zp=n(Yu,"WandB and Tensorboard summaries"),Yu.forEach(l),qp=n(gr,",\xA0"),St=a(gr,"A",{href:!0,rel:!0});var Lu=r(St);Hp=n(Lu,"custom metrics"),Lu.forEach(l),gr.forEach(l),Fp=f(b),Vl=a(b,"LI",{});var Im=r(Vl);Wt=a(Im,"A",{href:!0,rel:!0});var Pu=r(Wt);Yp=n(Pu,"HuggingFace \u{1F917} integration"),Pu.forEach(l),Lp=n(Im,"\xA0(upload trained models and metrics to the Hub)"),Im.forEach(l),Pp=f(b),U=a(b,"LI",{});var pe=r(U);At=a(pe,"A",{href:!0,rel:!0});var Qu=r(At);Qp=n(Qu,"Multiple"),Qu.forEach(l),Dp=n(pe,"\xA0"),Nt=a(pe,"A",{href:!0,rel:!0});var Du=r(Nt);xp=n(Du,"example"),Du.forEach(l),Op=n(pe,"\xA0"),Rt=a(pe,"A",{href:!0,rel:!0});var xu=r(Rt);Kp=n(xu,"environment"),xu.forEach(l),eh=n(pe,"\xA0"),Ct=a(pe,"A",{href:!0,rel:!0});var Ou=r(Ct);th=n(Ou,"integrations"),Ou.forEach(l),lh=n(pe,"\xA0with tuned parameters and trained models"),pe.forEach(l),b.forEach(l),Dr=f(e),Vt=a(e,"P",{});var Em=r(Vt);oh=n(Em,"All of the above policies are available on the \u{1F917} hub. Search for the tag "),Xt=a(Em,"A",{href:!0,rel:!0});var Ku=r(Xt);ah=n(Ku,"sample-factory"),Ku.forEach(l),Em.forEach(l),xr=f(e),Y=a(e,"H3",{class:!0});var Zn=r(Y);be=a(Zn,"A",{id:!0,class:!0,href:!0});var ed=r(be);Ho=a(ed,"SPAN",{});var td=r(Ho);u(zt.$$.fragment,td),td.forEach(l),ed.forEach(l),rh=f(Zn),Fo=a(Zn,"SPAN",{});var ld=r(Fo);sh=n(ld,"How sample-factory works"),ld.forEach(l),Zn.forEach(l),Or=f(e),Je=a(e,"P",{});var Gn=r(Je);nh=n(Gn,"Sample-factory is one of the "),Yo=a(Gn,"STRONG",{});var od=r(Yo);ih=n(od,"most highly optimized RL implementations available to the community"),od.forEach(l),ph=n(Gn,"."),Gn.forEach(l),Kr=f(e),Te=a(e,"P",{});var $n=r(Te);hh=n($n,"It works by "),Lo=a($n,"STRONG",{});var ad=r(Lo);ch=n(ad,"spawning multiple processes that run rollout workers, inference workers and a learner worker"),ad.forEach(l),fh=n($n,"."),$n.forEach(l),es=f(e),G=a(e,"P",{});var ro=r(G);mh=n(ro,"The "),Po=a(ro,"EM",{});var rd=r(Po);uh=n(rd,"workers"),rd.forEach(l),dh=f(ro),Qo=a(ro,"STRONG",{});var sd=r(Qo);yh=n(sd,"communicate through shared memory, which lowers the communication cost between processes"),sd.forEach(l),vh=n(ro,"."),ro.forEach(l),ts=f(e),$=a(e,"P",{});var so=r($);wh=n(so,"The "),Do=a(so,"EM",{});var nd=r(Do);Mh=n(nd,"rollout workers"),nd.forEach(l),gh=n(so," interact with the environment and send observations to the "),xo=a(so,"EM",{});var id=r(xo);bh=n(id,"inference workers"),id.forEach(l),Jh=n(so,"."),so.forEach(l),ls=f(e),B=a(e,"P",{});var no=r(B);Th=n(no,"The "),Oo=a(no,"EM",{});var pd=r(Oo);_h=n(pd,"inferences workers"),pd.forEach(l),Uh=n(no," query a fixed version of the policy and "),Ko=a(no,"STRONG",{});var hd=r(Ko);jh=n(hd,"send actions back to the rollout worker"),hd.forEach(l),Ih=n(no,"."),no.forEach(l),os=f(e),S=a(e,"P",{});var io=r(S);Eh=n(io,"After "),ea=a(io,"EM",{});var cd=r(ea);kh=n(cd,"k"),cd.forEach(l),Zh=n(io," steps the rollout works send a trajectory of experience to the learner worker, "),ta=a(io,"STRONG",{});var fd=r(ta);Gh=n(fd,"which it uses to update the agent\u2019s policy network"),fd.forEach(l),$h=n(io,"."),io.forEach(l),as=f(e),qt=a(e,"IMG",{src:!0,alt:!0}),rs=f(e),L=a(e,"H3",{class:!0});var Bn=r(L);_e=a(Bn,"A",{id:!0,class:!0,href:!0});var md=r(_e);la=a(md,"SPAN",{});var ud=r(la);u(Ht.$$.fragment,ud),ud.forEach(l),md.forEach(l),Bh=f(Bn),oa=a(Bn,"SPAN",{});var dd=r(oa);Sh=n(dd,"Actor Critic models in Sample-factory"),dd.forEach(l),Bn.forEach(l),ss=f(e),Xl=a(e,"P",{});var yd=r(Xl);Wh=n(yd,"Actor Critic models in Sample Factory are composed of three components:"),yd.forEach(l),ns=f(e),W=a(e,"UL",{});var po=r(W);zl=a(po,"LI",{});var km=r(zl);aa=a(km,"STRONG",{});var vd=r(aa);Ah=n(vd,"Encoder"),vd.forEach(l),Nh=n(km," - Process input observations (images, vectors) and map them to a vector. This is the part of the model you will most likely want to customize."),km.forEach(l),Rh=f(po),ql=a(po,"LI",{});var Zm=r(ql);ra=a(Zm,"STRONG",{});var wd=r(ra);Ch=n(wd,"Core"),wd.forEach(l),Vh=n(Zm," - Intergrate vectors from one or more encoders, can optionally include a single- or multi-layer LSTM/GRU in a memory-based agent."),Zm.forEach(l),Xh=f(po),Hl=a(po,"LI",{});var Gm=r(Hl);sa=a(Gm,"STRONG",{});var Md=r(sa);zh=n(Md,"Decoder"),Md.forEach(l),qh=n(Gm," - Apply additional layers to the output of the model core before computing the policy and value outputs."),Gm.forEach(l),po.forEach(l),is=f(e),Ue=a(e,"P",{});var Sn=r(Ue);Hh=n(Sn,"The library has been designed to automatically support any observation and action spaces. Users can easily add their custom models. You can find out more in the "),Ft=a(Sn,"A",{href:!0,rel:!0});var gd=r(Ft);Fh=n(gd,"documentation"),gd.forEach(l),Yh=n(Sn,"."),Sn.forEach(l),ps=f(e),P=a(e,"H2",{class:!0});var Wn=r(P);je=a(Wn,"A",{id:!0,class:!0,href:!0});var bd=r(je);na=a(bd,"SPAN",{});var Jd=r(na);u(Yt.$$.fragment,Jd),Jd.forEach(l),bd.forEach(l),Lh=f(Wn),ia=a(Wn,"SPAN",{});var Td=r(ia);Ph=n(Td,"ViZDoom"),Td.forEach(l),Wn.forEach(l),hs=f(e),Q=a(e,"P",{});var br=r(Q);Lt=a(br,"A",{href:!0,rel:!0});var _d=r(Lt);Qh=n(_d,"ViZDoom"),_d.forEach(l),Dh=n(br," is an "),pa=a(br,"STRONG",{});var Ud=r(pa);xh=n(Ud,"open-source python interface for the Doom Engine"),Ud.forEach(l),Oh=n(br,"."),br.forEach(l),cs=f(e),Fl=a(e,"P",{});var jd=r(Fl);Kh=n(jd,"The library was created in 2016 by Marek Wydmuch, Michal Kempka  at the\xA0Institute of Computing Science, Poznan University of Technology, Poland."),jd.forEach(l),fs=f(e),A=a(e,"P",{});var ho=r(A);ec=n(ho,"The library enables the "),ha=a(ho,"STRONG",{});var Id=r(ha);tc=n(Id,"training of agents directly from the screen pixels in a number of scenarios"),Id.forEach(l),lc=n(ho,", including team deathmatch, shown in the video below. Because the ViZDoom environment is based on a game the was created in the 90s, it can be run on modern hardware at accelerated speeds, "),ca=a(ho,"STRONG",{});var Ed=r(ca);oc=n(Ed,"allowing us to learn complex AI behaviors fairly quickly"),Ed.forEach(l),ac=n(ho,"."),ho.forEach(l),ms=f(e),Yl=a(e,"P",{});var kd=r(Yl);rc=n(kd,"The library includes feature such as:"),kd.forEach(l),us=f(e),g=a(e,"UL",{});var J=r(g);fa=a(J,"LI",{});var Zd=r(fa);sc=n(Zd,"Multi-platform (Linux, macOS, Windows),"),Zd.forEach(l),nc=f(J),ma=a(J,"LI",{});var Gd=r(ma);ic=n(Gd,"API for Python and C++,"),Gd.forEach(l),pc=f(J),Ll=a(J,"LI",{});var $m=r(Ll);Pt=a($m,"A",{href:!0,rel:!0});var $d=r(Pt);hc=n($d,"OpenAI Gym"),$d.forEach(l),cc=n($m,"\xA0environment wrappers"),$m.forEach(l),fc=f(J),ua=a(J,"LI",{});var Bd=r(ua);mc=n(Bd,"Easy-to-create custom scenarios (visual editors, scripting language, and examples available),"),Bd.forEach(l),uc=f(J),da=a(J,"LI",{});var Sd=r(da);dc=n(Sd,"Async and sync single-player and multiplayer modes,"),Sd.forEach(l),yc=f(J),ya=a(J,"LI",{});var Wd=r(ya);vc=n(Wd,"Lightweight (few MBs) and fast (up to 7000 fps in sync mode, single-threaded),"),Wd.forEach(l),wc=f(J),va=a(J,"LI",{});var Ad=r(va);Mc=n(Ad,"Customizable resolution and rendering parameters,"),Ad.forEach(l),gc=f(J),wa=a(J,"LI",{});var Nd=r(wa);bc=n(Nd,"Access to the depth buffer (3D vision),"),Nd.forEach(l),Jc=f(J),Ma=a(J,"LI",{});var Rd=r(Ma);Tc=n(Rd,"Automatic labeling of game objects visible in the frame,"),Rd.forEach(l),_c=f(J),ga=a(J,"LI",{});var Cd=r(ga);Uc=n(Cd,"Access to the audio buffer"),Cd.forEach(l),jc=f(J),ba=a(J,"LI",{});var Vd=r(ba);Ic=n(Vd,"Access to the list of actors/objects and map geometry,"),Vd.forEach(l),Ec=f(J),Ja=a(J,"LI",{});var Xd=r(Ja);kc=n(Xd,"Off-screen rendering and episode recording,"),Xd.forEach(l),Zc=f(J),Ta=a(J,"LI",{});var zd=r(Ta);Gc=n(zd,"Time scaling in async mode."),zd.forEach(l),J.forEach(l),ds=f(e),D=a(e,"H2",{class:!0});var An=r(D);Ie=a(An,"A",{id:!0,class:!0,href:!0});var qd=r(Ie);_a=a(qd,"SPAN",{});var Hd=r(_a);u(Qt.$$.fragment,Hd),Hd.forEach(l),qd.forEach(l),$c=f(An),Ua=a(An,"SPAN",{});var Fd=r(Ua);Bc=n(Fd,"We first need to install some dependencies that are required for the ViZDoom environment"),Fd.forEach(l),An.forEach(l),ys=f(e),Pl=a(e,"P",{});var Yd=r(Pl);Sc=n(Yd,"Now that our Colab runtime is set up, we can start by installing the dependencies required to run ViZDoom on linux."),Yd.forEach(l),vs=f(e),Ee=a(e,"P",{});var Nn=r(Ee);Wc=n(Nn,"If you are following on your machine on Mac, you will want to follow the installation instructions on the "),Dt=a(Nn,"A",{href:!0,rel:!0});var Ld=r(Dt);Ac=n(Ld,"github page"),Ld.forEach(l),Nc=n(Nn,"."),Nn.forEach(l),ws=f(e),u(xt.$$.fragment,e),Ms=f(e),x=a(e,"H2",{class:!0});var Rn=r(x);ke=a(Rn,"A",{id:!0,class:!0,href:!0});var Pd=r(ke);ja=a(Pd,"SPAN",{});var Qd=r(ja);u(Ot.$$.fragment,Qd),Qd.forEach(l),Pd.forEach(l),Rc=f(Rn),Ia=a(Rn,"SPAN",{});var Dd=r(Ia);Cc=n(Dd,"Then we can install Sample Factory and ViZDoom"),Dd.forEach(l),Rn.forEach(l),gs=f(e),Ql=a(e,"UL",{});var xd=r(Ql);Ea=a(xd,"LI",{});var Od=r(Ea);Vc=n(Od,"This can take 7min"),Od.forEach(l),xd.forEach(l),bs=f(e),u(Kt.$$.fragment,e),Js=f(e),O=a(e,"H2",{class:!0});var Cn=r(O);Ze=a(Cn,"A",{id:!0,class:!0,href:!0});var Kd=r(Ze);ka=a(Kd,"SPAN",{});var ey=r(ka);u(el.$$.fragment,ey),ey.forEach(l),Kd.forEach(l),Xc=f(Cn),Za=a(Cn,"SPAN",{});var ty=r(Za);zc=n(ty,"Setting up the Doom Environment in sample-factory"),ty.forEach(l),Cn.forEach(l),Ts=f(e),u(tl.$$.fragment,e),_s=f(e),Ge=a(e,"P",{});var Vn=r(Ge);qc=n(Vn,"Now that the setup if complete, we can train the agent. We have chosen here to learn a ViZDoom task called "),Ga=a(Vn,"CODE",{});var ly=r(Ga);Hc=n(ly,"Health Gathering Supreme"),ly.forEach(l),Fc=n(Vn,"."),Vn.forEach(l),Us=f(e),K=a(e,"H3",{class:!0});var Xn=r(K);$e=a(Xn,"A",{id:!0,class:!0,href:!0});var oy=r($e);$a=a(oy,"SPAN",{});var ay=r($a);u(ll.$$.fragment,ay),ay.forEach(l),oy.forEach(l),Yc=f(Xn),Ba=a(Xn,"SPAN",{});var ry=r(Ba);Lc=n(ry,"The scenario: Health Gathering Supreme"),ry.forEach(l),Xn.forEach(l),js=f(e),ol=a(e,"IMG",{src:!0,alt:!0}),Is=f(e),j=a(e,"P",{});var Qe=r(j);Pc=n(Qe,"The objective of this scenario is to "),Sa=a(Qe,"STRONG",{});var sy=r(Sa);Qc=n(sy,"teach the agent how to survive without knowing what makes it survive"),sy.forEach(l),Dc=n(Qe,". The Agent know only that "),Wa=a(Qe,"STRONG",{});var ny=r(Wa);xc=n(ny,"life is precious"),ny.forEach(l),Oc=n(Qe," and death is bad so "),Aa=a(Qe,"STRONG",{});var iy=r(Aa);Kc=n(iy,"it must learn what prolongs its existence and that its health is connected with survival"),iy.forEach(l),ef=n(Qe,"."),Qe.forEach(l),Es=f(e),N=a(e,"P",{});var co=r(N);tf=n(co,"The map is a rectangle containing walls and with a green, acidic floor which "),Na=a(co,"STRONG",{});var py=r(Na);lf=n(py,"hurts the player periodically"),py.forEach(l),of=n(co,". Initially there are some medkits spread uniformly over the map. A new medkit falls from the skies every now and then. "),Ra=a(co,"STRONG",{});var hy=r(Ra);af=n(hy,"Medkits heal some portions of player\u2019s health"),hy.forEach(l),rf=n(co," - to survive, the agent needs to pick them up. The episode finishes after the player\u2019s death or on timeout."),co.forEach(l),ks=f(e),Dl=a(e,"P",{});var cy=r(Dl);sf=n(cy,"Further configuration:"),cy.forEach(l),Zs=f(e),I=a(e,"UL",{});var De=r(I);Ca=a(De,"LI",{});var fy=r(Ca);nf=n(fy,"Living_reward = 1"),fy.forEach(l),pf=f(De),Va=a(De,"LI",{});var my=r(Va);hf=n(my,"3 available buttons: turn left, turn right, move forward"),my.forEach(l),cf=f(De),Xa=a(De,"LI",{});var uy=r(Xa);ff=n(uy,"1 available game variable: HEALTH"),uy.forEach(l),mf=f(De),za=a(De,"LI",{});var dy=r(za);uf=n(dy,"death penalty = 100"),dy.forEach(l),De.forEach(l),Gs=f(e),Be=a(e,"P",{});var zn=r(Be);df=n(zn,"You can find out more about the scenarios available in ViZDoom "),al=a(zn,"A",{href:!0,rel:!0});var yy=r(al);yf=n(yy,"here"),yy.forEach(l),vf=n(zn,"."),zn.forEach(l),$s=f(e),Se=a(e,"P",{});var qn=r(Se);wf=n(qn,"There are also a number of more complex scenarios that have been create for ViZDoom, such as the ones detailed on "),rl=a(qn,"A",{href:!0,rel:!0});var vy=r(rl);Mf=n(vy,"this github page"),vy.forEach(l),gf=n(qn,"."),qn.forEach(l),Bs=f(e),ee=a(e,"H2",{class:!0});var Hn=r(ee);We=a(Hn,"A",{id:!0,class:!0,href:!0});var wy=r(We);qa=a(wy,"SPAN",{});var My=r(qa);u(sl.$$.fragment,My),My.forEach(l),wy.forEach(l),bf=f(Hn),Ha=a(Hn,"SPAN",{});var gy=r(Ha);Jf=n(gy,"Training the agent"),gy.forEach(l),Hn.forEach(l),Ss=f(e),xl=a(e,"UL",{});var by=r(xl);Fa=a(by,"LI",{});var Jy=r(Fa);Tf=n(Jy,"We\u2019re going to train the agent for 4000000 steps. It will take approximately 20min"),Jy.forEach(l),by.forEach(l),Ws=f(e),u(nl.$$.fragment,e),As=f(e),te=a(e,"H2",{class:!0});var Fn=r(te);Ae=a(Fn,"A",{id:!0,class:!0,href:!0});var Ty=r(Ae);Ya=a(Ty,"SPAN",{});var _y=r(Ya);u(il.$$.fragment,_y),_y.forEach(l),Ty.forEach(l),_f=f(Fn),La=a(Fn,"SPAN",{});var Uy=r(La);Uf=n(Uy,"Let's take a look at the performance of the trained policy and output a video of the agent."),Uy.forEach(l),Fn.forEach(l),Ns=f(e),u(pl.$$.fragment,e),Rs=f(e),le=a(e,"H2",{class:!0});var Yn=r(le);Ne=a(Yn,"A",{id:!0,class:!0,href:!0});var jy=r(Ne);Pa=a(jy,"SPAN",{});var Iy=r(Pa);u(hl.$$.fragment,Iy),Iy.forEach(l),jy.forEach(l),jf=f(Yn),Qa=a(Yn,"SPAN",{});var Ey=r(Qa);If=n(Ey,"Now lets visualize the performance of the agent"),Ey.forEach(l),Yn.forEach(l),Cs=f(e),u(cl.$$.fragment,e),Vs=f(e),Ol=a(e,"P",{});var ky=r(Ol);Ef=n(ky,"The agent has learned something, but its performance could be better. We would clearly need to train for longer. But let\u2019s upload this model to the Hub."),ky.forEach(l),Xs=f(e),oe=a(e,"H2",{class:!0});var Ln=r(oe);Re=a(Ln,"A",{id:!0,class:!0,href:!0});var Zy=r(Re);Da=a(Zy,"SPAN",{});var Gy=r(Da);u(fl.$$.fragment,Gy),Gy.forEach(l),Zy.forEach(l),kf=f(Ln),xa=a(Ln,"SPAN",{});var $y=r(xa);Zf=n($y,"Now lets upload your checkpoint and video to the Hugging Face Hub"),$y.forEach(l),Ln.forEach(l),zs=f(e),Kl=a(e,"P",{});var By=r(Kl);Gf=n(By,"To be able to share your model with the community there are three more steps to follow:"),By.forEach(l),qs=f(e),ml=a(e,"P",{});var Bm=r(ml);$f=n(Bm,"1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),ul=a(Bm,"A",{href:!0,rel:!0});var Sy=r(ul);Bf=n(Sy,"https://huggingface.co/join"),Sy.forEach(l),Bm.forEach(l),Hs=f(e),eo=a(e,"P",{});var Wy=r(eo);Sf=n(Wy,"2\uFE0F\u20E3 Sign in and get your authentication token from the Hugging Face website."),Wy.forEach(l),Fs=f(e),to=a(e,"UL",{});var Ay=r(to);Ce=a(Ay,"LI",{});var Jr=r(Ce);Wf=n(Jr,"Create a new token ("),dl=a(Jr,"A",{href:!0,rel:!0});var Ny=r(dl);Af=n(Ny,"https://huggingface.co/settings/tokens"),Ny.forEach(l),Nf=n(Jr,") "),Oa=a(Jr,"STRONG",{});var Ry=r(Oa);Rf=n(Ry,"with write role"),Ry.forEach(l),Jr.forEach(l),Ay.forEach(l),Ys=f(e),yl=a(e,"IMG",{src:!0,alt:!0}),Ls=f(e),Ve=a(e,"UL",{});var Pn=r(Ve);Ka=a(Pn,"LI",{});var Cy=r(Ka);Cf=n(Cy,"Copy the token"),Cy.forEach(l),Vf=f(Pn),er=a(Pn,"LI",{});var Vy=r(er);Xf=n(Vy,"Run the cell below and paste the token"),Vy.forEach(l),Pn.forEach(l),Ps=f(e),vl=a(e,"P",{});var Sm=r(vl);zf=n(Sm,"If you don\u2019t want to use Google Colab or a Jupyter Notebook, you need to use this command instead: "),tr=a(Sm,"CODE",{});var Xy=r(tr);qf=n(Xy,"huggingface-cli login"),Xy.forEach(l),Sm.forEach(l),Qs=f(e),u(wl.$$.fragment,e),Ds=f(e),u(Ml.$$.fragment,e),xs=f(e),ae=a(e,"H2",{class:!0});var Qn=r(ae);Xe=a(Qn,"A",{id:!0,class:!0,href:!0});var zy=r(Xe);lr=a(zy,"SPAN",{});var qy=r(lr);u(gl.$$.fragment,qy),qy.forEach(l),zy.forEach(l),Hf=f(Qn),or=a(Qn,"SPAN",{});var Hy=r(or);Ff=n(Hy,"Let's load another model"),Hy.forEach(l),Qn.forEach(l),Os=f(e),lo=a(e,"P",{});var Fy=r(lo);Yf=n(Fy,"This agent\u2019s performance was good, but we can do better! Let\u2019s download and visualize an agent trained for 10B timesteps from the hub."),Fy.forEach(l),Ks=f(e),u(bl.$$.fragment,e),en=f(e),u(Jl.$$.fragment,e),tn=f(e),u(Tl.$$.fragment,e),ln=f(e),u(_l.$$.fragment,e),on=f(e),re=a(e,"H2",{class:!0});var Dn=r(re);ze=a(Dn,"A",{id:!0,class:!0,href:!0});var Yy=r(ze);ar=a(Yy,"SPAN",{});var Ly=r(ar);u(Ul.$$.fragment,Ly),Ly.forEach(l),Yy.forEach(l),Lf=f(Dn),rr=a(Dn,"SPAN",{});var Py=r(rr);Pf=n(Py,"Some additional challenges \u{1F3C6}: Doom Deathmatch"),Py.forEach(l),Dn.forEach(l),an=f(e),qe=a(e,"P",{});var xn=r(qe);Qf=n(xn,"Training an agent to play a Doom deathmatch "),sr=a(xn,"STRONG",{});var Qy=r(sr);Df=n(Qy,"takes many hours on a more beefy machine than is available in Colab"),Qy.forEach(l),xf=n(xn,"."),xn.forEach(l),rn=f(e),He=a(e,"P",{});var On=r(He);Of=n(On,"Fortunately, we have have "),nr=a(On,"STRONG",{});var Dy=r(nr);Kf=n(Dy,"already trained an agent in this scenario and it is available in the \u{1F917} Hub!"),Dy.forEach(l),em=n(On," Let\u2019s download the model and visualize the agent\u2019s performance."),On.forEach(l),sn=f(e),u(jl.$$.fragment,e),nn=f(e),Fe=a(e,"P",{});var Kn=r(Fe);tm=n(Kn,"Given the agent plays for a long time the video generation can take "),ir=a(Kn,"STRONG",{});var xy=r(ir);lm=n(xy,"10 minutes"),xy.forEach(l),om=n(Kn,"."),Kn.forEach(l),pn=f(e),u(Il.$$.fragment,e),hn=f(e),se=a(e,"P",{});var Tr=r(se);am=n(Tr,"You "),pr=a(Tr,"STRONG",{});var Oy=r(pr);rm=n(Oy,"can try to train your agent in this environment"),Oy.forEach(l),sm=n(Tr,` using the code above, but not on colab.
`),hr=a(Tr,"STRONG",{});var Ky=r(hr);nm=n(Ky,"Good luck \u{1F91E}"),Ky.forEach(l),Tr.forEach(l),cn=f(e),El=a(e,"P",{});var Wm=r(El);im=n(Wm,"If you prefer an easier scenario, "),ne=a(Wm,"STRONG",{});var fo=r(ne);pm=n(fo,"why not try training in another ViZDoom scenario such as "),cr=a(fo,"CODE",{});var ev=r(cr);hm=n(ev,"doom_deadly_corridor"),ev.forEach(l),cm=n(fo," or "),fr=a(fo,"CODE",{});var tv=r(fr);fm=n(tv,"doom_defend_the_center"),tv.forEach(l),mm=n(fo,"."),fo.forEach(l),Wm.forEach(l),fn=f(e),mn=a(e,"HR",{}),un=f(e),Ye=a(e,"P",{});var ei=r(Ye);um=n(ei,"This concludes the last unit. But we are not finished yet! \u{1F917} The following "),mr=a(ei,"STRONG",{});var lv=r(mr);dm=n(lv,"bonus section include some of the most interesting, advanced, and cutting edge work in Deep Reinforcement Learning"),lv.forEach(l),ym=n(ei,"."),ei.forEach(l),dn=f(e),ie=a(e,"H2",{class:!0});var ti=r(ie);Le=a(ti,"A",{id:!0,class:!0,href:!0});var ov=r(Le);ur=a(ov,"SPAN",{});var av=r(ur);u(kl.$$.fragment,av),av.forEach(l),ov.forEach(l),vm=f(ti),dr=a(ti,"SPAN",{});var rv=r(dr);wm=n(rv,"Keep learning, stay awesome \u{1F917}"),rv.forEach(l),ti.forEach(l),this.h()},h(){h(R,"name","hf:doc:metadata"),h(R,"content",JSON.stringify(uv)),h(he,"id","handson-advanced-deep-reinforcement-learning-using-sample-factory-to-play-doom-from-pixels"),h(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(he,"href","#handson-advanced-deep-reinforcement-learning-using-sample-factory-to-play-doom-from-pixels"),h(C,"class","relative group"),xe(Zl.src,Nm="https://colab.research.google.com/assets/colab-badge.svg")||h(Zl,"src",Nm),h(Zl,"alt","Open In Colab"),h(tt,"href","https://colab.research.google.com/github/huggingface/deep-rl-class/blob/master/notebooks/unit8/unit8_part2.ipynb"),h(tt,"rel","nofollow"),h(ce,"id","unit-8-part-2-advanced-deep-reinforcement-learning-using-sample-factory-to-play-doom-from-pixels"),h(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ce,"href","#unit-8-part-2-advanced-deep-reinforcement-learning-using-sample-factory-to-play-doom-from-pixels"),h(V,"class","relative group"),xe(ot.src,Rm="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit9/thumbnail2.png")||h(ot,"src",Rm),h(ot,"alt","Thumbnail"),h(at,"href","https://www.samplefactory.dev/"),h(at,"rel","nofollow"),h(rt,"href","https://www.samplefactory.dev/"),h(rt,"rel","nofollow"),h(nt,"href","https://github.com/alex-petrenko/sample-factory/tree/master/sf_examples"),h(nt,"rel","nofollow"),h(pt,"href","https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),h(pt,"rel","nofollow"),h(ht,"href","https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),h(ht,"rel","nofollow"),h(mt,"href","https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),h(mt,"rel","nofollow"),h(de,"id","set-the-gpu"),h(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(de,"href","#set-the-gpu"),h(z,"class","relative group"),xe(dt.src,Cm="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step1.jpg")||h(dt,"src",Cm),h(dt,"alt","GPU Step 1"),xe(yt.src,Vm="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step2.jpg")||h(yt,"src",Vm),h(yt,"alt","GPU Step 2"),h(we,"id","sample-factory"),h(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(we,"href","#sample-factory"),h(q,"class","relative group"),h(wt,"href","https://www.samplefactory.dev/"),h(wt,"rel","nofollow"),xe(Mt.src,Xm="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit9/samplefactoryenvs.png")||h(Mt,"src",Xm),h(Mt,"alt","Sample factory"),h(Me,"id","key-features"),h(Me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Me,"href","#key-features"),h(F,"class","relative group"),h(Jt,"href","https://www.samplefactory.dev/06-architecture/overview/"),h(Jt,"rel","nofollow"),h(Tt,"href","https://www.samplefactory.dev/07-advanced-topics/sync-async/"),h(Tt,"rel","nofollow"),h(_t,"href","https://www.samplefactory.dev/07-advanced-topics/serial-mode/"),h(_t,"rel","nofollow"),h(Ut,"href","https://www.samplefactory.dev/09-environment-integrations/isaacgym/"),h(Ut,"rel","nofollow"),h(It,"href","https://www.samplefactory.dev/07-advanced-topics/multi-policy-training/"),h(It,"rel","nofollow"),h(kt,"href","https://www.samplefactory.dev/07-advanced-topics/pbt/"),h(kt,"rel","nofollow"),h(Zt,"href","https://www.samplefactory.dev/03-customization/custom-models/"),h(Zt,"rel","nofollow"),h($t,"href","https://www.samplefactory.dev/03-customization/custom-environments/"),h($t,"rel","nofollow"),h(Bt,"href","https://www.samplefactory.dev/05-monitoring/metrics-reference/"),h(Bt,"rel","nofollow"),h(St,"href","https://www.samplefactory.dev/05-monitoring/custom-metrics/"),h(St,"rel","nofollow"),h(Wt,"href","https://www.samplefactory.dev/10-huggingface/huggingface/"),h(Wt,"rel","nofollow"),h(At,"href","https://www.samplefactory.dev/09-environment-integrations/mujoco/"),h(At,"rel","nofollow"),h(Nt,"href","https://www.samplefactory.dev/09-environment-integrations/atari/"),h(Nt,"rel","nofollow"),h(Rt,"href","https://www.samplefactory.dev/09-environment-integrations/vizdoom/"),h(Rt,"rel","nofollow"),h(Ct,"href","https://www.samplefactory.dev/09-environment-integrations/dmlab/"),h(Ct,"rel","nofollow"),h(Xt,"href","https://huggingface.co/models?library=sample-factory&sort=downloads"),h(Xt,"rel","nofollow"),h(be,"id","how-samplefactory-works"),h(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(be,"href","#how-samplefactory-works"),h(Y,"class","relative group"),xe(qt.src,zm="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit9/samplefactory.png")||h(qt,"src",zm),h(qt,"alt","Sample factory"),h(_e,"id","actor-critic-models-in-samplefactory"),h(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(_e,"href","#actor-critic-models-in-samplefactory"),h(L,"class","relative group"),h(Ft,"href","https://www.samplefactory.dev/03-customization/custom-models/#actor-critic-models-in-sample-factory"),h(Ft,"rel","nofollow"),h(je,"id","vizdoom"),h(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(je,"href","#vizdoom"),h(P,"class","relative group"),h(Lt,"href","https://vizdoom.cs.put.edu.pl/"),h(Lt,"rel","nofollow"),h(Pt,"href","https://www.gymlibrary.dev/"),h(Pt,"rel","nofollow"),h(Ie,"id","we-first-need-to-install-some-dependencies-that-are-required-for-the-vizdoom-environment"),h(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ie,"href","#we-first-need-to-install-some-dependencies-that-are-required-for-the-vizdoom-environment"),h(D,"class","relative group"),h(Dt,"href","https://github.com/Farama-Foundation/ViZDoom/blob/master/doc/Quickstart.md#-quickstart-for-macos-and-anaconda3-python-36"),h(Dt,"rel","nofollow"),h(ke,"id","then-we-can-install-sample-factory-and-vizdoom"),h(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ke,"href","#then-we-can-install-sample-factory-and-vizdoom"),h(x,"class","relative group"),h(Ze,"id","setting-up-the-doom-environment-in-samplefactory"),h(Ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ze,"href","#setting-up-the-doom-environment-in-samplefactory"),h(O,"class","relative group"),h($e,"id","the-scenario-health-gathering-supreme"),h($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h($e,"href","#the-scenario-health-gathering-supreme"),h(K,"class","relative group"),xe(ol.src,qm="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit9/Health-Gathering-Supreme.png")||h(ol,"src",qm),h(ol,"alt","Health-Gathering-Supreme"),h(al,"href","https://github.com/Farama-Foundation/ViZDoom/tree/master/scenarios"),h(al,"rel","nofollow"),h(rl,"href","https://github.com/edbeeching/3d_control_deep_rl"),h(rl,"rel","nofollow"),h(We,"id","training-the-agent"),h(We,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(We,"href","#training-the-agent"),h(ee,"class","relative group"),h(Ae,"id","lets-take-a-look-at-the-performance-of-the-trained-policy-and-output-a-video-of-the-agent"),h(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ae,"href","#lets-take-a-look-at-the-performance-of-the-trained-policy-and-output-a-video-of-the-agent"),h(te,"class","relative group"),h(Ne,"id","now-lets-visualize-the-performance-of-the-agent"),h(Ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ne,"href","#now-lets-visualize-the-performance-of-the-agent"),h(le,"class","relative group"),h(Re,"id","now-lets-upload-your-checkpoint-and-video-to-the-hugging-face-hub"),h(Re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Re,"href","#now-lets-upload-your-checkpoint-and-video-to-the-hugging-face-hub"),h(oe,"class","relative group"),h(ul,"href","https://huggingface.co/join"),h(ul,"rel","nofollow"),h(dl,"href","https://huggingface.co/settings/tokens"),h(dl,"rel","nofollow"),xe(yl.src,Hm="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/create-token.jpg")||h(yl,"src",Hm),h(yl,"alt","Create HF Token"),h(Xe,"id","lets-load-another-model"),h(Xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Xe,"href","#lets-load-another-model"),h(ae,"class","relative group"),h(ze,"id","some-additional-challenges-doom-deathmatch"),h(ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ze,"href","#some-additional-challenges-doom-deathmatch"),h(re,"class","relative group"),h(Le,"id","keep-learning-stay-awesome"),h(Le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Le,"href","#keep-learning-stay-awesome"),h(ie,"class","relative group")},m(e,i){t(document.head,R),p(e,_r,i),p(e,C,i),t(C,he),t(he,mo),d(Oe,mo,null),t(C,li),t(C,uo),t(uo,oi),p(e,Ur,i),d(Ke,e,i),p(e,jr,i),p(e,et,i),t(et,ai),t(et,tt),t(tt,Zl),p(e,Ir,i),p(e,V,i),t(V,ce),t(ce,yo),d(lt,yo,null),t(V,ri),t(V,vo),t(vo,si),p(e,Er,i),p(e,ot,i),p(e,kr,i),p(e,fe,i),t(fe,ni),t(fe,at),t(at,ii),t(fe,pi),p(e,Zr,i),p(e,Gl,i),t(Gl,hi),p(e,Gr,i),p(e,k,i),t(k,wo),t(wo,me),t(me,rt),t(rt,ci),t(me,fi),t(me,Mo),t(Mo,mi),t(me,ui),t(k,di),t(k,go),t(go,E),t(E,yi),t(E,bo),t(bo,vi),t(E,wi),t(E,Jo),t(Jo,Mi),t(E,gi),t(E,To),t(To,bi),t(E,Ji),t(k,Ti),t(k,_o),t(_o,st),t(st,_i),t(st,nt),t(nt,Ui),t(st,ji),p(e,$r,i),d(it,e,i),p(e,Br,i),p(e,ue,i),t(ue,Ii),t(ue,pt),t(pt,Ei),t(ue,ki),p(e,Sr,i),p(e,$l,i),t($l,Bl),t(Bl,Uo),t(Uo,Zi),t(Bl,Gi),p(e,Wr,i),p(e,X,i),t(X,$i),t(X,ht),t(ht,Bi),t(X,Si),t(X,jo),t(jo,Wi),p(e,Ar,i),p(e,ct,i),t(ct,Ai),t(ct,Io),t(Io,Ni),p(e,Nr,i),p(e,ft,i),t(ft,Ri),t(ft,mt),t(mt,Ci),p(e,Rr,i),p(e,z,i),t(z,de),t(de,Eo),d(ut,Eo,null),t(z,Vi),t(z,ko),t(ko,Xi),p(e,Cr,i),p(e,Sl,i),t(Sl,ye),t(ye,zi),t(ye,Zo),t(Zo,qi),t(ye,Hi),t(ye,Go),t(Go,Fi),p(e,Vr,i),p(e,dt,i),p(e,Xr,i),p(e,Wl,i),t(Wl,$o),t($o,Bo),t(Bo,Yi),p(e,zr,i),p(e,yt,i),p(e,qr,i),p(e,ve,i),t(ve,Li),t(ve,So),t(So,Pi),t(ve,Qi),p(e,Hr,i),p(e,q,i),t(q,we),t(we,Wo),d(vt,Wo,null),t(q,Di),t(q,Ao),t(Ao,xi),p(e,Fr,i),p(e,H,i),t(H,wt),t(wt,Oi),t(H,Ki),t(H,No),t(No,ep),t(H,tp),p(e,Yr,i),p(e,Z,i),t(Z,lp),t(Z,Ro),t(Ro,op),t(Z,ap),t(Z,Co),t(Co,rp),t(Z,sp),p(e,Lr,i),p(e,Mt,i),p(e,Pr,i),p(e,F,i),t(F,Me),t(Me,Vo),d(gt,Vo,null),t(F,np),t(F,Xo),t(Xo,ip),p(e,Qr,i),p(e,M,i),t(M,bt),t(bt,pp),t(bt,Jt),t(Jt,hp),t(bt,cp),t(M,fp),t(M,Al),t(Al,Tt),t(Tt,mp),t(Al,up),t(M,dp),t(M,Nl),t(Nl,_t),t(_t,yp),t(Nl,vp),t(M,wp),t(M,Rl),t(Rl,Mp),t(Rl,Ut),t(Ut,gp),t(M,bp),t(M,jt),t(jt,Jp),t(jt,It),t(It,Tp),t(jt,_p),t(M,Up),t(M,Et),t(Et,jp),t(Et,kt),t(kt,Ip),t(Et,Ep),t(M,kp),t(M,zo),t(zo,Zp),t(M,Gp),t(M,qo),t(qo,$p),t(M,Bp),t(M,Cl),t(Cl,Sp),t(Cl,Zt),t(Zt,Wp),t(M,Ap),t(M,Gt),t(Gt,Np),t(Gt,$t),t($t,Rp),t(Gt,Cp),t(M,Vp),t(M,ge),t(ge,Xp),t(ge,Bt),t(Bt,zp),t(ge,qp),t(ge,St),t(St,Hp),t(M,Fp),t(M,Vl),t(Vl,Wt),t(Wt,Yp),t(Vl,Lp),t(M,Pp),t(M,U),t(U,At),t(At,Qp),t(U,Dp),t(U,Nt),t(Nt,xp),t(U,Op),t(U,Rt),t(Rt,Kp),t(U,eh),t(U,Ct),t(Ct,th),t(U,lh),p(e,Dr,i),p(e,Vt,i),t(Vt,oh),t(Vt,Xt),t(Xt,ah),p(e,xr,i),p(e,Y,i),t(Y,be),t(be,Ho),d(zt,Ho,null),t(Y,rh),t(Y,Fo),t(Fo,sh),p(e,Or,i),p(e,Je,i),t(Je,nh),t(Je,Yo),t(Yo,ih),t(Je,ph),p(e,Kr,i),p(e,Te,i),t(Te,hh),t(Te,Lo),t(Lo,ch),t(Te,fh),p(e,es,i),p(e,G,i),t(G,mh),t(G,Po),t(Po,uh),t(G,dh),t(G,Qo),t(Qo,yh),t(G,vh),p(e,ts,i),p(e,$,i),t($,wh),t($,Do),t(Do,Mh),t($,gh),t($,xo),t(xo,bh),t($,Jh),p(e,ls,i),p(e,B,i),t(B,Th),t(B,Oo),t(Oo,_h),t(B,Uh),t(B,Ko),t(Ko,jh),t(B,Ih),p(e,os,i),p(e,S,i),t(S,Eh),t(S,ea),t(ea,kh),t(S,Zh),t(S,ta),t(ta,Gh),t(S,$h),p(e,as,i),p(e,qt,i),p(e,rs,i),p(e,L,i),t(L,_e),t(_e,la),d(Ht,la,null),t(L,Bh),t(L,oa),t(oa,Sh),p(e,ss,i),p(e,Xl,i),t(Xl,Wh),p(e,ns,i),p(e,W,i),t(W,zl),t(zl,aa),t(aa,Ah),t(zl,Nh),t(W,Rh),t(W,ql),t(ql,ra),t(ra,Ch),t(ql,Vh),t(W,Xh),t(W,Hl),t(Hl,sa),t(sa,zh),t(Hl,qh),p(e,is,i),p(e,Ue,i),t(Ue,Hh),t(Ue,Ft),t(Ft,Fh),t(Ue,Yh),p(e,ps,i),p(e,P,i),t(P,je),t(je,na),d(Yt,na,null),t(P,Lh),t(P,ia),t(ia,Ph),p(e,hs,i),p(e,Q,i),t(Q,Lt),t(Lt,Qh),t(Q,Dh),t(Q,pa),t(pa,xh),t(Q,Oh),p(e,cs,i),p(e,Fl,i),t(Fl,Kh),p(e,fs,i),p(e,A,i),t(A,ec),t(A,ha),t(ha,tc),t(A,lc),t(A,ca),t(ca,oc),t(A,ac),p(e,ms,i),p(e,Yl,i),t(Yl,rc),p(e,us,i),p(e,g,i),t(g,fa),t(fa,sc),t(g,nc),t(g,ma),t(ma,ic),t(g,pc),t(g,Ll),t(Ll,Pt),t(Pt,hc),t(Ll,cc),t(g,fc),t(g,ua),t(ua,mc),t(g,uc),t(g,da),t(da,dc),t(g,yc),t(g,ya),t(ya,vc),t(g,wc),t(g,va),t(va,Mc),t(g,gc),t(g,wa),t(wa,bc),t(g,Jc),t(g,Ma),t(Ma,Tc),t(g,_c),t(g,ga),t(ga,Uc),t(g,jc),t(g,ba),t(ba,Ic),t(g,Ec),t(g,Ja),t(Ja,kc),t(g,Zc),t(g,Ta),t(Ta,Gc),p(e,ds,i),p(e,D,i),t(D,Ie),t(Ie,_a),d(Qt,_a,null),t(D,$c),t(D,Ua),t(Ua,Bc),p(e,ys,i),p(e,Pl,i),t(Pl,Sc),p(e,vs,i),p(e,Ee,i),t(Ee,Wc),t(Ee,Dt),t(Dt,Ac),t(Ee,Nc),p(e,ws,i),d(xt,e,i),p(e,Ms,i),p(e,x,i),t(x,ke),t(ke,ja),d(Ot,ja,null),t(x,Rc),t(x,Ia),t(Ia,Cc),p(e,gs,i),p(e,Ql,i),t(Ql,Ea),t(Ea,Vc),p(e,bs,i),d(Kt,e,i),p(e,Js,i),p(e,O,i),t(O,Ze),t(Ze,ka),d(el,ka,null),t(O,Xc),t(O,Za),t(Za,zc),p(e,Ts,i),d(tl,e,i),p(e,_s,i),p(e,Ge,i),t(Ge,qc),t(Ge,Ga),t(Ga,Hc),t(Ge,Fc),p(e,Us,i),p(e,K,i),t(K,$e),t($e,$a),d(ll,$a,null),t(K,Yc),t(K,Ba),t(Ba,Lc),p(e,js,i),p(e,ol,i),p(e,Is,i),p(e,j,i),t(j,Pc),t(j,Sa),t(Sa,Qc),t(j,Dc),t(j,Wa),t(Wa,xc),t(j,Oc),t(j,Aa),t(Aa,Kc),t(j,ef),p(e,Es,i),p(e,N,i),t(N,tf),t(N,Na),t(Na,lf),t(N,of),t(N,Ra),t(Ra,af),t(N,rf),p(e,ks,i),p(e,Dl,i),t(Dl,sf),p(e,Zs,i),p(e,I,i),t(I,Ca),t(Ca,nf),t(I,pf),t(I,Va),t(Va,hf),t(I,cf),t(I,Xa),t(Xa,ff),t(I,mf),t(I,za),t(za,uf),p(e,Gs,i),p(e,Be,i),t(Be,df),t(Be,al),t(al,yf),t(Be,vf),p(e,$s,i),p(e,Se,i),t(Se,wf),t(Se,rl),t(rl,Mf),t(Se,gf),p(e,Bs,i),p(e,ee,i),t(ee,We),t(We,qa),d(sl,qa,null),t(ee,bf),t(ee,Ha),t(Ha,Jf),p(e,Ss,i),p(e,xl,i),t(xl,Fa),t(Fa,Tf),p(e,Ws,i),d(nl,e,i),p(e,As,i),p(e,te,i),t(te,Ae),t(Ae,Ya),d(il,Ya,null),t(te,_f),t(te,La),t(La,Uf),p(e,Ns,i),d(pl,e,i),p(e,Rs,i),p(e,le,i),t(le,Ne),t(Ne,Pa),d(hl,Pa,null),t(le,jf),t(le,Qa),t(Qa,If),p(e,Cs,i),d(cl,e,i),p(e,Vs,i),p(e,Ol,i),t(Ol,Ef),p(e,Xs,i),p(e,oe,i),t(oe,Re),t(Re,Da),d(fl,Da,null),t(oe,kf),t(oe,xa),t(xa,Zf),p(e,zs,i),p(e,Kl,i),t(Kl,Gf),p(e,qs,i),p(e,ml,i),t(ml,$f),t(ml,ul),t(ul,Bf),p(e,Hs,i),p(e,eo,i),t(eo,Sf),p(e,Fs,i),p(e,to,i),t(to,Ce),t(Ce,Wf),t(Ce,dl),t(dl,Af),t(Ce,Nf),t(Ce,Oa),t(Oa,Rf),p(e,Ys,i),p(e,yl,i),p(e,Ls,i),p(e,Ve,i),t(Ve,Ka),t(Ka,Cf),t(Ve,Vf),t(Ve,er),t(er,Xf),p(e,Ps,i),p(e,vl,i),t(vl,zf),t(vl,tr),t(tr,qf),p(e,Qs,i),d(wl,e,i),p(e,Ds,i),d(Ml,e,i),p(e,xs,i),p(e,ae,i),t(ae,Xe),t(Xe,lr),d(gl,lr,null),t(ae,Hf),t(ae,or),t(or,Ff),p(e,Os,i),p(e,lo,i),t(lo,Yf),p(e,Ks,i),d(bl,e,i),p(e,en,i),d(Jl,e,i),p(e,tn,i),d(Tl,e,i),p(e,ln,i),d(_l,e,i),p(e,on,i),p(e,re,i),t(re,ze),t(ze,ar),d(Ul,ar,null),t(re,Lf),t(re,rr),t(rr,Pf),p(e,an,i),p(e,qe,i),t(qe,Qf),t(qe,sr),t(sr,Df),t(qe,xf),p(e,rn,i),p(e,He,i),t(He,Of),t(He,nr),t(nr,Kf),t(He,em),p(e,sn,i),d(jl,e,i),p(e,nn,i),p(e,Fe,i),t(Fe,tm),t(Fe,ir),t(ir,lm),t(Fe,om),p(e,pn,i),d(Il,e,i),p(e,hn,i),p(e,se,i),t(se,am),t(se,pr),t(pr,rm),t(se,sm),t(se,hr),t(hr,nm),p(e,cn,i),p(e,El,i),t(El,im),t(El,ne),t(ne,pm),t(ne,cr),t(cr,hm),t(ne,cm),t(ne,fr),t(fr,fm),t(ne,mm),p(e,fn,i),p(e,mn,i),p(e,un,i),p(e,Ye,i),t(Ye,um),t(Ye,mr),t(mr,dm),t(Ye,ym),p(e,dn,i),p(e,ie,i),t(ie,Le),t(Le,ur),d(kl,ur,null),t(ie,vm),t(ie,dr),t(dr,wm),yn=!0},p:hv,i(e){yn||(y(Oe.$$.fragment,e),y(Ke.$$.fragment,e),y(lt.$$.fragment,e),y(it.$$.fragment,e),y(ut.$$.fragment,e),y(vt.$$.fragment,e),y(gt.$$.fragment,e),y(zt.$$.fragment,e),y(Ht.$$.fragment,e),y(Yt.$$.fragment,e),y(Qt.$$.fragment,e),y(xt.$$.fragment,e),y(Ot.$$.fragment,e),y(Kt.$$.fragment,e),y(el.$$.fragment,e),y(tl.$$.fragment,e),y(ll.$$.fragment,e),y(sl.$$.fragment,e),y(nl.$$.fragment,e),y(il.$$.fragment,e),y(pl.$$.fragment,e),y(hl.$$.fragment,e),y(cl.$$.fragment,e),y(fl.$$.fragment,e),y(wl.$$.fragment,e),y(Ml.$$.fragment,e),y(gl.$$.fragment,e),y(bl.$$.fragment,e),y(Jl.$$.fragment,e),y(Tl.$$.fragment,e),y(_l.$$.fragment,e),y(Ul.$$.fragment,e),y(jl.$$.fragment,e),y(Il.$$.fragment,e),y(kl.$$.fragment,e),yn=!0)},o(e){v(Oe.$$.fragment,e),v(Ke.$$.fragment,e),v(lt.$$.fragment,e),v(it.$$.fragment,e),v(ut.$$.fragment,e),v(vt.$$.fragment,e),v(gt.$$.fragment,e),v(zt.$$.fragment,e),v(Ht.$$.fragment,e),v(Yt.$$.fragment,e),v(Qt.$$.fragment,e),v(xt.$$.fragment,e),v(Ot.$$.fragment,e),v(Kt.$$.fragment,e),v(el.$$.fragment,e),v(tl.$$.fragment,e),v(ll.$$.fragment,e),v(sl.$$.fragment,e),v(nl.$$.fragment,e),v(il.$$.fragment,e),v(pl.$$.fragment,e),v(hl.$$.fragment,e),v(cl.$$.fragment,e),v(fl.$$.fragment,e),v(wl.$$.fragment,e),v(Ml.$$.fragment,e),v(gl.$$.fragment,e),v(bl.$$.fragment,e),v(Jl.$$.fragment,e),v(Tl.$$.fragment,e),v(_l.$$.fragment,e),v(Ul.$$.fragment,e),v(jl.$$.fragment,e),v(Il.$$.fragment,e),v(kl.$$.fragment,e),yn=!1},d(e){l(R),e&&l(_r),e&&l(C),w(Oe),e&&l(Ur),w(Ke,e),e&&l(jr),e&&l(et),e&&l(Ir),e&&l(V),w(lt),e&&l(Er),e&&l(ot),e&&l(kr),e&&l(fe),e&&l(Zr),e&&l(Gl),e&&l(Gr),e&&l(k),e&&l($r),w(it,e),e&&l(Br),e&&l(ue),e&&l(Sr),e&&l($l),e&&l(Wr),e&&l(X),e&&l(Ar),e&&l(ct),e&&l(Nr),e&&l(ft),e&&l(Rr),e&&l(z),w(ut),e&&l(Cr),e&&l(Sl),e&&l(Vr),e&&l(dt),e&&l(Xr),e&&l(Wl),e&&l(zr),e&&l(yt),e&&l(qr),e&&l(ve),e&&l(Hr),e&&l(q),w(vt),e&&l(Fr),e&&l(H),e&&l(Yr),e&&l(Z),e&&l(Lr),e&&l(Mt),e&&l(Pr),e&&l(F),w(gt),e&&l(Qr),e&&l(M),e&&l(Dr),e&&l(Vt),e&&l(xr),e&&l(Y),w(zt),e&&l(Or),e&&l(Je),e&&l(Kr),e&&l(Te),e&&l(es),e&&l(G),e&&l(ts),e&&l($),e&&l(ls),e&&l(B),e&&l(os),e&&l(S),e&&l(as),e&&l(qt),e&&l(rs),e&&l(L),w(Ht),e&&l(ss),e&&l(Xl),e&&l(ns),e&&l(W),e&&l(is),e&&l(Ue),e&&l(ps),e&&l(P),w(Yt),e&&l(hs),e&&l(Q),e&&l(cs),e&&l(Fl),e&&l(fs),e&&l(A),e&&l(ms),e&&l(Yl),e&&l(us),e&&l(g),e&&l(ds),e&&l(D),w(Qt),e&&l(ys),e&&l(Pl),e&&l(vs),e&&l(Ee),e&&l(ws),w(xt,e),e&&l(Ms),e&&l(x),w(Ot),e&&l(gs),e&&l(Ql),e&&l(bs),w(Kt,e),e&&l(Js),e&&l(O),w(el),e&&l(Ts),w(tl,e),e&&l(_s),e&&l(Ge),e&&l(Us),e&&l(K),w(ll),e&&l(js),e&&l(ol),e&&l(Is),e&&l(j),e&&l(Es),e&&l(N),e&&l(ks),e&&l(Dl),e&&l(Zs),e&&l(I),e&&l(Gs),e&&l(Be),e&&l($s),e&&l(Se),e&&l(Bs),e&&l(ee),w(sl),e&&l(Ss),e&&l(xl),e&&l(Ws),w(nl,e),e&&l(As),e&&l(te),w(il),e&&l(Ns),w(pl,e),e&&l(Rs),e&&l(le),w(hl),e&&l(Cs),w(cl,e),e&&l(Vs),e&&l(Ol),e&&l(Xs),e&&l(oe),w(fl),e&&l(zs),e&&l(Kl),e&&l(qs),e&&l(ml),e&&l(Hs),e&&l(eo),e&&l(Fs),e&&l(to),e&&l(Ys),e&&l(yl),e&&l(Ls),e&&l(Ve),e&&l(Ps),e&&l(vl),e&&l(Qs),w(wl,e),e&&l(Ds),w(Ml,e),e&&l(xs),e&&l(ae),w(gl),e&&l(Os),e&&l(lo),e&&l(Ks),w(bl,e),e&&l(en),w(Jl,e),e&&l(tn),w(Tl,e),e&&l(ln),w(_l,e),e&&l(on),e&&l(re),w(Ul),e&&l(an),e&&l(qe),e&&l(rn),e&&l(He),e&&l(sn),w(jl,e),e&&l(nn),e&&l(Fe),e&&l(pn),w(Il,e),e&&l(hn),e&&l(se),e&&l(cn),e&&l(El),e&&l(fn),e&&l(mn),e&&l(un),e&&l(Ye),e&&l(dn),e&&l(ie),w(kl)}}}const uv={local:"unit-8-part-2-advanced-deep-reinforcement-learning-using-sample-factory-to-play-doom-from-pixels",sections:[{local:"set-the-gpu",title:"Set the GPU \u{1F4AA}"},{local:"sample-factory",sections:[{local:"key-features",title:"Key features"},{local:"how-samplefactory-works",title:"How sample-factory works"},{local:"actor-critic-models-in-samplefactory",title:"Actor Critic models in Sample-factory"}],title:"Sample Factory"},{local:"vizdoom",title:"ViZDoom"},{local:"we-first-need-to-install-some-dependencies-that-are-required-for-the-vizdoom-environment",title:"We first need to install some dependencies that are required for the ViZDoom environment"},{local:"then-we-can-install-sample-factory-and-vizdoom",title:"Then we can install Sample Factory and ViZDoom"},{local:"setting-up-the-doom-environment-in-samplefactory",sections:[{local:"the-scenario-health-gathering-supreme",title:"The scenario: Health Gathering Supreme"}],title:"Setting up the Doom Environment in sample-factory"},{local:"training-the-agent",title:"Training the agent"},{local:"lets-take-a-look-at-the-performance-of-the-trained-policy-and-output-a-video-of-the-agent",title:"Let's take a look at the performance of the trained policy and output a video of the agent."},{local:"now-lets-visualize-the-performance-of-the-agent",title:"Now lets visualize the performance of the agent"},{local:"now-lets-upload-your-checkpoint-and-video-to-the-hugging-face-hub",title:"Now lets upload your checkpoint and video to the Hugging Face Hub"},{local:"lets-load-another-model",title:"Let's load another model"},{local:"some-additional-challenges-doom-deathmatch",title:"Some additional challenges \u{1F3C6}: Doom Deathmatch"},{local:"keep-learning-stay-awesome",title:"Keep learning, stay awesome \u{1F917}"}],title:"Unit 8 Part 2: Advanced Deep Reinforcement Learning. Using Sample Factory to play Doom from pixels"};function dv(Am){return cv(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class gv extends sv{constructor(R){super();nv(this,R,dv,mv,iv,{})}}export{gv as default,uv as metadata};
