import{S as Og,i as zg,s as Yg,e as o,k as f,w as c,t as i,M as xg,c as a,d as t,m as u,a as r,x as m,h as n,b as h,N as ne,G as l,g as p,y as d,L as Qg,q as v,o as y,B as g,v as qg}from"../../chunks/vendor-hf-doc-builder.js";import{I as w}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as _}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as Vg}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function Kg(Od){let G,gs,J,pe,xo,at,Up,Qo,Cp,ws,rt,_s,fe,Zp,qo,Hp,Wp,bs,st,zd,Es,ue,Xp,it,Dp,Op,$s,B,zp,Vo,Yp,xp,nt,Qp,ks,he,qp,Ko,Vp,Kp,Ts,pt,ef,ea,tf,Is,ao,ta,lf,Ms,ft,of,ut,af,Ls,ht,rf,ct,sf,As,mt,la,nf,pf,Rs,ro,dt,so,Yd,Ns,F,ce,oa,vt,ff,aa,uf,Ss,yt,xd,js,b,hf,ra,cf,mf,gt,df,vf,wt,yf,gf,Gs,me,wf,_t,_f,bf,Js,P,de,sa,bt,Ef,ia,$f,Bs,io,na,Et,kf,Fs,$t,Tf,kt,If,Ps,U,ve,pa,Tt,Mf,fa,Lf,Us,no,ua,It,Af,Cs,C,ye,ha,Mt,Rf,ca,Nf,Zs,po,Sf,Hs,ge,Lt,jf,ma,Gf,Jf,Bf,At,Ff,da,Pf,Uf,Ws,Z,we,va,Rt,Cf,ya,Zf,Xs,fo,Hf,Ds,_e,Wf,ga,Nt,Xf,Df,Os,A,Of,wa,zf,Yf,St,xf,Qf,zs,H,be,_a,jt,qf,ba,Vf,Ys,Ee,Kf,Ea,eu,tu,xs,$e,lu,$a,ou,au,Qs,ke,ru,ka,su,iu,qs,Gt,nu,Ta,pu,Vs,Jt,fu,Bt,uu,Ks,W,Te,Ia,Ft,hu,Ma,cu,ei,uo,Ie,mu,La,du,vu,Aa,yu,ti,Pt,Qd,li,ho,Ra,Na,gu,oi,Ut,qd,ai,X,Me,Sa,Ct,wu,ja,_u,ri,R,bu,Ga,Eu,$u,Ja,ku,Tu,si,Zt,ii,co,Iu,ni,Ht,pi,Wt,fi,mo,Mu,ui,Xt,hi,D,Le,Ba,Dt,Lu,Fa,Au,ci,Ae,Ru,Pa,Nu,Su,mi,vo,ju,di,Ot,vi,zt,yi,O,Re,Ua,Yt,Gu,Ca,Ju,gi,yo,Bu,wi,go,xt,Fu,Za,Pu,Uu,_i,wo,Cu,bi,Qt,Ei,_o,Zu,$i,E,qt,Hu,Ha,Wu,Xu,Du,Vt,Ou,Wa,zu,Yu,xu,bo,Qu,Xa,qu,Vu,Da,Ku,ki,$,eh,Oa,th,lh,za,oh,ah,Ya,rh,sh,Ti,Eo,ih,Ii,N,xa,Qa,nh,ph,qa,Va,fh,uh,Ka,er,hh,Mi,z,ch,tr,mh,dh,Kt,vh,Li,el,tl,yh,lr,gh,wh,Ai,$o,L,_h,or,bh,Eh,ar,$h,kh,rr,Th,Ih,Ri,ll,Ni,Y,Ne,sr,ol,Mh,ir,Lh,Si,al,ji,x,Se,nr,rl,Ah,pr,Rh,Gi,je,Q,Nh,fr,Sh,jh,ur,Gh,Jh,Bh,hr,Fh,Ji,sl,Bi,q,Ge,cr,il,Ph,mr,Uh,Fi,nl,Pi,V,Je,dr,pl,Ch,vr,Zh,Ui,fl,Vd,Ci,S,Hh,yr,Wh,Xh,gr,Dh,Oh,Zi,ko,zh,Hi,k,ul,Yh,wr,xh,Qh,qh,hl,Vh,_r,Kh,ec,tc,cl,lc,br,oc,ac,rc,Be,sc,Er,ic,nc,ml,pc,Wi,To,fc,Xi,dl,uc,vl,hc,Di,Io,cc,Oi,Mo,Fe,mc,yl,dc,vc,$r,yc,zi,gl,Kd,Yi,Pe,kr,gc,wc,Tr,_c,xi,wl,Qi,_l,bc,Ir,Ec,qi,Lo,$c,Vi,Ao,kc,Ki,bl,Mr,Tc,Ic,en,El,Lr,Mc,Lc,tn,K,Ar,Ac,Rc,Rr,Nc,Sc,ln,$l,ev,on,kl,an,ee,Ue,Nr,Tl,jc,Sr,Gc,rn,Il,sn,Ro,Jc,nn,Ce,Bc,Ml,Fc,Pc,pn,T,Ll,Uc,jr,Cc,Zc,Hc,Gr,Wc,Xc,Jr,Dc,Oc,Al,zc,Br,Yc,xc,fn,No,Qc,un,te,Fr,qc,Vc,Rl,Kc,em,hn,le,Ze,Pr,Nl,tm,Ur,lm,cn,So,Sl,om,Cr,am,rm,mn,jl,sm,Gl,im,dn,jo,nm,vn,I,Go,pm,Jl,fm,um,Jo,hm,Bl,cm,mm,Bo,dm,Fl,vm,ym,Fo,gm,Pl,wm,yn,Ul,_m,Cl,bm,gn,Po,He,Em,Zr,$m,km,Hr,Tm,wn,Zl,_n,Hl,Wr,Im,bn,Wl,En,Xl,Mm,Xr,Lm,$n,oe,Am,Dl,Rm,Nm,Dr,Sm,kn,Ol,jm,Or,Gm,Tn,ae,We,zr,zl,Jm,Yr,Bm,In,Xe,Fm,xr,Pm,Um,Mn,De,Cm,Yl,Zm,Hm,Ln,Uo,Wm,An,M,Qr,Xm,Dm,qr,Om,zm,Vr,Ym,xm,Kr,Qm,Rn,re,qm,es,Vm,Km,xl,ed,Nn,Ql,tv,Sn,jn,Gn,Co,td,Jn,ql,ld,ts,od,Bn,Oe,ad,ls,rd,sd,Fn,ze,id,Zo,nd,Vl,pd,fd,Pn,se,Ye,os,Kl,ud,as,hd,Un,eo,cd,to,md,Cn,j,dd,rs,vd,yd,lo,gd,wd,Zn,Ho,_d,Hn,ie,xe,ss,oo,bd,is,Ed,Wn;return at=new w({}),rt=new Vg({props:{classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/deep-rl-class/blob/main/notebooks/unit3/unit3.ipynb"}],askForHelpUrl:"http://hf.co/join/discord"}}),vt=new w({}),bt=new w({}),Tt=new w({}),Mt=new w({}),Rt=new w({}),jt=new w({}),Ft=new w({}),Ct=new w({}),Zt=new _({props:{code:"JTIzJTIwRm9yJTIwbm93JTIwd2UlMjBpbnN0YWxsJTIwdGhpcyUyMHVwZGF0ZSUyMG9mJTIwUkwtQmFzZWxpbmVzMyUyMFpvbyUwQXBpcCUyMGluc3RhbGwlMjBnaXQlMkJodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZETFItUk0lMkZybC1iYXNlbGluZXMzLXpvbyU0MHVwZGF0ZSUyRmhm",highlighted:`<span class="hljs-comment"># For now we install this update of RL-Baselines3 Zoo</span>
pip install git+https://github.com/DLR-RM/rl-baselines3-zoo@update/hf`}}),Ht=new _({props:{code:"JTIzcGlwJTIwaW5zdGFsbCUyMHJsX3pvbzMlM0QlM0QyLjAuMGE5",highlighted:'<span class="hljs-comment">#pip install rl_zoo3==2.0.0a9</span>'}}),Wt=new _({props:{code:"YXB0LWdldCUyMGluc3RhbGwlMjBzd2lnJTIwY21ha2UlMjBmZm1wZWc=",highlighted:"apt-get install swig cmake ffmpeg"}}),Xt=new _({props:{code:"IXBpcCUyMGluc3RhbGwlMjBneW1uYXNpdW0lNUJhdGFyaSU1RCUwQSFwaXAlMjBpbnN0YWxsJTIwZ3ltbmFzaXVtJTVCYWNjZXB0LXJvbS1saWNlbnNlJTVE",highlighted:`!pip install gymnasium[atari]
!pip install gymnasium[accept-rom-license]`}}),Dt=new w({}),Ot=new _({props:{code:"YXB0JTIwaW5zdGFsbCUyMHB5dGhvbi1vcGVuZ2wlMEFhcHQlMjBpbnN0YWxsJTIwZmZtcGVnJTBBYXB0JTIwaW5zdGFsbCUyMHh2ZmIlMEFwaXAzJTIwaW5zdGFsbCUyMHB5dmlydHVhbGRpc3BsYXk=",highlighted:`apt install python-opengl
apt install ffmpeg
apt install xvfb
pip3 install pyvirtualdisplay`}}),zt=new _({props:{code:"JTIzJTIwVmlydHVhbCUyMGRpc3BsYXklMEFmcm9tJTIwcHl2aXJ0dWFsZGlzcGxheSUyMGltcG9ydCUyMERpc3BsYXklMEElMEF2aXJ0dWFsX2Rpc3BsYXklMjAlM0QlMjBEaXNwbGF5KHZpc2libGUlM0QwJTJDJTIwc2l6ZSUzRCgxNDAwJTJDJTIwOTAwKSklMEF2aXJ0dWFsX2Rpc3BsYXkuc3RhcnQoKQ==",highlighted:`<span class="hljs-comment"># Virtual display</span>
<span class="hljs-keyword">from</span> pyvirtualdisplay <span class="hljs-keyword">import</span> Display

virtual_display = Display(visible=<span class="hljs-number">0</span>, size=(<span class="hljs-number">1400</span>, <span class="hljs-number">900</span>))
virtual_display.start()`}}),Yt=new w({}),Qt=new _({props:{code:"U3BhY2VJbnZhZGVyc05vRnJhbWVza2lwLXY0JTNBJTBBJTIwJTIwZW52X3dyYXBwZXIlM0ElMEElMjAlMjAlMjAlMjAtJTIwc3RhYmxlX2Jhc2VsaW5lczMuY29tbW9uLmF0YXJpX3dyYXBwZXJzLkF0YXJpV3JhcHBlciUwQSUyMCUyMGZyYW1lX3N0YWNrJTNBJTIwNCUwQSUyMCUyMHBvbGljeSUzQSUyMCdDbm5Qb2xpY3knJTBBJTIwJTIwbl90aW1lc3RlcHMlM0ElMjAhIWZsb2F0JTIwMWU3JTBBJTIwJTIwYnVmZmVyX3NpemUlM0ElMjAxMDAwMDAlMEElMjAlMjBsZWFybmluZ19yYXRlJTNBJTIwISFmbG9hdCUyMDFlLTQlMEElMjAlMjBiYXRjaF9zaXplJTNBJTIwMzIlMEElMjAlMjBsZWFybmluZ19zdGFydHMlM0ElMjAxMDAwMDAlMEElMjAlMjB0YXJnZXRfdXBkYXRlX2ludGVydmFsJTNBJTIwMTAwMCUwQSUyMCUyMHRyYWluX2ZyZXElM0ElMjA0JTBBJTIwJTIwZ3JhZGllbnRfc3RlcHMlM0ElMjAxJTBBJTIwJTIwZXhwbG9yYXRpb25fZnJhY3Rpb24lM0ElMjAwLjElMEElMjAlMjBleHBsb3JhdGlvbl9maW5hbF9lcHMlM0ElMjAwLjAxJTBBJTIwJTIwJTIzJTIwSWYlMjBUcnVlJTJDJTIweW91JTIwbmVlZCUyMHRvJTIwZGVhY3RpdmF0ZSUyMGhhbmRsZV90aW1lb3V0X3Rlcm1pbmF0aW9uJTBBJTIwJTIwJTIzJTIwaW4lMjB0aGUlMjByZXBsYXlfYnVmZmVyX2t3YXJncyUwQSUyMCUyMG9wdGltaXplX21lbW9yeV91c2FnZSUzQSUyMEZhbHNl",highlighted:`<span class="hljs-attr">SpaceInvadersNoFrameskip-v4:</span>
  <span class="hljs-attr">env_wrapper:</span>
    <span class="hljs-bullet">-</span> <span class="hljs-string">stable_baselines3.common.atari_wrappers.AtariWrapper</span>
  <span class="hljs-attr">frame_stack:</span> <span class="hljs-number">4</span>
  <span class="hljs-attr">policy:</span> <span class="hljs-string">&#x27;CnnPolicy&#x27;</span>
  <span class="hljs-attr">n_timesteps:</span> <span class="hljs-type">!!float</span> <span class="hljs-number">1e7</span>
  <span class="hljs-attr">buffer_size:</span> <span class="hljs-number">100000</span>
  <span class="hljs-attr">learning_rate:</span> <span class="hljs-type">!!float</span> <span class="hljs-number">1e-4</span>
  <span class="hljs-attr">batch_size:</span> <span class="hljs-number">32</span>
  <span class="hljs-attr">learning_starts:</span> <span class="hljs-number">100000</span>
  <span class="hljs-attr">target_update_interval:</span> <span class="hljs-number">1000</span>
  <span class="hljs-attr">train_freq:</span> <span class="hljs-number">4</span>
  <span class="hljs-attr">gradient_steps:</span> <span class="hljs-number">1</span>
  <span class="hljs-attr">exploration_fraction:</span> <span class="hljs-number">0.1</span>
  <span class="hljs-attr">exploration_final_eps:</span> <span class="hljs-number">0.01</span>
  <span class="hljs-comment"># If True, you need to deactivate handle_timeout_termination</span>
  <span class="hljs-comment"># in the replay_buffer_kwargs</span>
  <span class="hljs-attr">optimize_memory_usage:</span> <span class="hljs-literal">False</span>`}}),ll=new _({props:{code:"cHl0aG9uJTIwLW0lMjBybF96b28zLnRyYWluJTIwLS1hbGdvJTIwX19fX19fX18lMjAtLWVudiUyMFNwYWNlSW52YWRlcnNOb0ZyYW1lc2tpcC12NCUyMCUyMC1mJTIwX19fX19fX19fJTIwJTIwLWMlMjBfX19fX19fX18=",highlighted:'python -m rl_zoo3.train --algo ________ --<span class="hljs-built_in">env</span> SpaceInvadersNoFrameskip-v4  -f _________  -c _________'}}),ol=new w({}),al=new _({props:{code:"cHl0aG9uJTIwLW0lMjBybF96b28zLnRyYWluJTIwLS1hbGdvJTIwZHFuJTIwJTIwLS1lbnYlMjBTcGFjZUludmFkZXJzTm9GcmFtZXNraXAtdjQlMjAtZiUyMGxvZ3MlMkYlMjAtYyUyMGRxbi55bWw=",highlighted:'python -m rl_zoo3.train --algo dqn  --<span class="hljs-built_in">env</span> SpaceInvadersNoFrameskip-v4 -f logs/ -c dqn.yml'}}),rl=new w({}),sl=new _({props:{code:"cHl0aG9uJTIwLW0lMjBybF96b28zLmVuam95JTIwJTIwLS1hbGdvJTIwZHFuJTIwJTIwLS1lbnYlMjBTcGFjZUludmFkZXJzTm9GcmFtZXNraXAtdjQlMjAlMjAtLW5vLXJlbmRlciUyMCUyMC0tbi10aW1lc3RlcHMlMjBfX19fX19fX18lMjAlMjAtLWZvbGRlciUyMGxvZ3MlMkY=",highlighted:'python -m rl_zoo3.enjoy  --algo dqn  --<span class="hljs-built_in">env</span> SpaceInvadersNoFrameskip-v4  --no-render  --n-timesteps _________  --folder logs/'}}),il=new w({}),nl=new _({props:{code:"cHl0aG9uJTIwLW0lMjBybF96b28zLmVuam95JTIwJTIwLS1hbGdvJTIwZHFuJTIwJTIwLS1lbnYlMjBTcGFjZUludmFkZXJzTm9GcmFtZXNraXAtdjQlMjAlMjAtLW5vLXJlbmRlciUyMCUyMC0tbi10aW1lc3RlcHMlMjA1MDAwJTIwJTIwLS1mb2xkZXIlMjBsb2dzJTJG",highlighted:'python -m rl_zoo3.enjoy  --algo dqn  --<span class="hljs-built_in">env</span> SpaceInvadersNoFrameskip-v4  --no-render  --n-timesteps 5000  --folder logs/'}}),pl=new w({}),wl=new _({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTIwJTIzJTIwVG8lMjBsb2clMjB0byUyMG91ciUyMEh1Z2dpbmclMjBGYWNlJTIwYWNjb3VudCUyMHRvJTIwYmUlMjBhYmxlJTIwdG8lMjB1cGxvYWQlMjBtb2RlbHMlMjB0byUyMHRoZSUyMEh1Yi4lMEFub3RlYm9va19sb2dpbigpJTBBIWdpdCUyMGNvbmZpZyUyMC0tZ2xvYmFsJTIwY3JlZGVudGlhbC5oZWxwZXIlMjBzdG9yZQ==",highlighted:`from huggingface_hub import notebook_login <span class="hljs-comment"># To log to our Hugging Face account to be able to upload models to the Hub.</span>
notebook_login()
!git config --global credential.helper store`}}),kl=new _({props:{code:"cHl0aG9uJTIwLW0lMjBybF96b28zLnB1c2hfdG9faHViJTIwJTIwLS1hbGdvJTIwZHFuJTIwJTIwLS1lbnYlMjBTcGFjZUludmFkZXJzTm9GcmFtZXNraXAtdjQlMjAlMjAtLXJlcG8tbmFtZSUyMF9fX19fX19fX19fX19fX19fX19fXyUyMC1vcmdhJTIwX19fX19fX19fX19fX19fX19fX19fJTIwLWYlMjBsb2dzJTJG",highlighted:'python -m rl_zoo3.push_to_hub  --algo dqn  --<span class="hljs-built_in">env</span> SpaceInvadersNoFrameskip-v4  --repo-name _____________________ -orga _____________________ -f logs/'}}),Tl=new w({}),Il=new _({props:{code:"cHl0aG9uJTIwLW0lMjBybF96b28zLnB1c2hfdG9faHViJTIwJTIwLS1hbGdvJTIwZHFuJTIwJTIwLS1lbnYlMjBTcGFjZUludmFkZXJzTm9GcmFtZXNraXAtdjQlMjAlMjAtLXJlcG8tbmFtZSUyMGRxbi1TcGFjZUludmFkZXJzTm9GcmFtZXNraXAtdjQlMjAlMjAtb3JnYSUyMFRob21hc1NpbW9uaW5pJTIwJTIwLWYlMjBsb2dzJTJG",highlighted:'python -m rl_zoo3.push_to_hub  --algo dqn  --<span class="hljs-built_in">env</span> SpaceInvadersNoFrameskip-v4  --repo-name dqn-SpaceInvadersNoFrameskip-v4  -orga ThomasSimonini  -f logs/'}}),Nl=new w({}),Zl=new _({props:{code:"JTIzJTIwRG93bmxvYWQlMjBtb2RlbCUyMGFuZCUyMHNhdmUlMjBpdCUyMGludG8lMjB0aGUlMjBsb2dzJTJGJTIwZm9sZGVyJTBBcHl0aG9uJTIwLW0lMjBybF96b28zLmxvYWRfZnJvbV9odWIlMjAtLWFsZ28lMjBkcW4lMjAtLWVudiUyMEJlYW1SaWRlck5vRnJhbWVza2lwLXY0JTIwLW9yZ2ElMjBzYjMlMjAtZiUyMHJsX3RyYWluZWQlMkY=",highlighted:`<span class="hljs-comment"># Download model and save it into the logs/ folder</span>
python -m rl_zoo3.load_from_hub --algo dqn --<span class="hljs-built_in">env</span> BeamRiderNoFrameskip-v4 -orga sb3 -f rl_trained/`}}),Wl=new _({props:{code:"cHl0aG9uJTIwLW0lMjBybF96b28zLmVuam95JTIwLS1hbGdvJTIwZHFuJTIwLS1lbnYlMjBCZWFtUmlkZXJOb0ZyYW1lc2tpcC12NCUyMC1uJTIwNTAwMCUyMCUyMC1mJTIwcmxfdHJhaW5lZCUyRiUyMC0tbm8tcmVuZGVy",highlighted:'python -m rl_zoo3.enjoy --algo dqn --<span class="hljs-built_in">env</span> BeamRiderNoFrameskip-v4 -n 5000  -f rl_trained/ --no-render'}}),zl=new w({}),Kl=new w({}),oo=new w({}),{c(){G=o("meta"),gs=f(),J=o("h1"),pe=o("a"),xo=o("span"),c(at.$$.fragment),Up=f(),Qo=o("span"),Cp=i("Hands-on"),ws=f(),c(rt.$$.fragment),_s=f(),fe=o("p"),Zp=i("Now that you\u2019ve studied the theory behind Deep Q-Learning, "),qo=o("strong"),Hp=i("you\u2019re ready to train your Deep Q-Learning agent to play Atari Games"),Wp=i(". We\u2019ll start with Space Invaders, but you\u2019ll be able to use any Atari game you want \u{1F525}"),bs=f(),st=o("img"),Es=f(),ue=o("p"),Xp=i("We\u2019re using the "),it=o("a"),Dp=i("RL-Baselines-3 Zoo integration"),Op=i(", a vanilla version of Deep Q-Learning with no extensions such as Double-DQN, Dueling-DQN, or Prioritized Experience Replay."),$s=f(),B=o("p"),zp=i("Also, "),Vo=o("strong"),Yp=i("if you want to learn to implement Deep Q-Learning by yourself after this hands-on"),xp=i(", you definitely should look at the CleanRL implementation: "),nt=o("a"),Qp=i("https://github.com/vwxyzjn/cleanrl/blob/master/cleanrl/dqn_atari.py"),ks=f(),he=o("p"),qp=i("To validate this hands-on for the certification process, you need to push your trained model to the Hub and "),Ko=o("strong"),Vp=i("get a result of >= 200"),Kp=i("."),Ts=f(),pt=o("p"),ef=i("To find your result, go to the leaderboard and find your model, "),ea=o("strong"),tf=i("the result = mean_reward - std of reward"),Is=f(),ao=o("p"),ta=o("strong"),lf=i("If you don\u2019t find your model, go to the bottom of the page and click on the refresh button."),Ms=f(),ft=o("p"),of=i("For more information about the certification process, check this section \u{1F449} "),ut=o("a"),af=i("https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),Ls=f(),ht=o("p"),rf=i("And you can check your progress here \u{1F449} "),ct=o("a"),sf=i("https://huggingface.co/spaces/ThomasSimonini/Check-my-progress-Deep-RL-Course"),As=f(),mt=o("p"),la=o("strong"),nf=i("To start the hands-on click on Open In Colab button"),pf=i(" \u{1F447} :"),Rs=f(),ro=o("p"),dt=o("a"),so=o("img"),Ns=f(),F=o("h1"),ce=o("a"),oa=o("span"),c(vt.$$.fragment),ff=f(),aa=o("span"),uf=i("Unit 3: Deep Q-Learning with Atari Games \u{1F47E} using RL Baselines3 Zoo"),Ss=f(),yt=o("img"),js=f(),b=o("p"),hf=i("In this hands-on, "),ra=o("strong"),cf=i("you\u2019ll train a Deep Q-Learning agent"),mf=i(" playing Space Invaders using "),gt=o("a"),df=i("RL Baselines3 Zoo"),vf=i(", a training framework based on "),wt=o("a"),yf=i("Stable-Baselines3"),gf=i(" that provides scripts for training, evaluating agents, tuning hyperparameters, plotting results and recording videos."),Gs=f(),me=o("p"),wf=i("We\u2019re using the "),_t=o("a"),_f=i("RL-Baselines-3 Zoo integration, a vanilla version of Deep Q-Learning"),bf=i(" with no extensions such as Double-DQN, Dueling-DQN, and Prioritized Experience Replay."),Js=f(),P=o("h3"),de=o("a"),sa=o("span"),c(bt.$$.fragment),Ef=f(),ia=o("span"),$f=i("\u{1F3AE} Environments:"),Bs=f(),io=o("ul"),na=o("li"),Et=o("a"),kf=i("SpacesInvadersNoFrameskip-v4"),Fs=f(),$t=o("p"),Tf=i("You can see the difference between Space Invaders versions here \u{1F449} "),kt=o("a"),If=i("https://gymnasium.farama.org/environments/atari/space_invaders/#variants"),Ps=f(),U=o("h3"),ve=o("a"),pa=o("span"),c(Tt.$$.fragment),Mf=f(),fa=o("span"),Lf=i("\u{1F4DA} RL-Library:"),Us=f(),no=o("ul"),ua=o("li"),It=o("a"),Af=i("RL-Baselines3-Zoo"),Cs=f(),C=o("h2"),ye=o("a"),ha=o("span"),c(Mt.$$.fragment),Rf=f(),ca=o("span"),Nf=i("Objectives of this hands-on \u{1F3C6}"),Zs=f(),po=o("p"),Sf=i("At the end of the hands-on, you will:"),Hs=f(),ge=o("ul"),Lt=o("li"),jf=i("Be able to understand deeper "),ma=o("strong"),Gf=i("how RL Baselines3 Zoo works"),Jf=i("."),Bf=f(),At=o("li"),Ff=i("Be able to "),da=o("strong"),Pf=i("push your trained agent and the code to the Hub"),Uf=i(" with a nice video replay and an evaluation score \u{1F525}."),Ws=f(),Z=o("h2"),we=o("a"),va=o("span"),c(Rt.$$.fragment),Cf=f(),ya=o("span"),Zf=i("Prerequisites \u{1F3D7}\uFE0F"),Xs=f(),fo=o("p"),Hf=i("Before diving into the hands-on, you need to:"),Ds=f(),_e=o("p"),Wf=i("\u{1F532} \u{1F4DA} "),ga=o("strong"),Nt=o("a"),Xf=i("Study Deep Q-Learning by reading Unit 3"),Df=i("  \u{1F917}"),Os=f(),A=o("p"),Of=i("We\u2019re constantly trying to improve our tutorials, so "),wa=o("strong"),zf=i("if you find some issues in this hands-on"),Yf=i(", please "),St=o("a"),xf=i("open an issue on the Github Repo"),Qf=i("."),zs=f(),H=o("h1"),be=o("a"),_a=o("span"),c(jt.$$.fragment),qf=f(),ba=o("span"),Vf=i("Let's train a Deep Q-Learning agent playing Atari' Space Invaders \u{1F47E} and upload it to the Hub."),Ys=f(),Ee=o("p"),Kf=i("We strongly recommend students "),Ea=o("strong"),eu=i("to use Google Colab for the hands-on exercises instead of running them on their personal computers"),tu=i("."),xs=f(),$e=o("p"),lu=i("By using Google Colab, "),$a=o("strong"),ou=i("you can focus on learning and experimenting without worrying about the technical aspects of setting up your environments"),au=i("."),Qs=f(),ke=o("p"),ru=i("To validate this hands-on for the certification process, you need to push your trained model to the Hub and "),ka=o("strong"),su=i("get a result of >= 200"),iu=i("."),qs=f(),Gt=o("p"),nu=i("To find your result, go to the leaderboard and find your model, "),Ta=o("strong"),pu=i("the result = mean_reward - std of reward"),Vs=f(),Jt=o("p"),fu=i("For more information about the certification process, check this section \u{1F449} "),Bt=o("a"),uu=i("https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),Ks=f(),W=o("h2"),Te=o("a"),Ia=o("span"),c(Ft.$$.fragment),hu=f(),Ma=o("span"),cu=i("Set the GPU \u{1F4AA}"),ei=f(),uo=o("ul"),Ie=o("li"),mu=i("To "),La=o("strong"),du=i("accelerate the agent\u2019s training, we\u2019ll use a GPU"),vu=i(". To do that, go to "),Aa=o("code"),yu=i("Runtime > Change Runtime type"),ti=f(),Pt=o("img"),li=f(),ho=o("ul"),Ra=o("li"),Na=o("code"),gu=i("Hardware Accelerator > GPU"),oi=f(),Ut=o("img"),ai=f(),X=o("h1"),Me=o("a"),Sa=o("span"),c(Ct.$$.fragment),wu=f(),ja=o("span"),_u=i("Install RL-Baselines3 Zoo and its dependencies \u{1F4DA}"),ri=f(),R=o("p"),bu=i("If you see "),Ga=o("code"),Eu=i("ERROR: pip's dependency resolver does not currently take into account all the packages that are installed."),$u=f(),Ja=o("strong"),ku=i("this is normal and it\u2019s not a critical error"),Tu=i(" there\u2019s a conflict of version. But the packages we need are installed."),si=f(),c(Zt.$$.fragment),ii=f(),co=o("p"),Iu=i("IF AND ONLY IF THE VERSION ABOVE DOES NOT EXIST ANYMORE. UNCOMMENT AND INSTALL THE ONE BELOW"),ni=f(),c(Ht.$$.fragment),pi=f(),c(Wt.$$.fragment),fi=f(),mo=o("p"),Mu=i("To be able to use Atari games in Gymnasium we need to install atari package. And accept-rom-license to download the rom files (games files)."),ui=f(),c(Xt.$$.fragment),hi=f(),D=o("h2"),Le=o("a"),Ba=o("span"),c(Dt.$$.fragment),Lu=f(),Fa=o("span"),Au=i("Create a virtual display \u{1F53D}"),ci=f(),Ae=o("p"),Ru=i("During the hands-on, we\u2019ll need to generate a replay video. To do so, if you train it on a headless machine, "),Pa=o("strong"),Nu=i("we need to have a virtual screen to be able to render the environment"),Su=i(" (and thus record the frames)."),mi=f(),vo=o("p"),ju=i("Hence the following cell will install the librairies and create and run a virtual screen \u{1F5A5}"),di=f(),c(Ot.$$.fragment),vi=f(),c(zt.$$.fragment),yi=f(),O=o("h2"),Re=o("a"),Ua=o("span"),c(Yt.$$.fragment),Gu=f(),Ca=o("span"),Ju=i("Train our Deep Q-Learning Agent to Play Space Invaders \u{1F47E}"),gi=f(),yo=o("p"),Bu=i("To train an agent with RL-Baselines3-Zoo, we just need to do two things:"),wi=f(),go=o("ol"),xt=o("li"),Fu=i("Create a hyperparameter config file that will contain our training hyperparameters called "),Za=o("code"),Pu=i("dqn.yml"),Uu=i("."),_i=f(),wo=o("p"),Cu=i("This is a template example:"),bi=f(),c(Qt.$$.fragment),Ei=f(),_o=o("p"),Zu=i("Here we see that:"),$i=f(),E=o("ul"),qt=o("li"),Hu=i("We use the "),Ha=o("code"),Wu=i("Atari Wrapper"),Xu=i(" that preprocess the input (Frame reduction ,grayscale, stack 4 frames)"),Du=f(),Vt=o("li"),Ou=i("We use "),Wa=o("code"),zu=i("CnnPolicy"),Yu=i(", since we use Convolutional layers to process the frames"),xu=f(),bo=o("li"),Qu=i("We train it for 10 million "),Xa=o("code"),qu=i("n_timesteps"),Vu=f(),Da=o("li"),Ku=i("Memory (Experience Replay) size is 100000, aka the amount of experience steps you saved to train again your agent with."),ki=f(),$=o("p"),eh=i("\u{1F4A1} My advice is to "),Oa=o("strong"),th=i("reduce the training timesteps to 1M,"),lh=i(" which will take about 90 minutes on a P100. "),za=o("code"),oh=i("!nvidia-smi"),ah=i(" will tell you what GPU you\u2019re using. At 10 million steps, this will take about 9 hours. I recommend running this on your local computer (or somewhere else). Just click on: "),Ya=o("code"),rh=i("File>Download"),sh=i("."),Ti=f(),Eo=o("p"),ih=i("In terms of hyperparameters optimization, my advice is to focus on these 3 hyperparameters:"),Ii=f(),N=o("ul"),xa=o("li"),Qa=o("code"),nh=i("learning_rate"),ph=f(),qa=o("li"),Va=o("code"),fh=i("buffer_size (Experience Memory size)"),uh=f(),Ka=o("li"),er=o("code"),hh=i("batch_size"),Mi=f(),z=o("p"),ch=i("As a good practice, you need to "),tr=o("strong"),mh=i("check the documentation to understand what each hyperparameters does"),dh=i(": "),Kt=o("a"),vh=i("https://stable-baselines3.readthedocs.io/en/master/modules/dqn.html#parameters"),Li=f(),el=o("ol"),tl=o("li"),yh=i("We start the training and save the models on "),lr=o("code"),gh=i("logs"),wh=i(" folder \u{1F4C1}"),Ai=f(),$o=o("ul"),L=o("li"),_h=i("Define the algorithm after "),or=o("code"),bh=i("--algo"),Eh=i(", where we save the model after "),ar=o("code"),$h=i("-f"),kh=i(" and where the hyperparameter config is after "),rr=o("code"),Th=i("-c"),Ih=i("."),Ri=f(),c(ll.$$.fragment),Ni=f(),Y=o("h4"),Ne=o("a"),sr=o("span"),c(ol.$$.fragment),Mh=f(),ir=o("span"),Lh=i("Solution"),Si=f(),c(al.$$.fragment),ji=f(),x=o("h2"),Se=o("a"),nr=o("span"),c(rl.$$.fragment),Ah=f(),pr=o("span"),Rh=i("Let's evaluate our agent \u{1F440}"),Gi=f(),je=o("ul"),Q=o("li"),Nh=i("RL-Baselines3-Zoo provides "),fr=o("code"),Sh=i("enjoy.py"),jh=i(", a python script to evaluate our agent. In most RL libraries, we call the evaluation script "),ur=o("code"),Gh=i("enjoy.py"),Jh=i("."),Bh=f(),hr=o("li"),Fh=i("Let\u2019s evaluate it for 5000 timesteps \u{1F525}"),Ji=f(),c(sl.$$.fragment),Bi=f(),q=o("h4"),Ge=o("a"),cr=o("span"),c(il.$$.fragment),Ph=f(),mr=o("span"),Uh=i("Solution"),Fi=f(),c(nl.$$.fragment),Pi=f(),V=o("h2"),Je=o("a"),dr=o("span"),c(pl.$$.fragment),Ch=f(),vr=o("span"),Zh=i("Publish our trained model on the Hub \u{1F680}"),Ui=i(`

Now that we saw we got good results after the training, we can publish our trained model on the hub \u{1F917} with one line of code.
`),fl=o("img"),Ci=f(),S=o("p"),Hh=i("By using "),yr=o("code"),Wh=i("rl_zoo3.push_to_hub"),Xh=f(),gr=o("strong"),Dh=i("you evaluate, record a replay, generate a model card of your agent and push it to the hub"),Oh=i("."),Zi=f(),ko=o("p"),zh=i("This way:"),Hi=f(),k=o("ul"),ul=o("li"),Yh=i("You can "),wr=o("strong"),xh=i("showcase our work"),Qh=i(" \u{1F525}"),qh=f(),hl=o("li"),Vh=i("You can "),_r=o("strong"),Kh=i("visualize your agent playing"),ec=i(" \u{1F440}"),tc=f(),cl=o("li"),lc=i("You can "),br=o("strong"),oc=i("share with the community an agent that others can use"),ac=i(" \u{1F4BE}"),rc=f(),Be=o("li"),sc=i("You can "),Er=o("strong"),ic=i("access a leaderboard \u{1F3C6} to see how well your agent is performing compared to your classmates"),nc=i(" \u{1F449}  "),ml=o("a"),pc=i("https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),Wi=f(),To=o("p"),fc=i("To be able to share your model with the community there are three more steps to follow:"),Xi=f(),dl=o("p"),uc=i("1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),vl=o("a"),hc=i("https://huggingface.co/join"),Di=f(),Io=o("p"),cc=i("2\uFE0F\u20E3 Sign in and then, you need to store your authentication token from the Hugging Face website."),Oi=f(),Mo=o("ul"),Fe=o("li"),mc=i("Create a new token ("),yl=o("a"),dc=i("https://huggingface.co/settings/tokens"),vc=i(") "),$r=o("strong"),yc=i("with write role"),zi=f(),gl=o("img"),Yi=f(),Pe=o("ul"),kr=o("li"),gc=i("Copy the token"),wc=f(),Tr=o("li"),_c=i("Run the cell below and past the token"),xi=f(),c(wl.$$.fragment),Qi=f(),_l=o("p"),bc=i("If you don\u2019t want to use a Google Colab or a Jupyter Notebook, you need to use this command instead: "),Ir=o("code"),Ec=i("huggingface-cli login"),qi=f(),Lo=o("p"),$c=i("3\uFE0F\u20E3 We\u2019re now ready to push our trained agent to the \u{1F917} Hub \u{1F525}"),Vi=f(),Ao=o("p"),kc=i("Let\u2019s run push_to_hub.py file to upload our trained agent to the Hub."),Ki=f(),bl=o("p"),Mr=o("code"),Tc=i("--repo-name "),Ic=i(": The name of the repo"),en=f(),El=o("p"),Lr=o("code"),Mc=i("-orga"),Lc=i(": Your Hugging Face username"),tn=f(),K=o("p"),Ar=o("code"),Ac=i("-f"),Rc=i(": Where the trained model folder is (in our case "),Rr=o("code"),Nc=i("logs"),Sc=i(")"),ln=f(),$l=o("img"),on=f(),c(kl.$$.fragment),an=f(),ee=o("h4"),Ue=o("a"),Nr=o("span"),c(Tl.$$.fragment),jc=f(),Sr=o("span"),Gc=i("Solution"),rn=f(),c(Il.$$.fragment),sn=f(),Ro=o("p"),Jc=i("###."),nn=f(),Ce=o("p"),Bc=i("Congrats \u{1F973} you\u2019ve just trained and uploaded your first Deep Q-Learning agent using RL-Baselines-3 Zoo. The script above should have displayed a link to a model repository such as "),Ml=o("a"),Fc=i("https://huggingface.co/ThomasSimonini/dqn-SpaceInvadersNoFrameskip-v4"),Pc=i(". When you go to this link, you can:"),pn=f(),T=o("ul"),Ll=o("li"),Uc=i("See a "),jr=o("strong"),Cc=i("video preview of your agent"),Zc=i(" at the right."),Hc=f(),Gr=o("li"),Wc=i("Click \u201CFiles and versions\u201D to see all the files in the repository."),Xc=f(),Jr=o("li"),Dc=i("Click \u201CUse in stable-baselines3\u201D to get a code snippet that shows how to load the model."),Oc=f(),Al=o("li"),zc=i("A model card ("),Br=o("code"),Yc=i("README.md"),xc=i(" file) which gives a description of the model and the hyperparameters you used."),fn=f(),No=o("p"),Qc=i("Under the hood, the Hub uses git-based repositories (don\u2019t worry if you don\u2019t know what git is), which means you can update the model with new versions as you experiment and improve your agent."),un=f(),te=o("p"),Fr=o("strong"),qc=i("Compare the results of your agents with your classmates"),Vc=i(" using the "),Rl=o("a"),Kc=i("leaderboard"),em=i(" \u{1F3C6}"),hn=f(),le=o("h2"),Ze=o("a"),Pr=o("span"),c(Nl.$$.fragment),tm=f(),Ur=o("span"),lm=i("Load a powerful trained model \u{1F525}"),cn=f(),So=o("ul"),Sl=o("li"),om=i("The Stable-Baselines3 team uploaded "),Cr=o("strong"),am=i("more than 150 trained Deep Reinforcement Learning agents on the Hub"),rm=i("."),mn=f(),jl=o("p"),sm=i("You can find them here: \u{1F449} "),Gl=o("a"),im=i("https://huggingface.co/sb3"),dn=f(),jo=o("p"),nm=i("Some examples:"),vn=f(),I=o("ul"),Go=o("li"),pm=i("Asteroids: "),Jl=o("a"),fm=i("https://huggingface.co/sb3/dqn-AsteroidsNoFrameskip-v4"),um=f(),Jo=o("li"),hm=i("Beam Rider: "),Bl=o("a"),cm=i("https://huggingface.co/sb3/dqn-BeamRiderNoFrameskip-v4"),mm=f(),Bo=o("li"),dm=i("Breakout: "),Fl=o("a"),vm=i("https://huggingface.co/sb3/dqn-BreakoutNoFrameskip-v4"),ym=f(),Fo=o("li"),gm=i("Road Runner: "),Pl=o("a"),wm=i("https://huggingface.co/sb3/dqn-RoadRunnerNoFrameskip-v4"),yn=f(),Ul=o("p"),_m=i("Let\u2019s load an agent playing Beam Rider: "),Cl=o("a"),bm=i("https://huggingface.co/sb3/dqn-BeamRiderNoFrameskip-v4"),gn=f(),Po=o("ol"),He=o("li"),Em=i("We download the model using "),Zr=o("code"),$m=i("rl_zoo3.load_from_hub"),km=i(", and place it in a new folder that we can call "),Hr=o("code"),Tm=i("rl_trained"),wn=f(),c(Zl.$$.fragment),_n=f(),Hl=o("ol"),Wr=o("li"),Im=i("Let\u2019s evaluate if for 5000 timesteps"),bn=f(),c(Wl.$$.fragment),En=f(),Xl=o("p"),Mm=i("Why not trying to train your own "),Xr=o("strong"),Lm=i("Deep Q-Learning Agent playing BeamRiderNoFrameskip-v4? \u{1F3C6}."),$n=f(),oe=o("p"),Am=i("If you want to try, check "),Dl=o("a"),Rm=i("https://huggingface.co/sb3/dqn-BeamRiderNoFrameskip-v4#hyperparameters"),Nm=f(),Dr=o("strong"),Sm=i("in the model card, you have the hyperparameters of the trained agent."),kn=f(),Ol=o("p"),jm=i("But finding hyperparameters can be a daunting task. Fortunately, we\u2019ll see in the next Unit, how we can "),Or=o("strong"),Gm=i("use Optuna for optimizing the Hyperparameters \u{1F525}."),Tn=f(),ae=o("h2"),We=o("a"),zr=o("span"),c(zl.$$.fragment),Jm=f(),Yr=o("span"),Bm=i("Some additional challenges \u{1F3C6}"),In=f(),Xe=o("p"),Fm=i("The best way to learn "),xr=o("strong"),Pm=i("is to try things by your own"),Um=i("!"),Mn=f(),De=o("p"),Cm=i("In the "),Yl=o("a"),Zm=i("Leaderboard"),Hm=i(" you will find your agents. Can you get to the top?"),Ln=f(),Uo=o("p"),Wm=i("Here\u2019s a list of environments you can try to train your agent with:"),An=f(),M=o("ul"),Qr=o("li"),Xm=i("BeamRiderNoFrameskip-v4"),Dm=f(),qr=o("li"),Om=i("BreakoutNoFrameskip-v4"),zm=f(),Vr=o("li"),Ym=i("EnduroNoFrameskip-v4"),xm=f(),Kr=o("li"),Qm=i("PongNoFrameskip-v4"),Rn=f(),re=o("p"),qm=i("Also, "),es=o("strong"),Vm=i("if you want to learn to implement Deep Q-Learning by yourself"),Km=i(", you definitely should look at CleanRL implementation: "),xl=o("a"),ed=i("https://github.com/vwxyzjn/cleanrl/blob/master/cleanrl/dqn_atari.py"),Nn=f(),Ql=o("img"),Sn=f(),jn=o("hr"),Gn=f(),Co=o("p"),td=i("Congrats on finishing this chapter!"),Jn=f(),ql=o("p"),ld=i("If you\u2019re still feel confused with all these elements\u2026it\u2019s totally normal! "),ts=o("strong"),od=i("This was the same for me and for all people who studied RL."),Bn=f(),Oe=o("p"),ad=i("Take time to really "),ls=o("strong"),rd=i("grasp the material before continuing and try the additional challenges"),sd=i(". It\u2019s important to master these elements and having a solid foundations."),Fn=f(),ze=o("p"),id=i("In the next unit, "),Zo=o("strong"),nd=i("we\u2019re going to learn about "),Vl=o("a"),pd=i("Optuna"),fd=i(". One of the most critical task in Deep Reinforcement Learning is to find a good set of training hyperparameters. And Optuna is a library that helps you to automate the search."),Pn=f(),se=o("h3"),Ye=o("a"),os=o("span"),c(Kl.$$.fragment),ud=f(),as=o("span"),hd=i("This is a course built with you \u{1F477}\u{1F3FF}\u200D\u2640\uFE0F"),Un=f(),eo=o("p"),cd=i("Finally, we want to improve and update the course iteratively with your feedback. If you have some, please fill this form \u{1F449} "),to=o("a"),md=i("https://forms.gle/3HgA7bEHwAmmLfwh9"),Cn=f(),j=o("p"),dd=i("We\u2019re constantly trying to improve our tutorials, so "),rs=o("strong"),vd=i("if you find some issues in this notebook"),yd=i(", please "),lo=o("a"),gd=i("open an issue on the Github Repo"),wd=i("."),Zn=f(),Ho=o("p"),_d=i("See you on Bonus unit 2! \u{1F525}"),Hn=f(),ie=o("h3"),xe=o("a"),ss=o("span"),c(oo.$$.fragment),bd=f(),is=o("span"),Ed=i("Keep Learning, Stay Awesome \u{1F917}"),this.h()},l(e){const s=xg('[data-svelte="svelte-1phssyn"]',document.head);G=a(s,"META",{name:!0,content:!0}),s.forEach(t),gs=u(e),J=a(e,"H1",{class:!0});var Xn=r(J);pe=a(Xn,"A",{id:!0,class:!0,href:!0});var lv=r(pe);xo=a(lv,"SPAN",{});var ov=r(xo);m(at.$$.fragment,ov),ov.forEach(t),lv.forEach(t),Up=u(Xn),Qo=a(Xn,"SPAN",{});var av=r(Qo);Cp=n(av,"Hands-on"),av.forEach(t),Xn.forEach(t),ws=u(e),m(rt.$$.fragment,e),_s=u(e),fe=a(e,"P",{});var Dn=r(fe);Zp=n(Dn,"Now that you\u2019ve studied the theory behind Deep Q-Learning, "),qo=a(Dn,"STRONG",{});var rv=r(qo);Hp=n(rv,"you\u2019re ready to train your Deep Q-Learning agent to play Atari Games"),rv.forEach(t),Wp=n(Dn,". We\u2019ll start with Space Invaders, but you\u2019ll be able to use any Atari game you want \u{1F525}"),Dn.forEach(t),bs=u(e),st=a(e,"IMG",{src:!0,alt:!0}),Es=u(e),ue=a(e,"P",{});var On=r(ue);Xp=n(On,"We\u2019re using the "),it=a(On,"A",{href:!0,rel:!0});var sv=r(it);Dp=n(sv,"RL-Baselines-3 Zoo integration"),sv.forEach(t),Op=n(On,", a vanilla version of Deep Q-Learning with no extensions such as Double-DQN, Dueling-DQN, or Prioritized Experience Replay."),On.forEach(t),$s=u(e),B=a(e,"P",{});var ns=r(B);zp=n(ns,"Also, "),Vo=a(ns,"STRONG",{});var iv=r(Vo);Yp=n(iv,"if you want to learn to implement Deep Q-Learning by yourself after this hands-on"),iv.forEach(t),xp=n(ns,", you definitely should look at the CleanRL implementation: "),nt=a(ns,"A",{href:!0,rel:!0});var nv=r(nt);Qp=n(nv,"https://github.com/vwxyzjn/cleanrl/blob/master/cleanrl/dqn_atari.py"),nv.forEach(t),ns.forEach(t),ks=u(e),he=a(e,"P",{});var zn=r(he);qp=n(zn,"To validate this hands-on for the certification process, you need to push your trained model to the Hub and "),Ko=a(zn,"STRONG",{});var pv=r(Ko);Vp=n(pv,"get a result of >= 200"),pv.forEach(t),Kp=n(zn,"."),zn.forEach(t),Ts=u(e),pt=a(e,"P",{});var $d=r(pt);ef=n($d,"To find your result, go to the leaderboard and find your model, "),ea=a($d,"STRONG",{});var fv=r(ea);tf=n(fv,"the result = mean_reward - std of reward"),fv.forEach(t),$d.forEach(t),Is=u(e),ao=a(e,"P",{});var uv=r(ao);ta=a(uv,"STRONG",{});var hv=r(ta);lf=n(hv,"If you don\u2019t find your model, go to the bottom of the page and click on the refresh button."),hv.forEach(t),uv.forEach(t),Ms=u(e),ft=a(e,"P",{});var kd=r(ft);of=n(kd,"For more information about the certification process, check this section \u{1F449} "),ut=a(kd,"A",{href:!0,rel:!0});var cv=r(ut);af=n(cv,"https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),cv.forEach(t),kd.forEach(t),Ls=u(e),ht=a(e,"P",{});var Td=r(ht);rf=n(Td,"And you can check your progress here \u{1F449} "),ct=a(Td,"A",{href:!0,rel:!0});var mv=r(ct);sf=n(mv,"https://huggingface.co/spaces/ThomasSimonini/Check-my-progress-Deep-RL-Course"),mv.forEach(t),Td.forEach(t),As=u(e),mt=a(e,"P",{});var Id=r(mt);la=a(Id,"STRONG",{});var dv=r(la);nf=n(dv,"To start the hands-on click on Open In Colab button"),dv.forEach(t),pf=n(Id," \u{1F447} :"),Id.forEach(t),Rs=u(e),ro=a(e,"P",{});var vv=r(ro);dt=a(vv,"A",{href:!0,rel:!0});var yv=r(dt);so=a(yv,"IMG",{src:!0,alt:!0}),yv.forEach(t),vv.forEach(t),Ns=u(e),F=a(e,"H1",{class:!0});var Yn=r(F);ce=a(Yn,"A",{id:!0,class:!0,href:!0});var gv=r(ce);oa=a(gv,"SPAN",{});var wv=r(oa);m(vt.$$.fragment,wv),wv.forEach(t),gv.forEach(t),ff=u(Yn),aa=a(Yn,"SPAN",{});var _v=r(aa);uf=n(_v,"Unit 3: Deep Q-Learning with Atari Games \u{1F47E} using RL Baselines3 Zoo"),_v.forEach(t),Yn.forEach(t),Ss=u(e),yt=a(e,"IMG",{src:!0,alt:!0}),js=u(e),b=a(e,"P",{});var Qe=r(b);hf=n(Qe,"In this hands-on, "),ra=a(Qe,"STRONG",{});var bv=r(ra);cf=n(bv,"you\u2019ll train a Deep Q-Learning agent"),bv.forEach(t),mf=n(Qe," playing Space Invaders using "),gt=a(Qe,"A",{href:!0,rel:!0});var Ev=r(gt);df=n(Ev,"RL Baselines3 Zoo"),Ev.forEach(t),vf=n(Qe,", a training framework based on "),wt=a(Qe,"A",{href:!0,rel:!0});var $v=r(wt);yf=n($v,"Stable-Baselines3"),$v.forEach(t),gf=n(Qe," that provides scripts for training, evaluating agents, tuning hyperparameters, plotting results and recording videos."),Qe.forEach(t),Gs=u(e),me=a(e,"P",{});var xn=r(me);wf=n(xn,"We\u2019re using the "),_t=a(xn,"A",{href:!0,rel:!0});var kv=r(_t);_f=n(kv,"RL-Baselines-3 Zoo integration, a vanilla version of Deep Q-Learning"),kv.forEach(t),bf=n(xn," with no extensions such as Double-DQN, Dueling-DQN, and Prioritized Experience Replay."),xn.forEach(t),Js=u(e),P=a(e,"H3",{class:!0});var Qn=r(P);de=a(Qn,"A",{id:!0,class:!0,href:!0});var Tv=r(de);sa=a(Tv,"SPAN",{});var Iv=r(sa);m(bt.$$.fragment,Iv),Iv.forEach(t),Tv.forEach(t),Ef=u(Qn),ia=a(Qn,"SPAN",{});var Mv=r(ia);$f=n(Mv,"\u{1F3AE} Environments:"),Mv.forEach(t),Qn.forEach(t),Bs=u(e),io=a(e,"UL",{});var Lv=r(io);na=a(Lv,"LI",{});var Av=r(na);Et=a(Av,"A",{href:!0,rel:!0});var Rv=r(Et);kf=n(Rv,"SpacesInvadersNoFrameskip-v4"),Rv.forEach(t),Av.forEach(t),Lv.forEach(t),Fs=u(e),$t=a(e,"P",{});var Md=r($t);Tf=n(Md,"You can see the difference between Space Invaders versions here \u{1F449} "),kt=a(Md,"A",{href:!0,rel:!0});var Nv=r(kt);If=n(Nv,"https://gymnasium.farama.org/environments/atari/space_invaders/#variants"),Nv.forEach(t),Md.forEach(t),Ps=u(e),U=a(e,"H3",{class:!0});var qn=r(U);ve=a(qn,"A",{id:!0,class:!0,href:!0});var Sv=r(ve);pa=a(Sv,"SPAN",{});var jv=r(pa);m(Tt.$$.fragment,jv),jv.forEach(t),Sv.forEach(t),Mf=u(qn),fa=a(qn,"SPAN",{});var Gv=r(fa);Lf=n(Gv,"\u{1F4DA} RL-Library:"),Gv.forEach(t),qn.forEach(t),Us=u(e),no=a(e,"UL",{});var Jv=r(no);ua=a(Jv,"LI",{});var Bv=r(ua);It=a(Bv,"A",{href:!0,rel:!0});var Fv=r(It);Af=n(Fv,"RL-Baselines3-Zoo"),Fv.forEach(t),Bv.forEach(t),Jv.forEach(t),Cs=u(e),C=a(e,"H2",{class:!0});var Vn=r(C);ye=a(Vn,"A",{id:!0,class:!0,href:!0});var Pv=r(ye);ha=a(Pv,"SPAN",{});var Uv=r(ha);m(Mt.$$.fragment,Uv),Uv.forEach(t),Pv.forEach(t),Rf=u(Vn),ca=a(Vn,"SPAN",{});var Cv=r(ca);Nf=n(Cv,"Objectives of this hands-on \u{1F3C6}"),Cv.forEach(t),Vn.forEach(t),Zs=u(e),po=a(e,"P",{});var Zv=r(po);Sf=n(Zv,"At the end of the hands-on, you will:"),Zv.forEach(t),Hs=u(e),ge=a(e,"UL",{});var Kn=r(ge);Lt=a(Kn,"LI",{});var ep=r(Lt);jf=n(ep,"Be able to understand deeper "),ma=a(ep,"STRONG",{});var Hv=r(ma);Gf=n(Hv,"how RL Baselines3 Zoo works"),Hv.forEach(t),Jf=n(ep,"."),ep.forEach(t),Bf=u(Kn),At=a(Kn,"LI",{});var tp=r(At);Ff=n(tp,"Be able to "),da=a(tp,"STRONG",{});var Wv=r(da);Pf=n(Wv,"push your trained agent and the code to the Hub"),Wv.forEach(t),Uf=n(tp," with a nice video replay and an evaluation score \u{1F525}."),tp.forEach(t),Kn.forEach(t),Ws=u(e),Z=a(e,"H2",{class:!0});var lp=r(Z);we=a(lp,"A",{id:!0,class:!0,href:!0});var Xv=r(we);va=a(Xv,"SPAN",{});var Dv=r(va);m(Rt.$$.fragment,Dv),Dv.forEach(t),Xv.forEach(t),Cf=u(lp),ya=a(lp,"SPAN",{});var Ov=r(ya);Zf=n(Ov,"Prerequisites \u{1F3D7}\uFE0F"),Ov.forEach(t),lp.forEach(t),Xs=u(e),fo=a(e,"P",{});var zv=r(fo);Hf=n(zv,"Before diving into the hands-on, you need to:"),zv.forEach(t),Ds=u(e),_e=a(e,"P",{});var op=r(_e);Wf=n(op,"\u{1F532} \u{1F4DA} "),ga=a(op,"STRONG",{});var Yv=r(ga);Nt=a(Yv,"A",{href:!0,rel:!0});var xv=r(Nt);Xf=n(xv,"Study Deep Q-Learning by reading Unit 3"),xv.forEach(t),Yv.forEach(t),Df=n(op,"  \u{1F917}"),op.forEach(t),Os=u(e),A=a(e,"P",{});var Wo=r(A);Of=n(Wo,"We\u2019re constantly trying to improve our tutorials, so "),wa=a(Wo,"STRONG",{});var Qv=r(wa);zf=n(Qv,"if you find some issues in this hands-on"),Qv.forEach(t),Yf=n(Wo,", please "),St=a(Wo,"A",{href:!0,rel:!0});var qv=r(St);xf=n(qv,"open an issue on the Github Repo"),qv.forEach(t),Qf=n(Wo,"."),Wo.forEach(t),zs=u(e),H=a(e,"H1",{class:!0});var ap=r(H);be=a(ap,"A",{id:!0,class:!0,href:!0});var Vv=r(be);_a=a(Vv,"SPAN",{});var Kv=r(_a);m(jt.$$.fragment,Kv),Kv.forEach(t),Vv.forEach(t),qf=u(ap),ba=a(ap,"SPAN",{});var e1=r(ba);Vf=n(e1,"Let's train a Deep Q-Learning agent playing Atari' Space Invaders \u{1F47E} and upload it to the Hub."),e1.forEach(t),ap.forEach(t),Ys=u(e),Ee=a(e,"P",{});var rp=r(Ee);Kf=n(rp,"We strongly recommend students "),Ea=a(rp,"STRONG",{});var t1=r(Ea);eu=n(t1,"to use Google Colab for the hands-on exercises instead of running them on their personal computers"),t1.forEach(t),tu=n(rp,"."),rp.forEach(t),xs=u(e),$e=a(e,"P",{});var sp=r($e);lu=n(sp,"By using Google Colab, "),$a=a(sp,"STRONG",{});var l1=r($a);ou=n(l1,"you can focus on learning and experimenting without worrying about the technical aspects of setting up your environments"),l1.forEach(t),au=n(sp,"."),sp.forEach(t),Qs=u(e),ke=a(e,"P",{});var ip=r(ke);ru=n(ip,"To validate this hands-on for the certification process, you need to push your trained model to the Hub and "),ka=a(ip,"STRONG",{});var o1=r(ka);su=n(o1,"get a result of >= 200"),o1.forEach(t),iu=n(ip,"."),ip.forEach(t),qs=u(e),Gt=a(e,"P",{});var Ld=r(Gt);nu=n(Ld,"To find your result, go to the leaderboard and find your model, "),Ta=a(Ld,"STRONG",{});var a1=r(Ta);pu=n(a1,"the result = mean_reward - std of reward"),a1.forEach(t),Ld.forEach(t),Vs=u(e),Jt=a(e,"P",{});var Ad=r(Jt);fu=n(Ad,"For more information about the certification process, check this section \u{1F449} "),Bt=a(Ad,"A",{href:!0,rel:!0});var r1=r(Bt);uu=n(r1,"https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),r1.forEach(t),Ad.forEach(t),Ks=u(e),W=a(e,"H2",{class:!0});var np=r(W);Te=a(np,"A",{id:!0,class:!0,href:!0});var s1=r(Te);Ia=a(s1,"SPAN",{});var i1=r(Ia);m(Ft.$$.fragment,i1),i1.forEach(t),s1.forEach(t),hu=u(np),Ma=a(np,"SPAN",{});var n1=r(Ma);cu=n(n1,"Set the GPU \u{1F4AA}"),n1.forEach(t),np.forEach(t),ei=u(e),uo=a(e,"UL",{});var p1=r(uo);Ie=a(p1,"LI",{});var ps=r(Ie);mu=n(ps,"To "),La=a(ps,"STRONG",{});var f1=r(La);du=n(f1,"accelerate the agent\u2019s training, we\u2019ll use a GPU"),f1.forEach(t),vu=n(ps,". To do that, go to "),Aa=a(ps,"CODE",{});var u1=r(Aa);yu=n(u1,"Runtime > Change Runtime type"),u1.forEach(t),ps.forEach(t),p1.forEach(t),ti=u(e),Pt=a(e,"IMG",{src:!0,alt:!0}),li=u(e),ho=a(e,"UL",{});var h1=r(ho);Ra=a(h1,"LI",{});var c1=r(Ra);Na=a(c1,"CODE",{});var m1=r(Na);gu=n(m1,"Hardware Accelerator > GPU"),m1.forEach(t),c1.forEach(t),h1.forEach(t),oi=u(e),Ut=a(e,"IMG",{src:!0,alt:!0}),ai=u(e),X=a(e,"H1",{class:!0});var pp=r(X);Me=a(pp,"A",{id:!0,class:!0,href:!0});var d1=r(Me);Sa=a(d1,"SPAN",{});var v1=r(Sa);m(Ct.$$.fragment,v1),v1.forEach(t),d1.forEach(t),wu=u(pp),ja=a(pp,"SPAN",{});var y1=r(ja);_u=n(y1,"Install RL-Baselines3 Zoo and its dependencies \u{1F4DA}"),y1.forEach(t),pp.forEach(t),ri=u(e),R=a(e,"P",{});var Xo=r(R);bu=n(Xo,"If you see "),Ga=a(Xo,"CODE",{});var g1=r(Ga);Eu=n(g1,"ERROR: pip's dependency resolver does not currently take into account all the packages that are installed."),g1.forEach(t),$u=u(Xo),Ja=a(Xo,"STRONG",{});var w1=r(Ja);ku=n(w1,"this is normal and it\u2019s not a critical error"),w1.forEach(t),Tu=n(Xo," there\u2019s a conflict of version. But the packages we need are installed."),Xo.forEach(t),si=u(e),m(Zt.$$.fragment,e),ii=u(e),co=a(e,"P",{});var _1=r(co);Iu=n(_1,"IF AND ONLY IF THE VERSION ABOVE DOES NOT EXIST ANYMORE. UNCOMMENT AND INSTALL THE ONE BELOW"),_1.forEach(t),ni=u(e),m(Ht.$$.fragment,e),pi=u(e),m(Wt.$$.fragment,e),fi=u(e),mo=a(e,"P",{});var b1=r(mo);Mu=n(b1,"To be able to use Atari games in Gymnasium we need to install atari package. And accept-rom-license to download the rom files (games files)."),b1.forEach(t),ui=u(e),m(Xt.$$.fragment,e),hi=u(e),D=a(e,"H2",{class:!0});var fp=r(D);Le=a(fp,"A",{id:!0,class:!0,href:!0});var E1=r(Le);Ba=a(E1,"SPAN",{});var $1=r(Ba);m(Dt.$$.fragment,$1),$1.forEach(t),E1.forEach(t),Lu=u(fp),Fa=a(fp,"SPAN",{});var k1=r(Fa);Au=n(k1,"Create a virtual display \u{1F53D}"),k1.forEach(t),fp.forEach(t),ci=u(e),Ae=a(e,"P",{});var up=r(Ae);Ru=n(up,"During the hands-on, we\u2019ll need to generate a replay video. To do so, if you train it on a headless machine, "),Pa=a(up,"STRONG",{});var T1=r(Pa);Nu=n(T1,"we need to have a virtual screen to be able to render the environment"),T1.forEach(t),Su=n(up," (and thus record the frames)."),up.forEach(t),mi=u(e),vo=a(e,"P",{});var I1=r(vo);ju=n(I1,"Hence the following cell will install the librairies and create and run a virtual screen \u{1F5A5}"),I1.forEach(t),di=u(e),m(Ot.$$.fragment,e),vi=u(e),m(zt.$$.fragment,e),yi=u(e),O=a(e,"H2",{class:!0});var hp=r(O);Re=a(hp,"A",{id:!0,class:!0,href:!0});var M1=r(Re);Ua=a(M1,"SPAN",{});var L1=r(Ua);m(Yt.$$.fragment,L1),L1.forEach(t),M1.forEach(t),Gu=u(hp),Ca=a(hp,"SPAN",{});var A1=r(Ca);Ju=n(A1,"Train our Deep Q-Learning Agent to Play Space Invaders \u{1F47E}"),A1.forEach(t),hp.forEach(t),gi=u(e),yo=a(e,"P",{});var R1=r(yo);Bu=n(R1,"To train an agent with RL-Baselines3-Zoo, we just need to do two things:"),R1.forEach(t),wi=u(e),go=a(e,"OL",{});var N1=r(go);xt=a(N1,"LI",{});var cp=r(xt);Fu=n(cp,"Create a hyperparameter config file that will contain our training hyperparameters called "),Za=a(cp,"CODE",{});var S1=r(Za);Pu=n(S1,"dqn.yml"),S1.forEach(t),Uu=n(cp,"."),cp.forEach(t),N1.forEach(t),_i=u(e),wo=a(e,"P",{});var j1=r(wo);Cu=n(j1,"This is a template example:"),j1.forEach(t),bi=u(e),m(Qt.$$.fragment,e),Ei=u(e),_o=a(e,"P",{});var G1=r(_o);Zu=n(G1,"Here we see that:"),G1.forEach(t),$i=u(e),E=a(e,"UL",{});var qe=r(E);qt=a(qe,"LI",{});var mp=r(qt);Hu=n(mp,"We use the "),Ha=a(mp,"CODE",{});var J1=r(Ha);Wu=n(J1,"Atari Wrapper"),J1.forEach(t),Xu=n(mp," that preprocess the input (Frame reduction ,grayscale, stack 4 frames)"),mp.forEach(t),Du=u(qe),Vt=a(qe,"LI",{});var dp=r(Vt);Ou=n(dp,"We use "),Wa=a(dp,"CODE",{});var B1=r(Wa);zu=n(B1,"CnnPolicy"),B1.forEach(t),Yu=n(dp,", since we use Convolutional layers to process the frames"),dp.forEach(t),xu=u(qe),bo=a(qe,"LI",{});var Rd=r(bo);Qu=n(Rd,"We train it for 10 million "),Xa=a(Rd,"CODE",{});var F1=r(Xa);qu=n(F1,"n_timesteps"),F1.forEach(t),Rd.forEach(t),Vu=u(qe),Da=a(qe,"LI",{});var P1=r(Da);Ku=n(P1,"Memory (Experience Replay) size is 100000, aka the amount of experience steps you saved to train again your agent with."),P1.forEach(t),qe.forEach(t),ki=u(e),$=a(e,"P",{});var Ve=r($);eh=n(Ve,"\u{1F4A1} My advice is to "),Oa=a(Ve,"STRONG",{});var U1=r(Oa);th=n(U1,"reduce the training timesteps to 1M,"),U1.forEach(t),lh=n(Ve," which will take about 90 minutes on a P100. "),za=a(Ve,"CODE",{});var C1=r(za);oh=n(C1,"!nvidia-smi"),C1.forEach(t),ah=n(Ve," will tell you what GPU you\u2019re using. At 10 million steps, this will take about 9 hours. I recommend running this on your local computer (or somewhere else). Just click on: "),Ya=a(Ve,"CODE",{});var Z1=r(Ya);rh=n(Z1,"File>Download"),Z1.forEach(t),sh=n(Ve,"."),Ve.forEach(t),Ti=u(e),Eo=a(e,"P",{});var H1=r(Eo);ih=n(H1,"In terms of hyperparameters optimization, my advice is to focus on these 3 hyperparameters:"),H1.forEach(t),Ii=u(e),N=a(e,"UL",{});var Do=r(N);xa=a(Do,"LI",{});var W1=r(xa);Qa=a(W1,"CODE",{});var X1=r(Qa);nh=n(X1,"learning_rate"),X1.forEach(t),W1.forEach(t),ph=u(Do),qa=a(Do,"LI",{});var D1=r(qa);Va=a(D1,"CODE",{});var O1=r(Va);fh=n(O1,"buffer_size (Experience Memory size)"),O1.forEach(t),D1.forEach(t),uh=u(Do),Ka=a(Do,"LI",{});var z1=r(Ka);er=a(z1,"CODE",{});var Y1=r(er);hh=n(Y1,"batch_size"),Y1.forEach(t),z1.forEach(t),Do.forEach(t),Mi=u(e),z=a(e,"P",{});var fs=r(z);ch=n(fs,"As a good practice, you need to "),tr=a(fs,"STRONG",{});var x1=r(tr);mh=n(x1,"check the documentation to understand what each hyperparameters does"),x1.forEach(t),dh=n(fs,": "),Kt=a(fs,"A",{href:!0,rel:!0});var Q1=r(Kt);vh=n(Q1,"https://stable-baselines3.readthedocs.io/en/master/modules/dqn.html#parameters"),Q1.forEach(t),fs.forEach(t),Li=u(e),el=a(e,"OL",{start:!0});var q1=r(el);tl=a(q1,"LI",{});var vp=r(tl);yh=n(vp,"We start the training and save the models on "),lr=a(vp,"CODE",{});var V1=r(lr);gh=n(V1,"logs"),V1.forEach(t),wh=n(vp," folder \u{1F4C1}"),vp.forEach(t),q1.forEach(t),Ai=u(e),$o=a(e,"UL",{});var K1=r($o);L=a(K1,"LI",{});var Ke=r(L);_h=n(Ke,"Define the algorithm after "),or=a(Ke,"CODE",{});var ey=r(or);bh=n(ey,"--algo"),ey.forEach(t),Eh=n(Ke,", where we save the model after "),ar=a(Ke,"CODE",{});var ty=r(ar);$h=n(ty,"-f"),ty.forEach(t),kh=n(Ke," and where the hyperparameter config is after "),rr=a(Ke,"CODE",{});var ly=r(rr);Th=n(ly,"-c"),ly.forEach(t),Ih=n(Ke,"."),Ke.forEach(t),K1.forEach(t),Ri=u(e),m(ll.$$.fragment,e),Ni=u(e),Y=a(e,"H4",{class:!0});var yp=r(Y);Ne=a(yp,"A",{id:!0,class:!0,href:!0});var oy=r(Ne);sr=a(oy,"SPAN",{});var ay=r(sr);m(ol.$$.fragment,ay),ay.forEach(t),oy.forEach(t),Mh=u(yp),ir=a(yp,"SPAN",{});var ry=r(ir);Lh=n(ry,"Solution"),ry.forEach(t),yp.forEach(t),Si=u(e),m(al.$$.fragment,e),ji=u(e),x=a(e,"H2",{class:!0});var gp=r(x);Se=a(gp,"A",{id:!0,class:!0,href:!0});var sy=r(Se);nr=a(sy,"SPAN",{});var iy=r(nr);m(rl.$$.fragment,iy),iy.forEach(t),sy.forEach(t),Ah=u(gp),pr=a(gp,"SPAN",{});var ny=r(pr);Rh=n(ny,"Let's evaluate our agent \u{1F440}"),ny.forEach(t),gp.forEach(t),Gi=u(e),je=a(e,"UL",{});var wp=r(je);Q=a(wp,"LI",{});var Oo=r(Q);Nh=n(Oo,"RL-Baselines3-Zoo provides "),fr=a(Oo,"CODE",{});var py=r(fr);Sh=n(py,"enjoy.py"),py.forEach(t),jh=n(Oo,", a python script to evaluate our agent. In most RL libraries, we call the evaluation script "),ur=a(Oo,"CODE",{});var fy=r(ur);Gh=n(fy,"enjoy.py"),fy.forEach(t),Jh=n(Oo,"."),Oo.forEach(t),Bh=u(wp),hr=a(wp,"LI",{});var uy=r(hr);Fh=n(uy,"Let\u2019s evaluate it for 5000 timesteps \u{1F525}"),uy.forEach(t),wp.forEach(t),Ji=u(e),m(sl.$$.fragment,e),Bi=u(e),q=a(e,"H4",{class:!0});var _p=r(q);Ge=a(_p,"A",{id:!0,class:!0,href:!0});var hy=r(Ge);cr=a(hy,"SPAN",{});var cy=r(cr);m(il.$$.fragment,cy),cy.forEach(t),hy.forEach(t),Ph=u(_p),mr=a(_p,"SPAN",{});var my=r(mr);Uh=n(my,"Solution"),my.forEach(t),_p.forEach(t),Fi=u(e),m(nl.$$.fragment,e),Pi=u(e),V=a(e,"H2",{class:!0});var bp=r(V);Je=a(bp,"A",{id:!0,class:!0,href:!0});var dy=r(Je);dr=a(dy,"SPAN",{});var vy=r(dr);m(pl.$$.fragment,vy),vy.forEach(t),dy.forEach(t),Ch=u(bp),vr=a(bp,"SPAN",{});var yy=r(vr);Zh=n(yy,"Publish our trained model on the Hub \u{1F680}"),yy.forEach(t),bp.forEach(t),Ui=n(e,`

Now that we saw we got good results after the training, we can publish our trained model on the hub \u{1F917} with one line of code.
`),fl=a(e,"IMG",{src:!0,alt:!0}),Ci=u(e),S=a(e,"P",{});var zo=r(S);Hh=n(zo,"By using "),yr=a(zo,"CODE",{});var gy=r(yr);Wh=n(gy,"rl_zoo3.push_to_hub"),gy.forEach(t),Xh=u(zo),gr=a(zo,"STRONG",{});var wy=r(gr);Dh=n(wy,"you evaluate, record a replay, generate a model card of your agent and push it to the hub"),wy.forEach(t),Oh=n(zo,"."),zo.forEach(t),Zi=u(e),ko=a(e,"P",{});var _y=r(ko);zh=n(_y,"This way:"),_y.forEach(t),Hi=u(e),k=a(e,"UL",{});var et=r(k);ul=a(et,"LI",{});var Ep=r(ul);Yh=n(Ep,"You can "),wr=a(Ep,"STRONG",{});var by=r(wr);xh=n(by,"showcase our work"),by.forEach(t),Qh=n(Ep," \u{1F525}"),Ep.forEach(t),qh=u(et),hl=a(et,"LI",{});var $p=r(hl);Vh=n($p,"You can "),_r=a($p,"STRONG",{});var Ey=r(_r);Kh=n(Ey,"visualize your agent playing"),Ey.forEach(t),ec=n($p," \u{1F440}"),$p.forEach(t),tc=u(et),cl=a(et,"LI",{});var kp=r(cl);lc=n(kp,"You can "),br=a(kp,"STRONG",{});var $y=r(br);oc=n($y,"share with the community an agent that others can use"),$y.forEach(t),ac=n(kp," \u{1F4BE}"),kp.forEach(t),rc=u(et),Be=a(et,"LI",{});var us=r(Be);sc=n(us,"You can "),Er=a(us,"STRONG",{});var ky=r(Er);ic=n(ky,"access a leaderboard \u{1F3C6} to see how well your agent is performing compared to your classmates"),ky.forEach(t),nc=n(us," \u{1F449}  "),ml=a(us,"A",{href:!0,rel:!0});var Ty=r(ml);pc=n(Ty,"https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),Ty.forEach(t),us.forEach(t),et.forEach(t),Wi=u(e),To=a(e,"P",{});var Iy=r(To);fc=n(Iy,"To be able to share your model with the community there are three more steps to follow:"),Iy.forEach(t),Xi=u(e),dl=a(e,"P",{});var Nd=r(dl);uc=n(Nd,"1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),vl=a(Nd,"A",{href:!0,rel:!0});var My=r(vl);hc=n(My,"https://huggingface.co/join"),My.forEach(t),Nd.forEach(t),Di=u(e),Io=a(e,"P",{});var Ly=r(Io);cc=n(Ly,"2\uFE0F\u20E3 Sign in and then, you need to store your authentication token from the Hugging Face website."),Ly.forEach(t),Oi=u(e),Mo=a(e,"UL",{});var Ay=r(Mo);Fe=a(Ay,"LI",{});var hs=r(Fe);mc=n(hs,"Create a new token ("),yl=a(hs,"A",{href:!0,rel:!0});var Ry=r(yl);dc=n(Ry,"https://huggingface.co/settings/tokens"),Ry.forEach(t),vc=n(hs,") "),$r=a(hs,"STRONG",{});var Ny=r($r);yc=n(Ny,"with write role"),Ny.forEach(t),hs.forEach(t),Ay.forEach(t),zi=u(e),gl=a(e,"IMG",{src:!0,alt:!0}),Yi=u(e),Pe=a(e,"UL",{});var Tp=r(Pe);kr=a(Tp,"LI",{});var Sy=r(kr);gc=n(Sy,"Copy the token"),Sy.forEach(t),wc=u(Tp),Tr=a(Tp,"LI",{});var jy=r(Tr);_c=n(jy,"Run the cell below and past the token"),jy.forEach(t),Tp.forEach(t),xi=u(e),m(wl.$$.fragment,e),Qi=u(e),_l=a(e,"P",{});var Sd=r(_l);bc=n(Sd,"If you don\u2019t want to use a Google Colab or a Jupyter Notebook, you need to use this command instead: "),Ir=a(Sd,"CODE",{});var Gy=r(Ir);Ec=n(Gy,"huggingface-cli login"),Gy.forEach(t),Sd.forEach(t),qi=u(e),Lo=a(e,"P",{});var Jy=r(Lo);$c=n(Jy,"3\uFE0F\u20E3 We\u2019re now ready to push our trained agent to the \u{1F917} Hub \u{1F525}"),Jy.forEach(t),Vi=u(e),Ao=a(e,"P",{});var By=r(Ao);kc=n(By,"Let\u2019s run push_to_hub.py file to upload our trained agent to the Hub."),By.forEach(t),Ki=u(e),bl=a(e,"P",{});var jd=r(bl);Mr=a(jd,"CODE",{});var Fy=r(Mr);Tc=n(Fy,"--repo-name "),Fy.forEach(t),Ic=n(jd,": The name of the repo"),jd.forEach(t),en=u(e),El=a(e,"P",{});var Gd=r(El);Lr=a(Gd,"CODE",{});var Py=r(Lr);Mc=n(Py,"-orga"),Py.forEach(t),Lc=n(Gd,": Your Hugging Face username"),Gd.forEach(t),tn=u(e),K=a(e,"P",{});var cs=r(K);Ar=a(cs,"CODE",{});var Uy=r(Ar);Ac=n(Uy,"-f"),Uy.forEach(t),Rc=n(cs,": Where the trained model folder is (in our case "),Rr=a(cs,"CODE",{});var Cy=r(Rr);Nc=n(Cy,"logs"),Cy.forEach(t),Sc=n(cs,")"),cs.forEach(t),ln=u(e),$l=a(e,"IMG",{src:!0,alt:!0}),on=u(e),m(kl.$$.fragment,e),an=u(e),ee=a(e,"H4",{class:!0});var Ip=r(ee);Ue=a(Ip,"A",{id:!0,class:!0,href:!0});var Zy=r(Ue);Nr=a(Zy,"SPAN",{});var Hy=r(Nr);m(Tl.$$.fragment,Hy),Hy.forEach(t),Zy.forEach(t),jc=u(Ip),Sr=a(Ip,"SPAN",{});var Wy=r(Sr);Gc=n(Wy,"Solution"),Wy.forEach(t),Ip.forEach(t),rn=u(e),m(Il.$$.fragment,e),sn=u(e),Ro=a(e,"P",{});var Xy=r(Ro);Jc=n(Xy,"###."),Xy.forEach(t),nn=u(e),Ce=a(e,"P",{});var Mp=r(Ce);Bc=n(Mp,"Congrats \u{1F973} you\u2019ve just trained and uploaded your first Deep Q-Learning agent using RL-Baselines-3 Zoo. The script above should have displayed a link to a model repository such as "),Ml=a(Mp,"A",{href:!0,rel:!0});var Dy=r(Ml);Fc=n(Dy,"https://huggingface.co/ThomasSimonini/dqn-SpaceInvadersNoFrameskip-v4"),Dy.forEach(t),Pc=n(Mp,". When you go to this link, you can:"),Mp.forEach(t),pn=u(e),T=a(e,"UL",{});var tt=r(T);Ll=a(tt,"LI",{});var Lp=r(Ll);Uc=n(Lp,"See a "),jr=a(Lp,"STRONG",{});var Oy=r(jr);Cc=n(Oy,"video preview of your agent"),Oy.forEach(t),Zc=n(Lp," at the right."),Lp.forEach(t),Hc=u(tt),Gr=a(tt,"LI",{});var zy=r(Gr);Wc=n(zy,"Click \u201CFiles and versions\u201D to see all the files in the repository."),zy.forEach(t),Xc=u(tt),Jr=a(tt,"LI",{});var Yy=r(Jr);Dc=n(Yy,"Click \u201CUse in stable-baselines3\u201D to get a code snippet that shows how to load the model."),Yy.forEach(t),Oc=u(tt),Al=a(tt,"LI",{});var Ap=r(Al);zc=n(Ap,"A model card ("),Br=a(Ap,"CODE",{});var xy=r(Br);Yc=n(xy,"README.md"),xy.forEach(t),xc=n(Ap," file) which gives a description of the model and the hyperparameters you used."),Ap.forEach(t),tt.forEach(t),fn=u(e),No=a(e,"P",{});var Qy=r(No);Qc=n(Qy,"Under the hood, the Hub uses git-based repositories (don\u2019t worry if you don\u2019t know what git is), which means you can update the model with new versions as you experiment and improve your agent."),Qy.forEach(t),un=u(e),te=a(e,"P",{});var ms=r(te);Fr=a(ms,"STRONG",{});var qy=r(Fr);qc=n(qy,"Compare the results of your agents with your classmates"),qy.forEach(t),Vc=n(ms," using the "),Rl=a(ms,"A",{href:!0,rel:!0});var Vy=r(Rl);Kc=n(Vy,"leaderboard"),Vy.forEach(t),em=n(ms," \u{1F3C6}"),ms.forEach(t),hn=u(e),le=a(e,"H2",{class:!0});var Rp=r(le);Ze=a(Rp,"A",{id:!0,class:!0,href:!0});var Ky=r(Ze);Pr=a(Ky,"SPAN",{});var eg=r(Pr);m(Nl.$$.fragment,eg),eg.forEach(t),Ky.forEach(t),tm=u(Rp),Ur=a(Rp,"SPAN",{});var tg=r(Ur);lm=n(tg,"Load a powerful trained model \u{1F525}"),tg.forEach(t),Rp.forEach(t),cn=u(e),So=a(e,"UL",{});var lg=r(So);Sl=a(lg,"LI",{});var Np=r(Sl);om=n(Np,"The Stable-Baselines3 team uploaded "),Cr=a(Np,"STRONG",{});var og=r(Cr);am=n(og,"more than 150 trained Deep Reinforcement Learning agents on the Hub"),og.forEach(t),rm=n(Np,"."),Np.forEach(t),lg.forEach(t),mn=u(e),jl=a(e,"P",{});var Jd=r(jl);sm=n(Jd,"You can find them here: \u{1F449} "),Gl=a(Jd,"A",{href:!0,rel:!0});var ag=r(Gl);im=n(ag,"https://huggingface.co/sb3"),ag.forEach(t),Jd.forEach(t),dn=u(e),jo=a(e,"P",{});var rg=r(jo);nm=n(rg,"Some examples:"),rg.forEach(t),vn=u(e),I=a(e,"UL",{});var lt=r(I);Go=a(lt,"LI",{});var Bd=r(Go);pm=n(Bd,"Asteroids: "),Jl=a(Bd,"A",{href:!0,rel:!0});var sg=r(Jl);fm=n(sg,"https://huggingface.co/sb3/dqn-AsteroidsNoFrameskip-v4"),sg.forEach(t),Bd.forEach(t),um=u(lt),Jo=a(lt,"LI",{});var Fd=r(Jo);hm=n(Fd,"Beam Rider: "),Bl=a(Fd,"A",{href:!0,rel:!0});var ig=r(Bl);cm=n(ig,"https://huggingface.co/sb3/dqn-BeamRiderNoFrameskip-v4"),ig.forEach(t),Fd.forEach(t),mm=u(lt),Bo=a(lt,"LI",{});var Pd=r(Bo);dm=n(Pd,"Breakout: "),Fl=a(Pd,"A",{href:!0,rel:!0});var ng=r(Fl);vm=n(ng,"https://huggingface.co/sb3/dqn-BreakoutNoFrameskip-v4"),ng.forEach(t),Pd.forEach(t),ym=u(lt),Fo=a(lt,"LI",{});var Ud=r(Fo);gm=n(Ud,"Road Runner: "),Pl=a(Ud,"A",{href:!0,rel:!0});var pg=r(Pl);wm=n(pg,"https://huggingface.co/sb3/dqn-RoadRunnerNoFrameskip-v4"),pg.forEach(t),Ud.forEach(t),lt.forEach(t),yn=u(e),Ul=a(e,"P",{});var Cd=r(Ul);_m=n(Cd,"Let\u2019s load an agent playing Beam Rider: "),Cl=a(Cd,"A",{href:!0,rel:!0});var fg=r(Cl);bm=n(fg,"https://huggingface.co/sb3/dqn-BeamRiderNoFrameskip-v4"),fg.forEach(t),Cd.forEach(t),gn=u(e),Po=a(e,"OL",{});var ug=r(Po);He=a(ug,"LI",{});var ds=r(He);Em=n(ds,"We download the model using "),Zr=a(ds,"CODE",{});var hg=r(Zr);$m=n(hg,"rl_zoo3.load_from_hub"),hg.forEach(t),km=n(ds,", and place it in a new folder that we can call "),Hr=a(ds,"CODE",{});var cg=r(Hr);Tm=n(cg,"rl_trained"),cg.forEach(t),ds.forEach(t),ug.forEach(t),wn=u(e),m(Zl.$$.fragment,e),_n=u(e),Hl=a(e,"OL",{start:!0});var mg=r(Hl);Wr=a(mg,"LI",{});var dg=r(Wr);Im=n(dg,"Let\u2019s evaluate if for 5000 timesteps"),dg.forEach(t),mg.forEach(t),bn=u(e),m(Wl.$$.fragment,e),En=u(e),Xl=a(e,"P",{});var Zd=r(Xl);Mm=n(Zd,"Why not trying to train your own "),Xr=a(Zd,"STRONG",{});var vg=r(Xr);Lm=n(vg,"Deep Q-Learning Agent playing BeamRiderNoFrameskip-v4? \u{1F3C6}."),vg.forEach(t),Zd.forEach(t),$n=u(e),oe=a(e,"P",{});var vs=r(oe);Am=n(vs,"If you want to try, check "),Dl=a(vs,"A",{href:!0,rel:!0});var yg=r(Dl);Rm=n(yg,"https://huggingface.co/sb3/dqn-BeamRiderNoFrameskip-v4#hyperparameters"),yg.forEach(t),Nm=u(vs),Dr=a(vs,"STRONG",{});var gg=r(Dr);Sm=n(gg,"in the model card, you have the hyperparameters of the trained agent."),gg.forEach(t),vs.forEach(t),kn=u(e),Ol=a(e,"P",{});var Hd=r(Ol);jm=n(Hd,"But finding hyperparameters can be a daunting task. Fortunately, we\u2019ll see in the next Unit, how we can "),Or=a(Hd,"STRONG",{});var wg=r(Or);Gm=n(wg,"use Optuna for optimizing the Hyperparameters \u{1F525}."),wg.forEach(t),Hd.forEach(t),Tn=u(e),ae=a(e,"H2",{class:!0});var Sp=r(ae);We=a(Sp,"A",{id:!0,class:!0,href:!0});var _g=r(We);zr=a(_g,"SPAN",{});var bg=r(zr);m(zl.$$.fragment,bg),bg.forEach(t),_g.forEach(t),Jm=u(Sp),Yr=a(Sp,"SPAN",{});var Eg=r(Yr);Bm=n(Eg,"Some additional challenges \u{1F3C6}"),Eg.forEach(t),Sp.forEach(t),In=u(e),Xe=a(e,"P",{});var jp=r(Xe);Fm=n(jp,"The best way to learn "),xr=a(jp,"STRONG",{});var $g=r(xr);Pm=n($g,"is to try things by your own"),$g.forEach(t),Um=n(jp,"!"),jp.forEach(t),Mn=u(e),De=a(e,"P",{});var Gp=r(De);Cm=n(Gp,"In the "),Yl=a(Gp,"A",{href:!0,rel:!0});var kg=r(Yl);Zm=n(kg,"Leaderboard"),kg.forEach(t),Hm=n(Gp," you will find your agents. Can you get to the top?"),Gp.forEach(t),Ln=u(e),Uo=a(e,"P",{});var Tg=r(Uo);Wm=n(Tg,"Here\u2019s a list of environments you can try to train your agent with:"),Tg.forEach(t),An=u(e),M=a(e,"UL",{});var ot=r(M);Qr=a(ot,"LI",{});var Ig=r(Qr);Xm=n(Ig,"BeamRiderNoFrameskip-v4"),Ig.forEach(t),Dm=u(ot),qr=a(ot,"LI",{});var Mg=r(qr);Om=n(Mg,"BreakoutNoFrameskip-v4"),Mg.forEach(t),zm=u(ot),Vr=a(ot,"LI",{});var Lg=r(Vr);Ym=n(Lg,"EnduroNoFrameskip-v4"),Lg.forEach(t),xm=u(ot),Kr=a(ot,"LI",{});var Ag=r(Kr);Qm=n(Ag,"PongNoFrameskip-v4"),Ag.forEach(t),ot.forEach(t),Rn=u(e),re=a(e,"P",{});var ys=r(re);qm=n(ys,"Also, "),es=a(ys,"STRONG",{});var Rg=r(es);Vm=n(Rg,"if you want to learn to implement Deep Q-Learning by yourself"),Rg.forEach(t),Km=n(ys,", you definitely should look at CleanRL implementation: "),xl=a(ys,"A",{href:!0,rel:!0});var Ng=r(xl);ed=n(Ng,"https://github.com/vwxyzjn/cleanrl/blob/master/cleanrl/dqn_atari.py"),Ng.forEach(t),ys.forEach(t),Nn=u(e),Ql=a(e,"IMG",{src:!0,alt:!0}),Sn=u(e),jn=a(e,"HR",{}),Gn=u(e),Co=a(e,"P",{});var Sg=r(Co);td=n(Sg,"Congrats on finishing this chapter!"),Sg.forEach(t),Jn=u(e),ql=a(e,"P",{});var Wd=r(ql);ld=n(Wd,"If you\u2019re still feel confused with all these elements\u2026it\u2019s totally normal! "),ts=a(Wd,"STRONG",{});var jg=r(ts);od=n(jg,"This was the same for me and for all people who studied RL."),jg.forEach(t),Wd.forEach(t),Bn=u(e),Oe=a(e,"P",{});var Jp=r(Oe);ad=n(Jp,"Take time to really "),ls=a(Jp,"STRONG",{});var Gg=r(ls);rd=n(Gg,"grasp the material before continuing and try the additional challenges"),Gg.forEach(t),sd=n(Jp,". It\u2019s important to master these elements and having a solid foundations."),Jp.forEach(t),Fn=u(e),ze=a(e,"P",{});var Bp=r(ze);id=n(Bp,"In the next unit, "),Zo=a(Bp,"STRONG",{});var Xd=r(Zo);nd=n(Xd,"we\u2019re going to learn about "),Vl=a(Xd,"A",{href:!0,rel:!0});var Jg=r(Vl);pd=n(Jg,"Optuna"),Jg.forEach(t),Xd.forEach(t),fd=n(Bp,". One of the most critical task in Deep Reinforcement Learning is to find a good set of training hyperparameters. And Optuna is a library that helps you to automate the search."),Bp.forEach(t),Pn=u(e),se=a(e,"H3",{class:!0});var Fp=r(se);Ye=a(Fp,"A",{id:!0,class:!0,href:!0});var Bg=r(Ye);os=a(Bg,"SPAN",{});var Fg=r(os);m(Kl.$$.fragment,Fg),Fg.forEach(t),Bg.forEach(t),ud=u(Fp),as=a(Fp,"SPAN",{});var Pg=r(as);hd=n(Pg,"This is a course built with you \u{1F477}\u{1F3FF}\u200D\u2640\uFE0F"),Pg.forEach(t),Fp.forEach(t),Un=u(e),eo=a(e,"P",{});var Dd=r(eo);cd=n(Dd,"Finally, we want to improve and update the course iteratively with your feedback. If you have some, please fill this form \u{1F449} "),to=a(Dd,"A",{href:!0,rel:!0});var Ug=r(to);md=n(Ug,"https://forms.gle/3HgA7bEHwAmmLfwh9"),Ug.forEach(t),Dd.forEach(t),Cn=u(e),j=a(e,"P",{});var Yo=r(j);dd=n(Yo,"We\u2019re constantly trying to improve our tutorials, so "),rs=a(Yo,"STRONG",{});var Cg=r(rs);vd=n(Cg,"if you find some issues in this notebook"),Cg.forEach(t),yd=n(Yo,", please "),lo=a(Yo,"A",{href:!0,rel:!0});var Zg=r(lo);gd=n(Zg,"open an issue on the Github Repo"),Zg.forEach(t),wd=n(Yo,"."),Yo.forEach(t),Zn=u(e),Ho=a(e,"P",{});var Hg=r(Ho);_d=n(Hg,"See you on Bonus unit 2! \u{1F525}"),Hg.forEach(t),Hn=u(e),ie=a(e,"H3",{class:!0});var Pp=r(ie);xe=a(Pp,"A",{id:!0,class:!0,href:!0});var Wg=r(xe);ss=a(Wg,"SPAN",{});var Xg=r(ss);m(oo.$$.fragment,Xg),Xg.forEach(t),Wg.forEach(t),bd=u(Pp),is=a(Pp,"SPAN",{});var Dg=r(is);Ed=n(Dg,"Keep Learning, Stay Awesome \u{1F917}"),Dg.forEach(t),Pp.forEach(t),this.h()},h(){h(G,"name","hf:doc:metadata"),h(G,"content",JSON.stringify(ew)),h(pe,"id","hands-on"),h(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(pe,"href","#hands-on"),h(J,"class","relative group"),ne(st.src,zd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/atari-envs.gif")||h(st,"src",zd),h(st,"alt","Environments"),h(it,"href","https://github.com/DLR-RM/rl-baselines3-zoo"),h(it,"rel","nofollow"),h(nt,"href","https://github.com/vwxyzjn/cleanrl/blob/master/cleanrl/dqn_atari.py"),h(nt,"rel","nofollow"),h(ut,"href","https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),h(ut,"rel","nofollow"),h(ct,"href","https://huggingface.co/spaces/ThomasSimonini/Check-my-progress-Deep-RL-Course"),h(ct,"rel","nofollow"),ne(so.src,Yd="https://colab.research.google.com/assets/colab-badge.svg")||h(so,"src",Yd),h(so,"alt","Open In Colab"),h(dt,"href","https://colab.research.google.com/github/huggingface/deep-rl-class/blob/master/notebooks/unit3/unit3.ipynb"),h(dt,"rel","nofollow"),h(ce,"id","unit-3-deep-qlearning-with-atari-games-using-rl-baselines3-zoo"),h(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ce,"href","#unit-3-deep-qlearning-with-atari-games-using-rl-baselines3-zoo"),h(F,"class","relative group"),ne(yt.src,xd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/thumbnail.jpg")||h(yt,"src",xd),h(yt,"alt","Unit 3 Thumbnail"),h(gt,"href","https://github.com/DLR-RM/rl-baselines3-zoo"),h(gt,"rel","nofollow"),h(wt,"href","https://stable-baselines3.readthedocs.io/en/master/"),h(wt,"rel","nofollow"),h(_t,"href","https://stable-baselines3.readthedocs.io/en/master/modules/dqn.html"),h(_t,"rel","nofollow"),h(de,"id","environments"),h(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(de,"href","#environments"),h(P,"class","relative group"),h(Et,"href","https://gymnasium.farama.org/environments/atari/space_invaders/"),h(Et,"rel","nofollow"),h(kt,"href","https://gymnasium.farama.org/environments/atari/space_invaders/#variants"),h(kt,"rel","nofollow"),h(ve,"id","rllibrary"),h(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ve,"href","#rllibrary"),h(U,"class","relative group"),h(It,"href","https://github.com/DLR-RM/rl-baselines3-zoo"),h(It,"rel","nofollow"),h(ye,"id","objectives-of-this-handson"),h(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ye,"href","#objectives-of-this-handson"),h(C,"class","relative group"),h(we,"id","prerequisites"),h(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(we,"href","#prerequisites"),h(Z,"class","relative group"),h(Nt,"href","https://huggingface.co/deep-rl-course/unit3/introduction"),h(Nt,"rel","nofollow"),h(St,"href","https://github.com/huggingface/deep-rl-class/issues"),h(St,"rel","nofollow"),h(be,"id","lets-train-a-deep-qlearning-agent-playing-atari-space-invaders-and-upload-it-to-the-hub"),h(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(be,"href","#lets-train-a-deep-qlearning-agent-playing-atari-space-invaders-and-upload-it-to-the-hub"),h(H,"class","relative group"),h(Bt,"href","https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),h(Bt,"rel","nofollow"),h(Te,"id","set-the-gpu"),h(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Te,"href","#set-the-gpu"),h(W,"class","relative group"),ne(Pt.src,Qd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step1.jpg")||h(Pt,"src",Qd),h(Pt,"alt","GPU Step 1"),ne(Ut.src,qd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step2.jpg")||h(Ut,"src",qd),h(Ut,"alt","GPU Step 2"),h(Me,"id","install-rlbaselines3-zoo-and-its-dependencies"),h(Me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Me,"href","#install-rlbaselines3-zoo-and-its-dependencies"),h(X,"class","relative group"),h(Le,"id","create-a-virtual-display"),h(Le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Le,"href","#create-a-virtual-display"),h(D,"class","relative group"),h(Re,"id","train-our-deep-qlearning-agent-to-play-space-invaders"),h(Re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Re,"href","#train-our-deep-qlearning-agent-to-play-space-invaders"),h(O,"class","relative group"),h(Kt,"href","https://stable-baselines3.readthedocs.io/en/master/modules/dqn.html#parameters"),h(Kt,"rel","nofollow"),h(el,"start","2"),h(Ne,"id","solution"),h(Ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ne,"href","#solution"),h(Y,"class","relative group"),h(Se,"id","lets-evaluate-our-agent"),h(Se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Se,"href","#lets-evaluate-our-agent"),h(x,"class","relative group"),h(Ge,"id","solution"),h(Ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ge,"href","#solution"),h(q,"class","relative group"),h(Je,"id","publish-our-trained-model-on-the-hub"),h(Je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Je,"href","#publish-our-trained-model-on-the-hub"),h(V,"class","relative group"),ne(fl.src,Vd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit3/space-invaders-model.gif")||h(fl,"src",Vd),h(fl,"alt","Space Invaders model"),h(ml,"href","https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),h(ml,"rel","nofollow"),h(vl,"href","https://huggingface.co/join"),h(vl,"rel","nofollow"),h(yl,"href","https://huggingface.co/settings/tokens"),h(yl,"rel","nofollow"),ne(gl.src,Kd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/create-token.jpg")||h(gl,"src",Kd),h(gl,"alt","Create HF Token"),ne($l.src,ev="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit3/select-id.png")||h($l,"src",ev),h($l,"alt","Select Id"),h(Ue,"id","solution"),h(Ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ue,"href","#solution"),h(ee,"class","relative group"),h(Ml,"href","https://huggingface.co/ThomasSimonini/dqn-SpaceInvadersNoFrameskip-v4"),h(Ml,"rel","nofollow"),h(Rl,"href","https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),h(Rl,"rel","nofollow"),h(Ze,"id","load-a-powerful-trained-model"),h(Ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ze,"href","#load-a-powerful-trained-model"),h(le,"class","relative group"),h(Gl,"href","https://huggingface.co/sb3"),h(Gl,"rel","nofollow"),h(Jl,"href","https://huggingface.co/sb3/dqn-AsteroidsNoFrameskip-v4"),h(Jl,"rel","nofollow"),h(Bl,"href","https://huggingface.co/sb3/dqn-BeamRiderNoFrameskip-v4"),h(Bl,"rel","nofollow"),h(Fl,"href","https://huggingface.co/sb3/dqn-BreakoutNoFrameskip-v4"),h(Fl,"rel","nofollow"),h(Pl,"href","https://huggingface.co/sb3/dqn-RoadRunnerNoFrameskip-v4"),h(Pl,"rel","nofollow"),h(Cl,"href","https://huggingface.co/sb3/dqn-BeamRiderNoFrameskip-v4"),h(Cl,"rel","nofollow"),h(Hl,"start","2"),h(Dl,"href","https://huggingface.co/sb3/dqn-BeamRiderNoFrameskip-v4#hyperparameters"),h(Dl,"rel","nofollow"),h(We,"id","some-additional-challenges"),h(We,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(We,"href","#some-additional-challenges"),h(ae,"class","relative group"),h(Yl,"href","https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),h(Yl,"rel","nofollow"),h(xl,"href","https://github.com/vwxyzjn/cleanrl/blob/master/cleanrl/dqn_atari.py"),h(xl,"rel","nofollow"),ne(Ql.src,tv="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/atari-envs.gif")||h(Ql,"src",tv),h(Ql,"alt","Environments"),h(Vl,"href","https://optuna.org/"),h(Vl,"rel","nofollow"),h(Ye,"id","this-is-a-course-built-with-you"),h(Ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ye,"href","#this-is-a-course-built-with-you"),h(se,"class","relative group"),h(to,"href","https://forms.gle/3HgA7bEHwAmmLfwh9"),h(to,"rel","nofollow"),h(lo,"href","https://github.com/huggingface/deep-rl-class/issues"),h(lo,"rel","nofollow"),h(xe,"id","keep-learning-stay-awesome"),h(xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(xe,"href","#keep-learning-stay-awesome"),h(ie,"class","relative group")},m(e,s){l(document.head,G),p(e,gs,s),p(e,J,s),l(J,pe),l(pe,xo),d(at,xo,null),l(J,Up),l(J,Qo),l(Qo,Cp),p(e,ws,s),d(rt,e,s),p(e,_s,s),p(e,fe,s),l(fe,Zp),l(fe,qo),l(qo,Hp),l(fe,Wp),p(e,bs,s),p(e,st,s),p(e,Es,s),p(e,ue,s),l(ue,Xp),l(ue,it),l(it,Dp),l(ue,Op),p(e,$s,s),p(e,B,s),l(B,zp),l(B,Vo),l(Vo,Yp),l(B,xp),l(B,nt),l(nt,Qp),p(e,ks,s),p(e,he,s),l(he,qp),l(he,Ko),l(Ko,Vp),l(he,Kp),p(e,Ts,s),p(e,pt,s),l(pt,ef),l(pt,ea),l(ea,tf),p(e,Is,s),p(e,ao,s),l(ao,ta),l(ta,lf),p(e,Ms,s),p(e,ft,s),l(ft,of),l(ft,ut),l(ut,af),p(e,Ls,s),p(e,ht,s),l(ht,rf),l(ht,ct),l(ct,sf),p(e,As,s),p(e,mt,s),l(mt,la),l(la,nf),l(mt,pf),p(e,Rs,s),p(e,ro,s),l(ro,dt),l(dt,so),p(e,Ns,s),p(e,F,s),l(F,ce),l(ce,oa),d(vt,oa,null),l(F,ff),l(F,aa),l(aa,uf),p(e,Ss,s),p(e,yt,s),p(e,js,s),p(e,b,s),l(b,hf),l(b,ra),l(ra,cf),l(b,mf),l(b,gt),l(gt,df),l(b,vf),l(b,wt),l(wt,yf),l(b,gf),p(e,Gs,s),p(e,me,s),l(me,wf),l(me,_t),l(_t,_f),l(me,bf),p(e,Js,s),p(e,P,s),l(P,de),l(de,sa),d(bt,sa,null),l(P,Ef),l(P,ia),l(ia,$f),p(e,Bs,s),p(e,io,s),l(io,na),l(na,Et),l(Et,kf),p(e,Fs,s),p(e,$t,s),l($t,Tf),l($t,kt),l(kt,If),p(e,Ps,s),p(e,U,s),l(U,ve),l(ve,pa),d(Tt,pa,null),l(U,Mf),l(U,fa),l(fa,Lf),p(e,Us,s),p(e,no,s),l(no,ua),l(ua,It),l(It,Af),p(e,Cs,s),p(e,C,s),l(C,ye),l(ye,ha),d(Mt,ha,null),l(C,Rf),l(C,ca),l(ca,Nf),p(e,Zs,s),p(e,po,s),l(po,Sf),p(e,Hs,s),p(e,ge,s),l(ge,Lt),l(Lt,jf),l(Lt,ma),l(ma,Gf),l(Lt,Jf),l(ge,Bf),l(ge,At),l(At,Ff),l(At,da),l(da,Pf),l(At,Uf),p(e,Ws,s),p(e,Z,s),l(Z,we),l(we,va),d(Rt,va,null),l(Z,Cf),l(Z,ya),l(ya,Zf),p(e,Xs,s),p(e,fo,s),l(fo,Hf),p(e,Ds,s),p(e,_e,s),l(_e,Wf),l(_e,ga),l(ga,Nt),l(Nt,Xf),l(_e,Df),p(e,Os,s),p(e,A,s),l(A,Of),l(A,wa),l(wa,zf),l(A,Yf),l(A,St),l(St,xf),l(A,Qf),p(e,zs,s),p(e,H,s),l(H,be),l(be,_a),d(jt,_a,null),l(H,qf),l(H,ba),l(ba,Vf),p(e,Ys,s),p(e,Ee,s),l(Ee,Kf),l(Ee,Ea),l(Ea,eu),l(Ee,tu),p(e,xs,s),p(e,$e,s),l($e,lu),l($e,$a),l($a,ou),l($e,au),p(e,Qs,s),p(e,ke,s),l(ke,ru),l(ke,ka),l(ka,su),l(ke,iu),p(e,qs,s),p(e,Gt,s),l(Gt,nu),l(Gt,Ta),l(Ta,pu),p(e,Vs,s),p(e,Jt,s),l(Jt,fu),l(Jt,Bt),l(Bt,uu),p(e,Ks,s),p(e,W,s),l(W,Te),l(Te,Ia),d(Ft,Ia,null),l(W,hu),l(W,Ma),l(Ma,cu),p(e,ei,s),p(e,uo,s),l(uo,Ie),l(Ie,mu),l(Ie,La),l(La,du),l(Ie,vu),l(Ie,Aa),l(Aa,yu),p(e,ti,s),p(e,Pt,s),p(e,li,s),p(e,ho,s),l(ho,Ra),l(Ra,Na),l(Na,gu),p(e,oi,s),p(e,Ut,s),p(e,ai,s),p(e,X,s),l(X,Me),l(Me,Sa),d(Ct,Sa,null),l(X,wu),l(X,ja),l(ja,_u),p(e,ri,s),p(e,R,s),l(R,bu),l(R,Ga),l(Ga,Eu),l(R,$u),l(R,Ja),l(Ja,ku),l(R,Tu),p(e,si,s),d(Zt,e,s),p(e,ii,s),p(e,co,s),l(co,Iu),p(e,ni,s),d(Ht,e,s),p(e,pi,s),d(Wt,e,s),p(e,fi,s),p(e,mo,s),l(mo,Mu),p(e,ui,s),d(Xt,e,s),p(e,hi,s),p(e,D,s),l(D,Le),l(Le,Ba),d(Dt,Ba,null),l(D,Lu),l(D,Fa),l(Fa,Au),p(e,ci,s),p(e,Ae,s),l(Ae,Ru),l(Ae,Pa),l(Pa,Nu),l(Ae,Su),p(e,mi,s),p(e,vo,s),l(vo,ju),p(e,di,s),d(Ot,e,s),p(e,vi,s),d(zt,e,s),p(e,yi,s),p(e,O,s),l(O,Re),l(Re,Ua),d(Yt,Ua,null),l(O,Gu),l(O,Ca),l(Ca,Ju),p(e,gi,s),p(e,yo,s),l(yo,Bu),p(e,wi,s),p(e,go,s),l(go,xt),l(xt,Fu),l(xt,Za),l(Za,Pu),l(xt,Uu),p(e,_i,s),p(e,wo,s),l(wo,Cu),p(e,bi,s),d(Qt,e,s),p(e,Ei,s),p(e,_o,s),l(_o,Zu),p(e,$i,s),p(e,E,s),l(E,qt),l(qt,Hu),l(qt,Ha),l(Ha,Wu),l(qt,Xu),l(E,Du),l(E,Vt),l(Vt,Ou),l(Vt,Wa),l(Wa,zu),l(Vt,Yu),l(E,xu),l(E,bo),l(bo,Qu),l(bo,Xa),l(Xa,qu),l(E,Vu),l(E,Da),l(Da,Ku),p(e,ki,s),p(e,$,s),l($,eh),l($,Oa),l(Oa,th),l($,lh),l($,za),l(za,oh),l($,ah),l($,Ya),l(Ya,rh),l($,sh),p(e,Ti,s),p(e,Eo,s),l(Eo,ih),p(e,Ii,s),p(e,N,s),l(N,xa),l(xa,Qa),l(Qa,nh),l(N,ph),l(N,qa),l(qa,Va),l(Va,fh),l(N,uh),l(N,Ka),l(Ka,er),l(er,hh),p(e,Mi,s),p(e,z,s),l(z,ch),l(z,tr),l(tr,mh),l(z,dh),l(z,Kt),l(Kt,vh),p(e,Li,s),p(e,el,s),l(el,tl),l(tl,yh),l(tl,lr),l(lr,gh),l(tl,wh),p(e,Ai,s),p(e,$o,s),l($o,L),l(L,_h),l(L,or),l(or,bh),l(L,Eh),l(L,ar),l(ar,$h),l(L,kh),l(L,rr),l(rr,Th),l(L,Ih),p(e,Ri,s),d(ll,e,s),p(e,Ni,s),p(e,Y,s),l(Y,Ne),l(Ne,sr),d(ol,sr,null),l(Y,Mh),l(Y,ir),l(ir,Lh),p(e,Si,s),d(al,e,s),p(e,ji,s),p(e,x,s),l(x,Se),l(Se,nr),d(rl,nr,null),l(x,Ah),l(x,pr),l(pr,Rh),p(e,Gi,s),p(e,je,s),l(je,Q),l(Q,Nh),l(Q,fr),l(fr,Sh),l(Q,jh),l(Q,ur),l(ur,Gh),l(Q,Jh),l(je,Bh),l(je,hr),l(hr,Fh),p(e,Ji,s),d(sl,e,s),p(e,Bi,s),p(e,q,s),l(q,Ge),l(Ge,cr),d(il,cr,null),l(q,Ph),l(q,mr),l(mr,Uh),p(e,Fi,s),d(nl,e,s),p(e,Pi,s),p(e,V,s),l(V,Je),l(Je,dr),d(pl,dr,null),l(V,Ch),l(V,vr),l(vr,Zh),p(e,Ui,s),p(e,fl,s),p(e,Ci,s),p(e,S,s),l(S,Hh),l(S,yr),l(yr,Wh),l(S,Xh),l(S,gr),l(gr,Dh),l(S,Oh),p(e,Zi,s),p(e,ko,s),l(ko,zh),p(e,Hi,s),p(e,k,s),l(k,ul),l(ul,Yh),l(ul,wr),l(wr,xh),l(ul,Qh),l(k,qh),l(k,hl),l(hl,Vh),l(hl,_r),l(_r,Kh),l(hl,ec),l(k,tc),l(k,cl),l(cl,lc),l(cl,br),l(br,oc),l(cl,ac),l(k,rc),l(k,Be),l(Be,sc),l(Be,Er),l(Er,ic),l(Be,nc),l(Be,ml),l(ml,pc),p(e,Wi,s),p(e,To,s),l(To,fc),p(e,Xi,s),p(e,dl,s),l(dl,uc),l(dl,vl),l(vl,hc),p(e,Di,s),p(e,Io,s),l(Io,cc),p(e,Oi,s),p(e,Mo,s),l(Mo,Fe),l(Fe,mc),l(Fe,yl),l(yl,dc),l(Fe,vc),l(Fe,$r),l($r,yc),p(e,zi,s),p(e,gl,s),p(e,Yi,s),p(e,Pe,s),l(Pe,kr),l(kr,gc),l(Pe,wc),l(Pe,Tr),l(Tr,_c),p(e,xi,s),d(wl,e,s),p(e,Qi,s),p(e,_l,s),l(_l,bc),l(_l,Ir),l(Ir,Ec),p(e,qi,s),p(e,Lo,s),l(Lo,$c),p(e,Vi,s),p(e,Ao,s),l(Ao,kc),p(e,Ki,s),p(e,bl,s),l(bl,Mr),l(Mr,Tc),l(bl,Ic),p(e,en,s),p(e,El,s),l(El,Lr),l(Lr,Mc),l(El,Lc),p(e,tn,s),p(e,K,s),l(K,Ar),l(Ar,Ac),l(K,Rc),l(K,Rr),l(Rr,Nc),l(K,Sc),p(e,ln,s),p(e,$l,s),p(e,on,s),d(kl,e,s),p(e,an,s),p(e,ee,s),l(ee,Ue),l(Ue,Nr),d(Tl,Nr,null),l(ee,jc),l(ee,Sr),l(Sr,Gc),p(e,rn,s),d(Il,e,s),p(e,sn,s),p(e,Ro,s),l(Ro,Jc),p(e,nn,s),p(e,Ce,s),l(Ce,Bc),l(Ce,Ml),l(Ml,Fc),l(Ce,Pc),p(e,pn,s),p(e,T,s),l(T,Ll),l(Ll,Uc),l(Ll,jr),l(jr,Cc),l(Ll,Zc),l(T,Hc),l(T,Gr),l(Gr,Wc),l(T,Xc),l(T,Jr),l(Jr,Dc),l(T,Oc),l(T,Al),l(Al,zc),l(Al,Br),l(Br,Yc),l(Al,xc),p(e,fn,s),p(e,No,s),l(No,Qc),p(e,un,s),p(e,te,s),l(te,Fr),l(Fr,qc),l(te,Vc),l(te,Rl),l(Rl,Kc),l(te,em),p(e,hn,s),p(e,le,s),l(le,Ze),l(Ze,Pr),d(Nl,Pr,null),l(le,tm),l(le,Ur),l(Ur,lm),p(e,cn,s),p(e,So,s),l(So,Sl),l(Sl,om),l(Sl,Cr),l(Cr,am),l(Sl,rm),p(e,mn,s),p(e,jl,s),l(jl,sm),l(jl,Gl),l(Gl,im),p(e,dn,s),p(e,jo,s),l(jo,nm),p(e,vn,s),p(e,I,s),l(I,Go),l(Go,pm),l(Go,Jl),l(Jl,fm),l(I,um),l(I,Jo),l(Jo,hm),l(Jo,Bl),l(Bl,cm),l(I,mm),l(I,Bo),l(Bo,dm),l(Bo,Fl),l(Fl,vm),l(I,ym),l(I,Fo),l(Fo,gm),l(Fo,Pl),l(Pl,wm),p(e,yn,s),p(e,Ul,s),l(Ul,_m),l(Ul,Cl),l(Cl,bm),p(e,gn,s),p(e,Po,s),l(Po,He),l(He,Em),l(He,Zr),l(Zr,$m),l(He,km),l(He,Hr),l(Hr,Tm),p(e,wn,s),d(Zl,e,s),p(e,_n,s),p(e,Hl,s),l(Hl,Wr),l(Wr,Im),p(e,bn,s),d(Wl,e,s),p(e,En,s),p(e,Xl,s),l(Xl,Mm),l(Xl,Xr),l(Xr,Lm),p(e,$n,s),p(e,oe,s),l(oe,Am),l(oe,Dl),l(Dl,Rm),l(oe,Nm),l(oe,Dr),l(Dr,Sm),p(e,kn,s),p(e,Ol,s),l(Ol,jm),l(Ol,Or),l(Or,Gm),p(e,Tn,s),p(e,ae,s),l(ae,We),l(We,zr),d(zl,zr,null),l(ae,Jm),l(ae,Yr),l(Yr,Bm),p(e,In,s),p(e,Xe,s),l(Xe,Fm),l(Xe,xr),l(xr,Pm),l(Xe,Um),p(e,Mn,s),p(e,De,s),l(De,Cm),l(De,Yl),l(Yl,Zm),l(De,Hm),p(e,Ln,s),p(e,Uo,s),l(Uo,Wm),p(e,An,s),p(e,M,s),l(M,Qr),l(Qr,Xm),l(M,Dm),l(M,qr),l(qr,Om),l(M,zm),l(M,Vr),l(Vr,Ym),l(M,xm),l(M,Kr),l(Kr,Qm),p(e,Rn,s),p(e,re,s),l(re,qm),l(re,es),l(es,Vm),l(re,Km),l(re,xl),l(xl,ed),p(e,Nn,s),p(e,Ql,s),p(e,Sn,s),p(e,jn,s),p(e,Gn,s),p(e,Co,s),l(Co,td),p(e,Jn,s),p(e,ql,s),l(ql,ld),l(ql,ts),l(ts,od),p(e,Bn,s),p(e,Oe,s),l(Oe,ad),l(Oe,ls),l(ls,rd),l(Oe,sd),p(e,Fn,s),p(e,ze,s),l(ze,id),l(ze,Zo),l(Zo,nd),l(Zo,Vl),l(Vl,pd),l(ze,fd),p(e,Pn,s),p(e,se,s),l(se,Ye),l(Ye,os),d(Kl,os,null),l(se,ud),l(se,as),l(as,hd),p(e,Un,s),p(e,eo,s),l(eo,cd),l(eo,to),l(to,md),p(e,Cn,s),p(e,j,s),l(j,dd),l(j,rs),l(rs,vd),l(j,yd),l(j,lo),l(lo,gd),l(j,wd),p(e,Zn,s),p(e,Ho,s),l(Ho,_d),p(e,Hn,s),p(e,ie,s),l(ie,xe),l(xe,ss),d(oo,ss,null),l(ie,bd),l(ie,is),l(is,Ed),Wn=!0},p:Qg,i(e){Wn||(v(at.$$.fragment,e),v(rt.$$.fragment,e),v(vt.$$.fragment,e),v(bt.$$.fragment,e),v(Tt.$$.fragment,e),v(Mt.$$.fragment,e),v(Rt.$$.fragment,e),v(jt.$$.fragment,e),v(Ft.$$.fragment,e),v(Ct.$$.fragment,e),v(Zt.$$.fragment,e),v(Ht.$$.fragment,e),v(Wt.$$.fragment,e),v(Xt.$$.fragment,e),v(Dt.$$.fragment,e),v(Ot.$$.fragment,e),v(zt.$$.fragment,e),v(Yt.$$.fragment,e),v(Qt.$$.fragment,e),v(ll.$$.fragment,e),v(ol.$$.fragment,e),v(al.$$.fragment,e),v(rl.$$.fragment,e),v(sl.$$.fragment,e),v(il.$$.fragment,e),v(nl.$$.fragment,e),v(pl.$$.fragment,e),v(wl.$$.fragment,e),v(kl.$$.fragment,e),v(Tl.$$.fragment,e),v(Il.$$.fragment,e),v(Nl.$$.fragment,e),v(Zl.$$.fragment,e),v(Wl.$$.fragment,e),v(zl.$$.fragment,e),v(Kl.$$.fragment,e),v(oo.$$.fragment,e),Wn=!0)},o(e){y(at.$$.fragment,e),y(rt.$$.fragment,e),y(vt.$$.fragment,e),y(bt.$$.fragment,e),y(Tt.$$.fragment,e),y(Mt.$$.fragment,e),y(Rt.$$.fragment,e),y(jt.$$.fragment,e),y(Ft.$$.fragment,e),y(Ct.$$.fragment,e),y(Zt.$$.fragment,e),y(Ht.$$.fragment,e),y(Wt.$$.fragment,e),y(Xt.$$.fragment,e),y(Dt.$$.fragment,e),y(Ot.$$.fragment,e),y(zt.$$.fragment,e),y(Yt.$$.fragment,e),y(Qt.$$.fragment,e),y(ll.$$.fragment,e),y(ol.$$.fragment,e),y(al.$$.fragment,e),y(rl.$$.fragment,e),y(sl.$$.fragment,e),y(il.$$.fragment,e),y(nl.$$.fragment,e),y(pl.$$.fragment,e),y(wl.$$.fragment,e),y(kl.$$.fragment,e),y(Tl.$$.fragment,e),y(Il.$$.fragment,e),y(Nl.$$.fragment,e),y(Zl.$$.fragment,e),y(Wl.$$.fragment,e),y(zl.$$.fragment,e),y(Kl.$$.fragment,e),y(oo.$$.fragment,e),Wn=!1},d(e){t(G),e&&t(gs),e&&t(J),g(at),e&&t(ws),g(rt,e),e&&t(_s),e&&t(fe),e&&t(bs),e&&t(st),e&&t(Es),e&&t(ue),e&&t($s),e&&t(B),e&&t(ks),e&&t(he),e&&t(Ts),e&&t(pt),e&&t(Is),e&&t(ao),e&&t(Ms),e&&t(ft),e&&t(Ls),e&&t(ht),e&&t(As),e&&t(mt),e&&t(Rs),e&&t(ro),e&&t(Ns),e&&t(F),g(vt),e&&t(Ss),e&&t(yt),e&&t(js),e&&t(b),e&&t(Gs),e&&t(me),e&&t(Js),e&&t(P),g(bt),e&&t(Bs),e&&t(io),e&&t(Fs),e&&t($t),e&&t(Ps),e&&t(U),g(Tt),e&&t(Us),e&&t(no),e&&t(Cs),e&&t(C),g(Mt),e&&t(Zs),e&&t(po),e&&t(Hs),e&&t(ge),e&&t(Ws),e&&t(Z),g(Rt),e&&t(Xs),e&&t(fo),e&&t(Ds),e&&t(_e),e&&t(Os),e&&t(A),e&&t(zs),e&&t(H),g(jt),e&&t(Ys),e&&t(Ee),e&&t(xs),e&&t($e),e&&t(Qs),e&&t(ke),e&&t(qs),e&&t(Gt),e&&t(Vs),e&&t(Jt),e&&t(Ks),e&&t(W),g(Ft),e&&t(ei),e&&t(uo),e&&t(ti),e&&t(Pt),e&&t(li),e&&t(ho),e&&t(oi),e&&t(Ut),e&&t(ai),e&&t(X),g(Ct),e&&t(ri),e&&t(R),e&&t(si),g(Zt,e),e&&t(ii),e&&t(co),e&&t(ni),g(Ht,e),e&&t(pi),g(Wt,e),e&&t(fi),e&&t(mo),e&&t(ui),g(Xt,e),e&&t(hi),e&&t(D),g(Dt),e&&t(ci),e&&t(Ae),e&&t(mi),e&&t(vo),e&&t(di),g(Ot,e),e&&t(vi),g(zt,e),e&&t(yi),e&&t(O),g(Yt),e&&t(gi),e&&t(yo),e&&t(wi),e&&t(go),e&&t(_i),e&&t(wo),e&&t(bi),g(Qt,e),e&&t(Ei),e&&t(_o),e&&t($i),e&&t(E),e&&t(ki),e&&t($),e&&t(Ti),e&&t(Eo),e&&t(Ii),e&&t(N),e&&t(Mi),e&&t(z),e&&t(Li),e&&t(el),e&&t(Ai),e&&t($o),e&&t(Ri),g(ll,e),e&&t(Ni),e&&t(Y),g(ol),e&&t(Si),g(al,e),e&&t(ji),e&&t(x),g(rl),e&&t(Gi),e&&t(je),e&&t(Ji),g(sl,e),e&&t(Bi),e&&t(q),g(il),e&&t(Fi),g(nl,e),e&&t(Pi),e&&t(V),g(pl),e&&t(Ui),e&&t(fl),e&&t(Ci),e&&t(S),e&&t(Zi),e&&t(ko),e&&t(Hi),e&&t(k),e&&t(Wi),e&&t(To),e&&t(Xi),e&&t(dl),e&&t(Di),e&&t(Io),e&&t(Oi),e&&t(Mo),e&&t(zi),e&&t(gl),e&&t(Yi),e&&t(Pe),e&&t(xi),g(wl,e),e&&t(Qi),e&&t(_l),e&&t(qi),e&&t(Lo),e&&t(Vi),e&&t(Ao),e&&t(Ki),e&&t(bl),e&&t(en),e&&t(El),e&&t(tn),e&&t(K),e&&t(ln),e&&t($l),e&&t(on),g(kl,e),e&&t(an),e&&t(ee),g(Tl),e&&t(rn),g(Il,e),e&&t(sn),e&&t(Ro),e&&t(nn),e&&t(Ce),e&&t(pn),e&&t(T),e&&t(fn),e&&t(No),e&&t(un),e&&t(te),e&&t(hn),e&&t(le),g(Nl),e&&t(cn),e&&t(So),e&&t(mn),e&&t(jl),e&&t(dn),e&&t(jo),e&&t(vn),e&&t(I),e&&t(yn),e&&t(Ul),e&&t(gn),e&&t(Po),e&&t(wn),g(Zl,e),e&&t(_n),e&&t(Hl),e&&t(bn),g(Wl,e),e&&t(En),e&&t(Xl),e&&t($n),e&&t(oe),e&&t(kn),e&&t(Ol),e&&t(Tn),e&&t(ae),g(zl),e&&t(In),e&&t(Xe),e&&t(Mn),e&&t(De),e&&t(Ln),e&&t(Uo),e&&t(An),e&&t(M),e&&t(Rn),e&&t(re),e&&t(Nn),e&&t(Ql),e&&t(Sn),e&&t(jn),e&&t(Gn),e&&t(Co),e&&t(Jn),e&&t(ql),e&&t(Bn),e&&t(Oe),e&&t(Fn),e&&t(ze),e&&t(Pn),e&&t(se),g(Kl),e&&t(Un),e&&t(eo),e&&t(Cn),e&&t(j),e&&t(Zn),e&&t(Ho),e&&t(Hn),e&&t(ie),g(oo)}}}const ew={local:"install-rlbaselines3-zoo-and-its-dependencies",sections:[{local:"create-a-virtual-display",title:"Create a virtual display \u{1F53D}"},{local:"train-our-deep-qlearning-agent-to-play-space-invaders",sections:[{local:"solution",title:"Solution"}],title:"Train our Deep Q-Learning Agent to Play Space Invaders \u{1F47E}"},{local:"lets-evaluate-our-agent",sections:[{local:"solution",title:"Solution"}],title:"Let's evaluate our agent \u{1F440}"},{local:"publish-our-trained-model-on-the-hub",sections:[{local:"solution",title:"Solution"}],title:"Publish our trained model on the Hub \u{1F680}"},{local:"load-a-powerful-trained-model",title:"Load a powerful trained model \u{1F525}"},{local:"some-additional-challenges",sections:[{local:"this-is-a-course-built-with-you",title:"This is a course built with you \u{1F477}\u{1F3FF}\u200D\u2640\uFE0F"},{local:"keep-learning-stay-awesome",title:"Keep Learning, Stay Awesome \u{1F917}"}],title:"Some additional challenges \u{1F3C6}"}],title:"Install RL-Baselines3 Zoo and its dependencies \u{1F4DA}"};function tw(Od){return qg(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class sw extends Og{constructor(G){super();zg(this,G,tw,Kg,Yg,{})}}export{sw as default,ew as metadata};
