import{S as Ob,i as Kb,s as ew,e as a,k as f,w as u,t as n,M as tw,c as s,d as t,m as c,a as o,x as m,h as i,b as p,N as fe,G as l,g as h,y as d,L as lw,q as v,o as y,B as b,v as aw}from"../../chunks/vendor-hf-doc-builder.js";import{I as w}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as _}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as sw}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function ow(od){let A,fr,N,ce,Va,ht,Th,za,Jh,cr,pt,ur,ue,$h,Xa,kh,jh,mr,me,Ha,Ih,Sh,La,Ch,dr,Kl,Gh,vr,de,xa,ft,Bh,Zh,Da,ct,Ah,yr,ut,rd,br,ea,Nh,wr,ve,ta,qa,Wh,Ph,Rh,la,Oa,Qh,Yh,_r,W,Fh,mt,Vh,zh,Ka,Xh,gr,aa,es,Hh,Mr,dt,Lh,vt,xh,Er,yt,ts,Dh,qh,Ur,sa,bt,oa,nd,Tr,P,ye,ls,wt,Oh,as,Kh,Jr,R,be,ss,_t,ep,os,tp,$r,we,rs,gt,lp,ap,ns,Mt,sp,kr,Q,_e,is,Et,op,hs,rp,jr,ra,ps,Ut,np,Ir,k,ip,fs,hp,pp,Tt,fp,cp,Sr,Y,ge,cs,Jt,up,us,mp,Cr,na,dp,Gr,T,F,vp,ms,yp,bp,ds,wp,_p,gp,$t,Mp,vs,Ep,Up,Tp,kt,Jp,ys,$p,kp,jp,jt,Ip,bs,Sp,Cp,Br,V,Me,ws,It,Gp,_s,Bp,Zr,Ee,Zp,St,Ap,Np,Ar,z,Ue,gs,Ct,Wp,Ms,Pp,Nr,X,Te,Es,Gt,Rp,Us,Qp,Wr,ia,Je,Yp,Ts,Fp,Vp,Js,zp,Pr,Bt,id,Rr,ha,$s,ks,Xp,Qr,Zt,hd,Yr,H,$e,js,At,Hp,Is,Lp,Fr,ke,xp,Ss,Dp,qp,Vr,pa,Op,zr,Nt,Xr,Wt,Hr,L,je,Cs,Pt,Kp,Gs,ef,Lr,E,fa,Bs,tf,lf,af,ca,Zs,sf,of,rf,ua,As,nf,hf,pf,ma,Ns,ff,cf,uf,da,Ws,mf,df,xr,Rt,Dr,x,Ie,Ps,Qt,vf,Rs,yf,qr,Yt,Or,D,Se,Qs,Ft,bf,Ys,wf,Kr,q,Ce,Fs,Vt,_f,Vs,gf,en,O,Ge,zs,zt,Mf,Xs,Ef,tn,Xt,Uf,Ht,Tf,ln,Lt,an,xt,sn,Be,Jf,Dt,$f,kf,on,qt,pd,rn,Ot,nn,Ze,jf,Kt,If,Sf,hn,el,fd,pn,K,Ae,Hs,tl,Cf,Ls,Gf,fn,Ne,Bf,ll,Zf,Af,cn,va,Nf,un,al,Wf,xs,Pf,mn,ya,sl,Rf,dn,ol,vn,ee,We,Ds,rl,Qf,qs,Yf,yn,nl,bn,te,Pe,Os,il,Ff,Ks,Vf,wn,ba,zf,_n,hl,Xf,pl,Hf,gn,Re,Lf,fl,xf,Df,Mn,cl,En,le,Qe,eo,ul,qf,to,Of,Un,ml,Tn,ae,Ye,lo,dl,Kf,ao,ec,Jn,wa,so,tc,$n,vl,kn,yl,jn,se,Fe,oo,bl,lc,ro,ac,In,wl,Sn,oe,Ve,no,_l,sc,io,oc,Cn,gl,rc,Ml,nc,Gn,_a,ic,Bn,El,cd,Zn,j,hc,ho,pc,fc,po,cc,uc,An,ga,mc,Nn,J,Ul,dc,fo,vc,yc,bc,Tl,wc,co,_c,gc,Mc,Jl,Ec,uo,Uc,Tc,Jc,ze,$c,mo,kc,jc,$l,Ic,Wn,Ma,Sc,Pn,kl,Cc,jl,Gc,Rn,Ea,Bc,Qn,Ua,Xe,Zc,Il,Ac,Nc,vo,Wc,Yn,Sl,ud,Fn,He,yo,Pc,Rc,bo,Qc,Vn,Cl,zn,Gl,Yc,wo,Fc,Xn,Le,Vc,_o,zc,Xc,Hn,Bl,Ln,re,xe,go,Zl,Hc,Mo,Lc,xn,ne,De,Eo,Al,xc,Uo,Dc,Dn,Ta,qc,qn,qe,Oc,To,Kc,eu,On,Oe,tu,Jo,lu,au,Kn,U,su,$o,ou,ru,ko,nu,iu,jo,hu,pu,Io,fu,cu,ei,I,uu,So,mu,du,Co,vu,yu,ti,Nl,md,li,Ke,bu,Go,wu,_u,ai,et,gu,Bo,Mu,Eu,si,Wl,oi,Pl,ri,tt,Uu,Zo,Tu,Ju,ni,S,Ja,Ao,$u,ku,ju,$a,No,Iu,Su,Cu,ka,Wo,Gu,Bu,ii,lt,Zu,Po,Au,Nu,hi,Rl,pi,ja,Wu,fi,Ia,Ro,Pu,ci,Sa,Ru,ui,g,Qo,Qu,Yu,Yo,Fu,Vu,Ca,zu,Ql,Xu,Hu,Fo,Lu,xu,Vo,Du,qu,zo,Ou,Ku,Xo,em,tm,Yl,lm,Ho,am,sm,mi,ie,at,Lo,Fl,om,xo,rm,di,Vl,vi,zl,yi,he,st,Do,Xl,nm,qo,im,bi,$,hm,Oo,pm,fm,Ko,cm,um,er,mm,dm,wi,C,vm,tr,ym,bm,lr,wm,_m,_i,Hl,gm,Ll,Mm,gi,xl,Em,Dl,Um,Mi,Ga,Tm,Ei,G,ar,Jm,$m,Ba,km,ql,jm,Im,Za,sr,Sm,Cm,Ui,Aa,Gm,Ti,pe,ot,or,Ol,Bm,rr,Zm,Ji;return ht=new w({}),pt=new sw({props:{classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/deep-rl-class/blob/main/notebooks/unit6/unit6.ipynb"}],askForHelpUrl:"http://hf.co/join/discord"}}),wt=new w({}),_t=new w({}),Et=new w({}),Jt=new w({}),It=new w({}),Ct=new w({}),Gt=new w({}),At=new w({}),Nt=new _({props:{code:"JTI1JTI1Y2FwdHVyZSUwQSFhcHQlMjBpbnN0YWxsJTIwcHl0aG9uLW9wZW5nbCUwQSFhcHQlMjBpbnN0YWxsJTIwZmZtcGVnJTBBIWFwdCUyMGluc3RhbGwlMjB4dmZiJTBBIXBpcDMlMjBpbnN0YWxsJTIwcHl2aXJ0dWFsZGlzcGxheQ==",highlighted:`%%capture
!apt install python-opengl
!apt install ffmpeg
!apt install xvfb
!pip3 install pyvirtualdisplay`}}),Wt=new _({props:{code:"JTIzJTIwVmlydHVhbCUyMGRpc3BsYXklMEFmcm9tJTIwcHl2aXJ0dWFsZGlzcGxheSUyMGltcG9ydCUyMERpc3BsYXklMEElMEF2aXJ0dWFsX2Rpc3BsYXklMjAlM0QlMjBEaXNwbGF5KHZpc2libGUlM0QwJTJDJTIwc2l6ZSUzRCgxNDAwJTJDJTIwOTAwKSklMEF2aXJ0dWFsX2Rpc3BsYXkuc3RhcnQoKQ==",highlighted:`<span class="hljs-comment"># Virtual display</span>
<span class="hljs-keyword">from</span> pyvirtualdisplay <span class="hljs-keyword">import</span> Display

virtual_display = Display(visible=<span class="hljs-number">0</span>, size=(<span class="hljs-number">1400</span>, <span class="hljs-number">900</span>))
virtual_display.start()`}}),Pt=new w({}),Rt=new _({props:{code:"IXBpcCUyMGluc3RhbGwlMjBzdGFibGUtYmFzZWxpbmVzMyU1QmV4dHJhJTVEJTNEJTNEMS44LjAlMEEhcGlwJTIwaW5zdGFsbCUyMGh1Z2dpbmdmYWNlX3NiMyUwQSFwaXAlMjBpbnN0YWxsJTIwcGFuZGFfZ3ltJTNEJTNEMi4wLjAlMEEhcGlwJTIwaW5zdGFsbCUyMHB5Z2xldCUzRCUzRDEuNS4x",highlighted:`!pip install stable-baselines3[extra]==1.8.0
!pip install huggingface_sb3
!pip install panda_gym==2.0.0
!pip install pyglet==1.5.1`}}),Qt=new w({}),Yt=new _({props:{code:"aW1wb3J0JTIwcHlidWxsZXRfZW52cyUwQWltcG9ydCUyMHBhbmRhX2d5bSUwQWltcG9ydCUyMGd5bSUwQSUwQWltcG9ydCUyMG9zJTBBJTBBZnJvbSUyMGh1Z2dpbmdmYWNlX3NiMyUyMGltcG9ydCUyMGxvYWRfZnJvbV9odWIlMkMlMjBwYWNrYWdlX3RvX2h1YiUwQSUwQWZyb20lMjBzdGFibGVfYmFzZWxpbmVzMyUyMGltcG9ydCUyMEEyQyUwQWZyb20lMjBzdGFibGVfYmFzZWxpbmVzMy5jb21tb24uZXZhbHVhdGlvbiUyMGltcG9ydCUyMGV2YWx1YXRlX3BvbGljeSUwQWZyb20lMjBzdGFibGVfYmFzZWxpbmVzMy5jb21tb24udmVjX2VudiUyMGltcG9ydCUyMER1bW15VmVjRW52JTJDJTIwVmVjTm9ybWFsaXplJTBBZnJvbSUyMHN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi5lbnZfdXRpbCUyMGltcG9ydCUyMG1ha2VfdmVjX2VudiUwQSUwQWZyb20lMjBodWdnaW5nZmFjZV9odWIlMjBpbXBvcnQlMjBub3RlYm9va19sb2dpbg==",highlighted:`<span class="hljs-keyword">import</span> pybullet_envs
<span class="hljs-keyword">import</span> panda_gym
<span class="hljs-keyword">import</span> gym

<span class="hljs-keyword">import</span> os

<span class="hljs-keyword">from</span> huggingface_sb3 <span class="hljs-keyword">import</span> load_from_hub, package_to_hub

<span class="hljs-keyword">from</span> stable_baselines3 <span class="hljs-keyword">import</span> A2C
<span class="hljs-keyword">from</span> stable_baselines3.common.evaluation <span class="hljs-keyword">import</span> evaluate_policy
<span class="hljs-keyword">from</span> stable_baselines3.common.vec_env <span class="hljs-keyword">import</span> DummyVecEnv, VecNormalize
<span class="hljs-keyword">from</span> stable_baselines3.common.env_util <span class="hljs-keyword">import</span> make_vec_env

<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login`}}),Ft=new w({}),Vt=new w({}),zt=new w({}),Lt=new _({props:{code:"ZW52X2lkJTIwJTNEJTIwJTIyQW50QnVsbGV0RW52LXYwJTIyJTBBJTIzJTIwQ3JlYXRlJTIwdGhlJTIwZW52JTBBZW52JTIwJTNEJTIwZ3ltLm1ha2UoZW52X2lkKSUwQSUwQSUyMyUyMEdldCUyMHRoZSUyMHN0YXRlJTIwc3BhY2UlMjBhbmQlMjBhY3Rpb24lMjBzcGFjZSUwQXNfc2l6ZSUyMCUzRCUyMGVudi5vYnNlcnZhdGlvbl9zcGFjZS5zaGFwZSU1QjAlNUQlMEFhX3NpemUlMjAlM0QlMjBlbnYuYWN0aW9uX3NwYWNl",highlighted:`env_id = <span class="hljs-string">&quot;AntBulletEnv-v0&quot;</span>
<span class="hljs-comment"># Create the env</span>
env = gym.make(env_id)

<span class="hljs-comment"># Get the state space and action space</span>
s_size = env.observation_space.shape[<span class="hljs-number">0</span>]
a_size = env.action_space`}}),xt=new _({props:{code:"cHJpbnQoJTIyX19fX19PQlNFUlZBVElPTiUyMFNQQUNFX19fX18lMjAlNUNuJTIyKSUwQXByaW50KCUyMlRoZSUyMFN0YXRlJTIwU3BhY2UlMjBpcyUzQSUyMCUyMiUyQyUyMHNfc2l6ZSklMEFwcmludCglMjJTYW1wbGUlMjBvYnNlcnZhdGlvbiUyMiUyQyUyMGVudi5vYnNlcnZhdGlvbl9zcGFjZS5zYW1wbGUoKSklMjAlMjAlMjMlMjBHZXQlMjBhJTIwcmFuZG9tJTIwb2JzZXJ2YXRpb24=",highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;_____OBSERVATION SPACE_____ \\n&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;The State Space is: &quot;</span>, s_size)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Sample observation&quot;</span>, env.observation_space.sample())  <span class="hljs-comment"># Get a random observation</span>`}}),Ot=new _({props:{code:"cHJpbnQoJTIyJTVDbiUyMF9fX19fQUNUSU9OJTIwU1BBQ0VfX19fXyUyMCU1Q24lMjIpJTBBcHJpbnQoJTIyVGhlJTIwQWN0aW9uJTIwU3BhY2UlMjBpcyUzQSUyMCUyMiUyQyUyMGFfc2l6ZSklMEFwcmludCglMjJBY3Rpb24lMjBTcGFjZSUyMFNhbXBsZSUyMiUyQyUyMGVudi5hY3Rpb25fc3BhY2Uuc2FtcGxlKCkpJTIwJTIwJTIzJTIwVGFrZSUyMGElMjByYW5kb20lMjBhY3Rpb24=",highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;\\n _____ACTION SPACE_____ \\n&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;The Action Space is: &quot;</span>, a_size)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Action Space Sample&quot;</span>, env.action_space.sample())  <span class="hljs-comment"># Take a random action</span>`}}),tl=new w({}),ol=new _({props:{code:"ZW52JTIwJTNEJTIwbWFrZV92ZWNfZW52KGVudl9pZCUyQyUyMG5fZW52cyUzRDQpJTBBJTBBJTIzJTIwQWRkaW5nJTIwdGhpcyUyMHdyYXBwZXIlMjB0byUyMG5vcm1hbGl6ZSUyMHRoZSUyMG9ic2VydmF0aW9uJTIwYW5kJTIwdGhlJTIwcmV3YXJkJTBBZW52JTIwJTNEJTIwJTIzJTIwVE9ETyUzQSUyMEFkZCUyMHRoZSUyMHdyYXBwZXI=",highlighted:`env = make_vec_env(env_id, n_envs=<span class="hljs-number">4</span>)

<span class="hljs-comment"># Adding this wrapper to normalize the observation and the reward</span>
env = <span class="hljs-comment"># <span class="hljs-doctag">TODO:</span> Add the wrapper</span>`}}),rl=new w({}),nl=new _({props:{code:"ZW52JTIwJTNEJTIwbWFrZV92ZWNfZW52KGVudl9pZCUyQyUyMG5fZW52cyUzRDQpJTBBJTBBZW52JTIwJTNEJTIwVmVjTm9ybWFsaXplKGVudiUyQyUyMG5vcm1fb2JzJTNEVHJ1ZSUyQyUyMG5vcm1fcmV3YXJkJTNEVHJ1ZSUyQyUyMGNsaXBfb2JzJTNEMTAuMCk=",highlighted:`env = make_vec_env(env_id, n_envs=<span class="hljs-number">4</span>)

env = VecNormalize(env, norm_obs=<span class="hljs-literal">True</span>, norm_reward=<span class="hljs-literal">True</span>, clip_obs=<span class="hljs-number">10.0</span>)`}}),il=new w({}),cl=new _({props:{code:"bW9kZWwlMjAlM0QlMjAlMjMlMjBDcmVhdGUlMjB0aGUlMjBBMkMlMjBtb2RlbCUyMGFuZCUyMHRyeSUyMHRvJTIwZmluZCUyMHRoZSUyMGJlc3QlMjBwYXJhbWV0ZXJz",highlighted:'model = <span class="hljs-comment"># Create the A2C model and try to find the best parameters</span>'}}),ul=new w({}),ml=new _({props:{code:"bW9kZWwlMjAlM0QlMjBBMkMoJTBBJTIwJTIwJTIwJTIwcG9saWN5JTNEJTIyTWxwUG9saWN5JTIyJTJDJTBBJTIwJTIwJTIwJTIwZW52JTNEZW52JTJDJTBBJTIwJTIwJTIwJTIwZ2FlX2xhbWJkYSUzRDAuOSUyQyUwQSUyMCUyMCUyMCUyMGdhbW1hJTNEMC45OSUyQyUwQSUyMCUyMCUyMCUyMGxlYXJuaW5nX3JhdGUlM0QwLjAwMDk2JTJDJTBBJTIwJTIwJTIwJTIwbWF4X2dyYWRfbm9ybSUzRDAuNSUyQyUwQSUyMCUyMCUyMCUyMG5fc3RlcHMlM0Q4JTJDJTBBJTIwJTIwJTIwJTIwdmZfY29lZiUzRDAuNCUyQyUwQSUyMCUyMCUyMCUyMGVudF9jb2VmJTNEMC4wJTJDJTBBJTIwJTIwJTIwJTIwcG9saWN5X2t3YXJncyUzRGRpY3QobG9nX3N0ZF9pbml0JTNELTIlMkMlMjBvcnRob19pbml0JTNERmFsc2UpJTJDJTBBJTIwJTIwJTIwJTIwbm9ybWFsaXplX2FkdmFudGFnZSUzREZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwdXNlX3Jtc19wcm9wJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHVzZV9zZGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwdmVyYm9zZSUzRDElMkMlMEEp",highlighted:`model = A2C(
    policy=<span class="hljs-string">&quot;MlpPolicy&quot;</span>,
    env=env,
    gae_lambda=<span class="hljs-number">0.9</span>,
    gamma=<span class="hljs-number">0.99</span>,
    learning_rate=<span class="hljs-number">0.00096</span>,
    max_grad_norm=<span class="hljs-number">0.5</span>,
    n_steps=<span class="hljs-number">8</span>,
    vf_coef=<span class="hljs-number">0.4</span>,
    ent_coef=<span class="hljs-number">0.0</span>,
    policy_kwargs=<span class="hljs-built_in">dict</span>(log_std_init=-<span class="hljs-number">2</span>, ortho_init=<span class="hljs-literal">False</span>),
    normalize_advantage=<span class="hljs-literal">False</span>,
    use_rms_prop=<span class="hljs-literal">True</span>,
    use_sde=<span class="hljs-literal">True</span>,
    verbose=<span class="hljs-number">1</span>,
)`}}),dl=new w({}),vl=new _({props:{code:"bW9kZWwubGVhcm4oMl8wMDBfMDAwKQ==",highlighted:'model.learn(<span class="hljs-number">2_000_000</span>)'}}),yl=new _({props:{code:"JTIzJTIwU2F2ZSUyMHRoZSUyMG1vZGVsJTIwYW5kJTIwJTIwVmVjTm9ybWFsaXplJTIwc3RhdGlzdGljcyUyMHdoZW4lMjBzYXZpbmclMjB0aGUlMjBhZ2VudCUwQW1vZGVsLnNhdmUoJTIyYTJjLUFudEJ1bGxldEVudi12MCUyMiklMEFlbnYuc2F2ZSglMjJ2ZWNfbm9ybWFsaXplLnBrbCUyMik=",highlighted:`<span class="hljs-comment"># Save the model and  VecNormalize statistics when saving the agent</span>
model.save(<span class="hljs-string">&quot;a2c-AntBulletEnv-v0&quot;</span>)
env.save(<span class="hljs-string">&quot;vec_normalize.pkl&quot;</span>)`}}),bl=new w({}),wl=new _({props:{code:"ZnJvbSUyMHN0YWJsZV9iYXNlbGluZXMzLmNvbW1vbi52ZWNfZW52JTIwaW1wb3J0JTIwRHVtbXlWZWNFbnYlMkMlMjBWZWNOb3JtYWxpemUlMEElMEElMjMlMjBMb2FkJTIwdGhlJTIwc2F2ZWQlMjBzdGF0aXN0aWNzJTBBZXZhbF9lbnYlMjAlM0QlMjBEdW1teVZlY0VudiglNUJsYW1iZGElM0ElMjBneW0ubWFrZSglMjJBbnRCdWxsZXRFbnYtdjAlMjIpJTVEKSUwQWV2YWxfZW52JTIwJTNEJTIwVmVjTm9ybWFsaXplLmxvYWQoJTIydmVjX25vcm1hbGl6ZS5wa2wlMjIlMkMlMjBldmFsX2VudiklMEElMEElMjMlMjAlMjBkbyUyMG5vdCUyMHVwZGF0ZSUyMHRoZW0lMjBhdCUyMHRlc3QlMjB0aW1lJTBBZXZhbF9lbnYudHJhaW5pbmclMjAlM0QlMjBGYWxzZSUwQSUyMyUyMHJld2FyZCUyMG5vcm1hbGl6YXRpb24lMjBpcyUyMG5vdCUyMG5lZWRlZCUyMGF0JTIwdGVzdCUyMHRpbWUlMEFldmFsX2Vudi5ub3JtX3Jld2FyZCUyMCUzRCUyMEZhbHNlJTBBJTBBJTIzJTIwTG9hZCUyMHRoZSUyMGFnZW50JTBBbW9kZWwlMjAlM0QlMjBBMkMubG9hZCglMjJhMmMtQW50QnVsbGV0RW52LXYwJTIyKSUwQSUwQW1lYW5fcmV3YXJkJTJDJTIwc3RkX3Jld2FyZCUyMCUzRCUyMGV2YWx1YXRlX3BvbGljeShtb2RlbCUyQyUyMGVudiklMEElMEFwcmludChmJTIyTWVhbiUyMHJld2FyZCUyMCUzRCUyMCU3Qm1lYW5fcmV3YXJkJTNBLjJmJTdEJTIwJTJCJTJGLSUyMCU3QnN0ZF9yZXdhcmQlM0EuMmYlN0QlMjIp",highlighted:`<span class="hljs-keyword">from</span> stable_baselines3.common.vec_env <span class="hljs-keyword">import</span> DummyVecEnv, VecNormalize

<span class="hljs-comment"># Load the saved statistics</span>
eval_env = DummyVecEnv([<span class="hljs-keyword">lambda</span>: gym.make(<span class="hljs-string">&quot;AntBulletEnv-v0&quot;</span>)])
eval_env = VecNormalize.load(<span class="hljs-string">&quot;vec_normalize.pkl&quot;</span>, eval_env)

<span class="hljs-comment">#  do not update them at test time</span>
eval_env.training = <span class="hljs-literal">False</span>
<span class="hljs-comment"># reward normalization is not needed at test time</span>
eval_env.norm_reward = <span class="hljs-literal">False</span>

<span class="hljs-comment"># Load the agent</span>
model = A2C.load(<span class="hljs-string">&quot;a2c-AntBulletEnv-v0&quot;</span>)

mean_reward, std_reward = evaluate_policy(model, env)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Mean reward = <span class="hljs-subst">{mean_reward:<span class="hljs-number">.2</span>f}</span> +/- <span class="hljs-subst">{std_reward:<span class="hljs-number">.2</span>f}</span>&quot;</span>)`}}),_l=new w({}),Cl=new _({props:{code:"bm90ZWJvb2tfbG9naW4oKSUwQSFnaXQlMjBjb25maWclMjAtLWdsb2JhbCUyMGNyZWRlbnRpYWwuaGVscGVyJTIwc3RvcmU=",highlighted:`notebook_login()
!git config --<span class="hljs-keyword">global</span> credential.helper store`}}),Bl=new _({props:{code:"cGFja2FnZV90b19odWIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0Rtb2RlbCUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0RmJTIyYTJjLSU3QmVudl9pZCU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsX2FyY2hpdGVjdHVyZSUzRCUyMkEyQyUyMiUyQyUwQSUyMCUyMCUyMCUyMGVudl9pZCUzRGVudl9pZCUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfZW52JTNEZXZhbF9lbnYlMkMlMEElMjAlMjAlMjAlMjByZXBvX2lkJTNEZiUyMlRob21hc1NpbW9uaW5pJTJGYTJjLSU3QmVudl9pZCU3RCUyMiUyQyUyMCUyMCUyMyUyMENoYW5nZSUyMHRoZSUyMHVzZXJuYW1lJTBBJTIwJTIwJTIwJTIwY29tbWl0X21lc3NhZ2UlM0QlMjJJbml0aWFsJTIwY29tbWl0JTIyJTJDJTBBKQ==",highlighted:`package_to_hub(
    model=model,
    model_name=<span class="hljs-string">f&quot;a2c-<span class="hljs-subst">{env_id}</span>&quot;</span>,
    model_architecture=<span class="hljs-string">&quot;A2C&quot;</span>,
    env_id=env_id,
    eval_env=eval_env,
    repo_id=<span class="hljs-string">f&quot;ThomasSimonini/a2c-<span class="hljs-subst">{env_id}</span>&quot;</span>,  <span class="hljs-comment"># Change the username</span>
    commit_message=<span class="hljs-string">&quot;Initial commit&quot;</span>,
)`}}),Zl=new w({}),Al=new w({}),Wl=new _({props:{code:"aW1wb3J0JTIwZ3ltJTBBJTBBZW52X2lkJTIwJTNEJTIwJTIyUGFuZGFSZWFjaERlbnNlLXYyJTIyJTBBJTBBJTIzJTIwQ3JlYXRlJTIwdGhlJTIwZW52JTBBZW52JTIwJTNEJTIwZ3ltLm1ha2UoZW52X2lkKSUwQSUwQSUyMyUyMEdldCUyMHRoZSUyMHN0YXRlJTIwc3BhY2UlMjBhbmQlMjBhY3Rpb24lMjBzcGFjZSUwQXNfc2l6ZSUyMCUzRCUyMGVudi5vYnNlcnZhdGlvbl9zcGFjZS5zaGFwZSUwQWFfc2l6ZSUyMCUzRCUyMGVudi5hY3Rpb25fc3BhY2U=",highlighted:`<span class="hljs-keyword">import</span> gym

env_id = <span class="hljs-string">&quot;PandaReachDense-v2&quot;</span>

<span class="hljs-comment"># Create the env</span>
env = gym.make(env_id)

<span class="hljs-comment"># Get the state space and action space</span>
s_size = env.observation_space.shape
a_size = env.action_space`}}),Pl=new _({props:{code:"cHJpbnQoJTIyX19fX19PQlNFUlZBVElPTiUyMFNQQUNFX19fX18lMjAlNUNuJTIyKSUwQXByaW50KCUyMlRoZSUyMFN0YXRlJTIwU3BhY2UlMjBpcyUzQSUyMCUyMiUyQyUyMHNfc2l6ZSklMEFwcmludCglMjJTYW1wbGUlMjBvYnNlcnZhdGlvbiUyMiUyQyUyMGVudi5vYnNlcnZhdGlvbl9zcGFjZS5zYW1wbGUoKSklMjAlMjAlMjMlMjBHZXQlMjBhJTIwcmFuZG9tJTIwb2JzZXJ2YXRpb24=",highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;_____OBSERVATION SPACE_____ \\n&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;The State Space is: &quot;</span>, s_size)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Sample observation&quot;</span>, env.observation_space.sample())  <span class="hljs-comment"># Get a random observation</span>`}}),Rl=new _({props:{code:"cHJpbnQoJTIyJTVDbiUyMF9fX19fQUNUSU9OJTIwU1BBQ0VfX19fXyUyMCU1Q24lMjIpJTBBcHJpbnQoJTIyVGhlJTIwQWN0aW9uJTIwU3BhY2UlMjBpcyUzQSUyMCUyMiUyQyUyMGFfc2l6ZSklMEFwcmludCglMjJBY3Rpb24lMjBTcGFjZSUyMFNhbXBsZSUyMiUyQyUyMGVudi5hY3Rpb25fc3BhY2Uuc2FtcGxlKCkpJTIwJTIwJTIzJTIwVGFrZSUyMGElMjByYW5kb20lMjBhY3Rpb24=",highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;\\n _____ACTION SPACE_____ \\n&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;The Action Space is: &quot;</span>, a_size)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Action Space Sample&quot;</span>, env.action_space.sample())  <span class="hljs-comment"># Take a random action</span>`}}),Fl=new w({}),Vl=new _({props:{code:"JTIzJTIwMSUyMC0lMjAyJTBBZW52X2lkJTIwJTNEJTIwJTIyUGFuZGFSZWFjaERlbnNlLXYyJTIyJTBBZW52JTIwJTNEJTIwbWFrZV92ZWNfZW52KGVudl9pZCUyQyUyMG5fZW52cyUzRDQpJTBBJTBBJTIzJTIwMyUwQWVudiUyMCUzRCUyMFZlY05vcm1hbGl6ZShlbnYlMkMlMjBub3JtX29icyUzRFRydWUlMkMlMjBub3JtX3Jld2FyZCUzREZhbHNlJTJDJTIwY2xpcF9vYnMlM0QxMC4wKSUwQSUwQSUyMyUyMDQlMEFtb2RlbCUyMCUzRCUyMEEyQyhwb2xpY3klM0QlMjJNdWx0aUlucHV0UG9saWN5JTIyJTJDJTIwZW52JTNEZW52JTJDJTIwdmVyYm9zZSUzRDEpJTBBJTIzJTIwNSUwQW1vZGVsLmxlYXJuKDFfMDAwXzAwMCk=",highlighted:`<span class="hljs-comment"># 1 - 2</span>
env_id = <span class="hljs-string">&quot;PandaReachDense-v2&quot;</span>
env = make_vec_env(env_id, n_envs=<span class="hljs-number">4</span>)

<span class="hljs-comment"># 3</span>
env = VecNormalize(env, norm_obs=<span class="hljs-literal">True</span>, norm_reward=<span class="hljs-literal">False</span>, clip_obs=<span class="hljs-number">10.0</span>)

<span class="hljs-comment"># 4</span>
model = A2C(policy=<span class="hljs-string">&quot;MultiInputPolicy&quot;</span>, env=env, verbose=<span class="hljs-number">1</span>)
<span class="hljs-comment"># 5</span>
model.learn(<span class="hljs-number">1_000_000</span>)`}}),zl=new _({props:{code:"JTIzJTIwNiUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJhMmMtUGFuZGFSZWFjaERlbnNlLXYyJTIyJTBBbW9kZWwuc2F2ZShtb2RlbF9uYW1lKSUwQWVudi5zYXZlKCUyMnZlY19ub3JtYWxpemUucGtsJTIyKSUwQSUwQSUyMyUyMDclMEFmcm9tJTIwc3RhYmxlX2Jhc2VsaW5lczMuY29tbW9uLnZlY19lbnYlMjBpbXBvcnQlMjBEdW1teVZlY0VudiUyQyUyMFZlY05vcm1hbGl6ZSUwQSUwQSUyMyUyMExvYWQlMjB0aGUlMjBzYXZlZCUyMHN0YXRpc3RpY3MlMEFldmFsX2VudiUyMCUzRCUyMER1bW15VmVjRW52KCU1QmxhbWJkYSUzQSUyMGd5bS5tYWtlKCUyMlBhbmRhUmVhY2hEZW5zZS12MiUyMiklNUQpJTBBZXZhbF9lbnYlMjAlM0QlMjBWZWNOb3JtYWxpemUubG9hZCglMjJ2ZWNfbm9ybWFsaXplLnBrbCUyMiUyQyUyMGV2YWxfZW52KSUwQSUwQSUyMyUyMCUyMGRvJTIwbm90JTIwdXBkYXRlJTIwdGhlbSUyMGF0JTIwdGVzdCUyMHRpbWUlMEFldmFsX2Vudi50cmFpbmluZyUyMCUzRCUyMEZhbHNlJTBBJTIzJTIwcmV3YXJkJTIwbm9ybWFsaXphdGlvbiUyMGlzJTIwbm90JTIwbmVlZGVkJTIwYXQlMjB0ZXN0JTIwdGltZSUwQWV2YWxfZW52Lm5vcm1fcmV3YXJkJTIwJTNEJTIwRmFsc2UlMEElMEElMjMlMjBMb2FkJTIwdGhlJTIwYWdlbnQlMEFtb2RlbCUyMCUzRCUyMEEyQy5sb2FkKG1vZGVsX25hbWUpJTBBJTBBbWVhbl9yZXdhcmQlMkMlMjBzdGRfcmV3YXJkJTIwJTNEJTIwZXZhbHVhdGVfcG9saWN5KG1vZGVsJTJDJTIwZW52KSUwQSUwQXByaW50KGYlMjJNZWFuJTIwcmV3YXJkJTIwJTNEJTIwJTdCbWVhbl9yZXdhcmQlM0EuMmYlN0QlMjAlMkIlMkYtJTIwJTdCc3RkX3Jld2FyZCUzQS4yZiU3RCUyMiklMEElMEElMjMlMjA4JTBBcGFja2FnZV90b19odWIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0Rtb2RlbCUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0RmJTIyYTJjLSU3QmVudl9pZCU3RCUyMiUyQyUwQSUyMCUyMCUyMCUyMG1vZGVsX2FyY2hpdGVjdHVyZSUzRCUyMkEyQyUyMiUyQyUwQSUyMCUyMCUyMCUyMGVudl9pZCUzRGVudl9pZCUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfZW52JTNEZXZhbF9lbnYlMkMlMEElMjAlMjAlMjAlMjByZXBvX2lkJTNEZiUyMlRob21hc1NpbW9uaW5pJTJGYTJjLSU3QmVudl9pZCU3RCUyMiUyQyUyMCUyMCUyMyUyMFRPRE8lM0ElMjBDaGFuZ2UlMjB0aGUlMjB1c2VybmFtZSUwQSUyMCUyMCUyMCUyMGNvbW1pdF9tZXNzYWdlJTNEJTIySW5pdGlhbCUyMGNvbW1pdCUyMiUyQyUwQSk=",highlighted:`<span class="hljs-comment"># 6</span>
model_name = <span class="hljs-string">&quot;a2c-PandaReachDense-v2&quot;</span>
model.save(model_name)
env.save(<span class="hljs-string">&quot;vec_normalize.pkl&quot;</span>)

<span class="hljs-comment"># 7</span>
<span class="hljs-keyword">from</span> stable_baselines3.common.vec_env <span class="hljs-keyword">import</span> DummyVecEnv, VecNormalize

<span class="hljs-comment"># Load the saved statistics</span>
eval_env = DummyVecEnv([<span class="hljs-keyword">lambda</span>: gym.make(<span class="hljs-string">&quot;PandaReachDense-v2&quot;</span>)])
eval_env = VecNormalize.load(<span class="hljs-string">&quot;vec_normalize.pkl&quot;</span>, eval_env)

<span class="hljs-comment">#  do not update them at test time</span>
eval_env.training = <span class="hljs-literal">False</span>
<span class="hljs-comment"># reward normalization is not needed at test time</span>
eval_env.norm_reward = <span class="hljs-literal">False</span>

<span class="hljs-comment"># Load the agent</span>
model = A2C.load(model_name)

mean_reward, std_reward = evaluate_policy(model, env)

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Mean reward = <span class="hljs-subst">{mean_reward:<span class="hljs-number">.2</span>f}</span> +/- <span class="hljs-subst">{std_reward:<span class="hljs-number">.2</span>f}</span>&quot;</span>)

<span class="hljs-comment"># 8</span>
package_to_hub(
    model=model,
    model_name=<span class="hljs-string">f&quot;a2c-<span class="hljs-subst">{env_id}</span>&quot;</span>,
    model_architecture=<span class="hljs-string">&quot;A2C&quot;</span>,
    env_id=env_id,
    eval_env=eval_env,
    repo_id=<span class="hljs-string">f&quot;ThomasSimonini/a2c-<span class="hljs-subst">{env_id}</span>&quot;</span>,  <span class="hljs-comment"># <span class="hljs-doctag">TODO:</span> Change the username</span>
    commit_message=<span class="hljs-string">&quot;Initial commit&quot;</span>,
)`}}),Xl=new w({}),Ol=new w({}),{c(){A=a("meta"),fr=f(),N=a("h1"),ce=a("a"),Va=a("span"),u(ht.$$.fragment),Th=f(),za=a("span"),Jh=n("Advantage Actor Critic (A2C) using Robotics Simulations with PyBullet and Panda-Gym \u{1F916}"),cr=f(),u(pt.$$.fragment),ur=f(),ue=a("p"),$h=n("Now that you\u2019ve studied the theory behind Advantage Actor Critic (A2C), "),Xa=a("strong"),kh=n("you\u2019re ready to train your A2C agent"),jh=n(" using Stable-Baselines3 in robotic environments. And train two robots:"),mr=f(),me=a("ul"),Ha=a("li"),Ih=n("A spider \u{1F577}\uFE0F to learn to move."),Sh=f(),La=a("li"),Ch=n("A robotic arm \u{1F9BE} to move to the correct position."),dr=f(),Kl=a("p"),Gh=n("We\u2019re going to use two Robotics environments:"),vr=f(),de=a("ul"),xa=a("li"),ft=a("a"),Bh=n("PyBullet"),Zh=f(),Da=a("li"),ct=a("a"),Ah=n("panda-gym"),yr=f(),ut=a("img"),br=f(),ea=a("p"),Nh=n("To validate this hands-on for the certification process, you need to push your two trained models to the Hub and get the following results:"),wr=f(),ve=a("ul"),ta=a("li"),qa=a("code"),Wh=n("AntBulletEnv-v0"),Ph=n(" get a result of >= 650."),Rh=f(),la=a("li"),Oa=a("code"),Qh=n("PandaReachDense-v2"),Yh=n(" get a result of >= -3.5."),_r=f(),W=a("p"),Fh=n("To find your result, "),mt=a("a"),Vh=n("go to the leaderboard"),zh=n(" and find your model, "),Ka=a("strong"),Xh=n("the result = mean_reward - std of reward"),gr=f(),aa=a("p"),es=a("strong"),Hh=n("If you don\u2019t find your model, go to the bottom of the page and click on the refresh button."),Mr=f(),dt=a("p"),Lh=n("For more information about the certification process, check this section \u{1F449} "),vt=a("a"),xh=n("https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),Er=f(),yt=a("p"),ts=a("strong"),Dh=n("To start the hands-on click on Open In Colab button"),qh=n(" \u{1F447} :"),Ur=f(),sa=a("p"),bt=a("a"),oa=a("img"),Tr=f(),P=a("h1"),ye=a("a"),ls=a("span"),u(wt.$$.fragment),Oh=f(),as=a("span"),Kh=n("Unit 6: Advantage Actor Critic (A2C) using Robotics Simulations with PyBullet and Panda-Gym \u{1F916}"),Jr=f(),R=a("h3"),be=a("a"),ss=a("span"),u(_t.$$.fragment),ep=f(),os=a("span"),tp=n("\u{1F3AE} Environments:"),$r=f(),we=a("ul"),rs=a("li"),gt=a("a"),lp=n("PyBullet"),ap=f(),ns=a("li"),Mt=a("a"),sp=n("Panda-Gym"),kr=f(),Q=a("h3"),_e=a("a"),is=a("span"),u(Et.$$.fragment),op=f(),hs=a("span"),rp=n("\u{1F4DA} RL-Library:"),jr=f(),ra=a("ul"),ps=a("li"),Ut=a("a"),np=n("Stable-Baselines3"),Ir=f(),k=a("p"),ip=n("We\u2019re constantly trying to improve our tutorials, so "),fs=a("strong"),hp=n("if you find some issues in this notebook"),pp=n(", please "),Tt=a("a"),fp=n("open an issue on the GitHub Repo"),cp=n("."),Sr=f(),Y=a("h2"),ge=a("a"),cs=a("span"),u(Jt.$$.fragment),up=f(),us=a("span"),mp=n("Objectives of this notebook \u{1F3C6}"),Cr=f(),na=a("p"),dp=n("At the end of the notebook, you will:"),Gr=f(),T=a("ul"),F=a("li"),vp=n("Be able to use the environment librairies "),ms=a("strong"),yp=n("PyBullet"),bp=n(" and "),ds=a("strong"),wp=n("Panda-Gym"),_p=n("."),gp=f(),$t=a("li"),Mp=n("Be able to "),vs=a("strong"),Ep=n("train robots using A2C"),Up=n("."),Tp=f(),kt=a("li"),Jp=n("Understand why "),ys=a("strong"),$p=n("we need to normalize the input"),kp=n("."),jp=f(),jt=a("li"),Ip=n("Be able to "),bs=a("strong"),Sp=n("push your trained agent and the code to the Hub"),Cp=n(" with a nice video replay and an evaluation score \u{1F525}."),Br=f(),V=a("h2"),Me=a("a"),ws=a("span"),u(It.$$.fragment),Gp=f(),_s=a("span"),Bp=n("Prerequisites \u{1F3D7}\uFE0F"),Zr=n(`

Before diving into the notebook, you need to:
`),Ee=a("p"),Zp=n("\u{1F532} \u{1F4DA} Study "),St=a("a"),Ap=n("Actor-Critic methods by reading Unit 6"),Np=n(" \u{1F917}"),Ar=f(),z=a("h1"),Ue=a("a"),gs=a("span"),u(Ct.$$.fragment),Wp=f(),Ms=a("span"),Pp=n("Let's train our first robots \u{1F916}"),Nr=f(),X=a("h2"),Te=a("a"),Es=a("span"),u(Gt.$$.fragment),Rp=f(),Us=a("span"),Qp=n("Set the GPU \u{1F4AA}"),Wr=f(),ia=a("ul"),Je=a("li"),Yp=n("To "),Ts=a("strong"),Fp=n("accelerate the agent\u2019s training, we\u2019ll use a GPU"),Vp=n(". To do that, go to "),Js=a("code"),zp=n("Runtime > Change Runtime type"),Pr=f(),Bt=a("img"),Rr=f(),ha=a("ul"),$s=a("li"),ks=a("code"),Xp=n("Hardware Accelerator > GPU"),Qr=f(),Zt=a("img"),Yr=f(),H=a("h2"),$e=a("a"),js=a("span"),u(At.$$.fragment),Hp=f(),Is=a("span"),Lp=n("Create a virtual display \u{1F53D}"),Fr=f(),ke=a("p"),xp=n("During the notebook, we\u2019ll need to generate a replay video. To do so, with colab, "),Ss=a("strong"),Dp=n("we need to have a virtual screen to be able to render the environment"),qp=n(" (and thus record the frames)."),Vr=f(),pa=a("p"),Op=n("The following cell will install the librairies and create and run a virtual screen \u{1F5A5}"),zr=f(),u(Nt.$$.fragment),Xr=f(),u(Wt.$$.fragment),Hr=f(),L=a("h3"),je=a("a"),Cs=a("span"),u(Pt.$$.fragment),Kp=f(),Gs=a("span"),ef=n("Install dependencies \u{1F53D}"),Lr=n(`

The first step is to install the dependencies, we\u2019ll install multiple ones:
`),E=a("ul"),fa=a("li"),Bs=a("code"),tf=n("pybullet"),lf=n(": Contains the walking robots environments."),af=f(),ca=a("li"),Zs=a("code"),sf=n("panda-gym"),of=n(": Contains the robotics arm environments."),rf=f(),ua=a("li"),As=a("code"),nf=n("stable-baselines3[extra]"),hf=n(": The SB3 deep reinforcement learning library."),pf=f(),ma=a("li"),Ns=a("code"),ff=n("huggingface_sb3"),cf=n(": Additional code for Stable-baselines3 to load and upload models from the Hugging Face \u{1F917} Hub."),uf=f(),da=a("li"),Ws=a("code"),mf=n("huggingface_hub"),df=n(": Library allowing anyone to work with the Hub repositories."),xr=f(),u(Rt.$$.fragment),Dr=f(),x=a("h2"),Ie=a("a"),Ps=a("span"),u(Qt.$$.fragment),vf=f(),Rs=a("span"),yf=n("Import the packages \u{1F4E6}"),qr=f(),u(Yt.$$.fragment),Or=f(),D=a("h2"),Se=a("a"),Qs=a("span"),u(Ft.$$.fragment),bf=f(),Ys=a("span"),wf=n("Environment 1: AntBulletEnv-v0 \u{1F578}"),Kr=f(),q=a("h3"),Ce=a("a"),Fs=a("span"),u(Vt.$$.fragment),_f=f(),Vs=a("span"),gf=n("Create the AntBulletEnv-v0"),en=f(),O=a("h4"),Ge=a("a"),zs=a("span"),u(zt.$$.fragment),Mf=f(),Xs=a("span"),Ef=n("The environment \u{1F3AE}"),tn=f(),Xt=a("p"),Uf=n(`In this environment, the agent needs to use its different joints correctly in order to walk.
You can find a detailled explanation of this environment here: `),Ht=a("a"),Tf=n("https://hackmd.io/@jeffreymo/SJJrSJh5_#PyBullet"),ln=f(),u(Lt.$$.fragment),an=f(),u(xt.$$.fragment),sn=f(),Be=a("p"),Jf=n("The observation Space (from "),Dt=a("a"),$f=n("Jeffrey Y Mo"),kf=n(`):
The difference is that our observation space is 28 not 29.`),on=f(),qt=a("img"),rn=f(),u(Ot.$$.fragment),nn=f(),Ze=a("p"),jf=n("The action Space (from "),Kt=a("a"),If=n("Jeffrey Y Mo"),Sf=n("):"),hn=f(),el=a("img"),pn=f(),K=a("h3"),Ae=a("a"),Hs=a("span"),u(tl.$$.fragment),Cf=f(),Ls=a("span"),Gf=n("Normalize observation and rewards"),fn=f(),Ne=a("p"),Bf=n("A good practice in reinforcement learning is to "),ll=a("a"),Zf=n("normalize input features"),Af=n("."),cn=f(),va=a("p"),Nf=n("For that purpose, there is a wrapper that will compute a running average and standard deviation of input features."),un=f(),al=a("p"),Wf=n("We also normalize rewards with this same wrapper by adding "),xs=a("code"),Pf=n("norm_reward = True"),mn=f(),ya=a("p"),sl=a("a"),Rf=n("You should check the documentation to fill this cell"),dn=f(),u(ol.$$.fragment),vn=f(),ee=a("h4"),We=a("a"),Ds=a("span"),u(rl.$$.fragment),Qf=f(),qs=a("span"),Yf=n("Solution"),yn=f(),u(nl.$$.fragment),bn=f(),te=a("h3"),Pe=a("a"),Os=a("span"),u(il.$$.fragment),Ff=f(),Ks=a("span"),Vf=n("Create the A2C Model \u{1F916}"),wn=f(),ba=a("p"),zf=n("In this case, because we have a vector of 28 values as input, we\u2019ll use an MLP (multi-layer perceptron) as policy."),_n=f(),hl=a("p"),Xf=n("For more information about A2C implementation with StableBaselines3 check: "),pl=a("a"),Hf=n("https://stable-baselines3.readthedocs.io/en/master/modules/a2c.html#notes"),gn=f(),Re=a("p"),Lf=n("To find the best parameters I checked the "),fl=a("a"),xf=n("official trained agents by Stable-Baselines3 team"),Df=n("."),Mn=f(),u(cl.$$.fragment),En=f(),le=a("h4"),Qe=a("a"),eo=a("span"),u(ul.$$.fragment),qf=f(),to=a("span"),Of=n("Solution"),Un=f(),u(ml.$$.fragment),Tn=f(),ae=a("h3"),Ye=a("a"),lo=a("span"),u(dl.$$.fragment),Kf=f(),ao=a("span"),ec=n("Train the A2C agent \u{1F3C3}"),Jn=f(),wa=a("ul"),so=a("li"),tc=n("Let\u2019s train our agent for 2,000,000 timesteps. Don\u2019t forget to use GPU on Colab. It will take approximately ~25-40min"),$n=f(),u(vl.$$.fragment),kn=f(),u(yl.$$.fragment),jn=f(),se=a("h3"),Fe=a("a"),oo=a("span"),u(bl.$$.fragment),lc=f(),ro=a("span"),ac=n("Evaluate the agent \u{1F4C8}"),In=n(`

- Now that our  agent is trained, we need to **check its performance**.
- Stable-Baselines3 provides a method to do that: \`evaluate_policy\`
- In my case, I got a mean reward of \`2371.90 +/- 16.50\`

	`),u(wl.$$.fragment),Sn=f(),oe=a("h3"),Ve=a("a"),no=a("span"),u(_l.$$.fragment),sc=f(),io=a("span"),oc=n("Publish your trained model on the Hub \u{1F525}"),Cn=n(`

Now that we saw we got good results after the training, we can publish our trained model on the Hub with one line of code.
`),gl=a("p"),rc=n("\u{1F4DA} The libraries documentation \u{1F449} "),Ml=a("a"),nc=n("https://github.com/huggingface/huggingface_sb3/tree/main#hugging-face\u2014x-stable-baselines3-v20"),Gn=f(),_a=a("p"),ic=n("Here\u2019s an example of a Model Card (with a PyBullet environment):"),Bn=f(),El=a("img"),Zn=f(),j=a("p"),hc=n("By using "),ho=a("code"),pc=n("package_to_hub"),fc=n(", as we already mentionned in the former units, "),po=a("strong"),cc=n("you evaluate, record a replay, generate a model card of your agent and push it to the hub"),uc=n("."),An=f(),ga=a("p"),mc=n("This way:"),Nn=f(),J=a("ul"),Ul=a("li"),dc=n("You can "),fo=a("strong"),vc=n("showcase our work"),yc=n(" \u{1F525}"),bc=f(),Tl=a("li"),wc=n("You can "),co=a("strong"),_c=n("visualize your agent playing"),gc=n(" \u{1F440}"),Mc=f(),Jl=a("li"),Ec=n("You can "),uo=a("strong"),Uc=n("share an agent with the community that others can use"),Tc=n(" \u{1F4BE}"),Jc=f(),ze=a("li"),$c=n("You can "),mo=a("strong"),kc=n("access a leaderboard \u{1F3C6} to see how well your agent is performing compared to your classmates"),jc=n(" \u{1F449} "),$l=a("a"),Ic=n("https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),Wn=f(),Ma=a("p"),Sc=n("To be able to share your model with the community there are three more steps to follow:"),Pn=f(),kl=a("p"),Cc=n("1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),jl=a("a"),Gc=n("https://huggingface.co/join"),Rn=f(),Ea=a("p"),Bc=n("2\uFE0F\u20E3 Sign in and then you need to get your authentication token from the Hugging Face website."),Qn=f(),Ua=a("ul"),Xe=a("li"),Zc=n("Create a new token ("),Il=a("a"),Ac=n("https://huggingface.co/settings/tokens"),Nc=n(") "),vo=a("strong"),Wc=n("with write role"),Yn=f(),Sl=a("img"),Fn=f(),He=a("ul"),yo=a("li"),Pc=n("Copy the token"),Rc=f(),bo=a("li"),Qc=n("Run the cell below and paste the token"),Vn=f(),u(Cl.$$.fragment),zn=f(),Gl=a("p"),Yc=n("If you don\u2019t want to use Google Colab or a Jupyter Notebook, you need to use this command instead: "),wo=a("code"),Fc=n("huggingface-cli login"),Xn=f(),Le=a("p"),Vc=n("3\uFE0F\u20E3 We\u2019re now ready to push our trained agent to the \u{1F917} Hub \u{1F525} using "),_o=a("code"),zc=n("package_to_hub()"),Xc=n(" function"),Hn=f(),u(Bl.$$.fragment),Ln=f(),re=a("h2"),xe=a("a"),go=a("span"),u(Zl.$$.fragment),Hc=f(),Mo=a("span"),Lc=n("Take a coffee break \u2615"),xn=n(`

- You already trained your first robot that learned to move congratutlations \u{1F973}!
- It's **time to take a break**. Don't hesitate to **save this notebook** \`File > Save a copy to Drive\` to work on this second part later.
`),ne=a("h2"),De=a("a"),Eo=a("span"),u(Al.$$.fragment),xc=f(),Uo=a("span"),Dc=n("Environment 2: PandaReachDense-v2 \u{1F9BE}"),Dn=f(),Ta=a("p"),qc=n("The agent we\u2019re going to train is a robotic arm that needs to do controls (moving the arm and using the end-effector)."),qn=f(),qe=a("p"),Oc=n("In robotics, the "),To=a("em"),Kc=n("end-effector"),eu=n(" is the device at the end of a robotic arm designed to interact with the environment."),On=f(),Oe=a("p"),tu=n("In "),Jo=a("code"),lu=n("PandaReach"),au=n(", the robot must place its end-effector at a target position (green ball)."),Kn=f(),U=a("p"),su=n("We\u2019re going to use the dense version of this environment. This means we\u2019ll get a "),$o=a("em"),ou=n("dense reward function"),ru=n(" that "),ko=a("strong"),nu=n("will provide a reward at each timestep"),iu=n(" (the closer the agent is to completing the task, the higher the reward). This is in contrast to a "),jo=a("em"),hu=n("sparse reward function"),pu=n(" where the environment "),Io=a("strong"),fu=n("return a reward if and only if the task is completed"),cu=n("."),ei=f(),I=a("p"),uu=n("Also, we\u2019re going to use the "),So=a("em"),mu=n("End-effector displacement control"),du=n(", which means the "),Co=a("strong"),vu=n("action corresponds to the displacement of the end-effector"),yu=n(". We don\u2019t control the individual motion of each joint (joint control)."),ti=f(),Nl=a("img"),li=f(),Ke=a("p"),bu=n("This way "),Go=a("strong"),wu=n("the training will be easier"),_u=n("."),ai=f(),et=a("p"),gu=n("In "),Bo=a("code"),Mu=n("PandaReachDense-v2"),Eu=n(", the robotic arm must place its end-effector at a target position (green ball)."),si=f(),u(Wl.$$.fragment),oi=f(),u(Pl.$$.fragment),ri=f(),tt=a("p"),Uu=n("The observation space "),Zo=a("strong"),Tu=n("is a dictionary with 3 different elements"),Ju=n(":"),ni=f(),S=a("ul"),Ja=a("li"),Ao=a("code"),$u=n("achieved_goal"),ku=n(": (x,y,z) position of the goal."),ju=f(),$a=a("li"),No=a("code"),Iu=n("desired_goal"),Su=n(": (x,y,z) distance between the goal position and the current object position."),Cu=f(),ka=a("li"),Wo=a("code"),Gu=n("observation"),Bu=n(": position (x,y,z) and velocity of the end-effector (vx, vy, vz)."),ii=f(),lt=a("p"),Zu=n("Given it\u2019s a dictionary as observation, "),Po=a("strong"),Au=n("we will need to use a MultiInputPolicy policy instead of MlpPolicy"),Nu=n("."),hi=f(),u(Rl.$$.fragment),pi=f(),ja=a("p"),Wu=n("The action space is a vector with 3 values:"),fi=f(),Ia=a("ul"),Ro=a("li"),Pu=n("Control x, y, z movement"),ci=f(),Sa=a("p"),Ru=n("Now it\u2019s your turn:"),ui=f(),g=a("ol"),Qo=a("li"),Qu=n("Define the environment called \u201CPandaReachDense-v2\u201D."),Yu=f(),Yo=a("li"),Fu=n("Make a vectorized environment."),Vu=f(),Ca=a("li"),zu=n("Add a wrapper to normalize the observations and rewards. "),Ql=a("a"),Xu=n("Check the documentation"),Hu=f(),Fo=a("li"),Lu=n("Create the A2C Model (don\u2019t forget verbose=1 to print the training logs)."),xu=f(),Vo=a("li"),Du=n("Train it for 1M Timesteps."),qu=f(),zo=a("li"),Ou=n("Save the model and  VecNormalize statistics when saving the agent."),Ku=f(),Xo=a("li"),em=n("Evaluate your agent."),tm=f(),Yl=a("li"),lm=n("Publish your trained model on the Hub \u{1F525} with "),Ho=a("code"),am=n("package_to_hub"),sm=n("."),mi=f(),ie=a("h3"),at=a("a"),Lo=a("span"),u(Fl.$$.fragment),om=f(),xo=a("span"),rm=n("Solution (fill the todo)"),di=f(),u(Vl.$$.fragment),vi=f(),u(zl.$$.fragment),yi=f(),he=a("h2"),st=a("a"),Do=a("span"),u(Xl.$$.fragment),nm=f(),qo=a("span"),im=n("Some additional challenges \u{1F3C6}"),bi=f(),$=a("p"),hm=n("The best way to learn "),Oo=a("strong"),pm=n("is to try things on your own"),fm=n("! Why not try  "),Ko=a("code"),cm=n("HalfCheetahBulletEnv-v0"),um=n(" for PyBullet and "),er=a("code"),mm=n("PandaPickAndPlace-v1"),dm=n(" for Panda-Gym?"),wi=f(),C=a("p"),vm=n("If you want to try more advanced tasks for panda-gym, you need to check what was done using "),tr=a("strong"),ym=n("TQC or SAC"),bm=n(" (a more sample-efficient algorithm suited for robotics tasks). In real robotics, you\u2019ll use a more sample-efficient algorithm for a simple reason: contrary to a simulation "),lr=a("strong"),wm=n("if you move your robotic arm too much, you have a risk of breaking it"),_m=n("."),_i=f(),Hl=a("p"),gm=n("PandaPickAndPlace-v1: "),Ll=a("a"),Mm=n("https://huggingface.co/sb3/tqc-PandaPickAndPlace-v1"),gi=f(),xl=a("p"),Em=n("And don\u2019t hesitate to check panda-gym documentation here: "),Dl=a("a"),Um=n("https://panda-gym.readthedocs.io/en/latest/usage/train_with_sb3.html"),Mi=f(),Ga=a("p"),Tm=n("Here are some ideas to go further:"),Ei=f(),G=a("ul"),ar=a("li"),Jm=n("Train more steps"),$m=f(),Ba=a("li"),km=n("Try different hyperparameters by looking at what your classmates have done \u{1F449} "),ql=a("a"),jm=n("https://huggingface.co/models?other=https://huggingface.co/models?other=AntBulletEnv-v0"),Im=f(),Za=a("li"),sr=a("strong"),Sm=n("Push your new trained model"),Cm=n(" on the Hub \u{1F525}"),Ui=f(),Aa=a("p"),Gm=n("See you on Unit 7! \u{1F525}"),Ti=f(),pe=a("h2"),ot=a("a"),or=a("span"),u(Ol.$$.fragment),Bm=f(),rr=a("span"),Zm=n("Keep learning, stay awesome \u{1F917}"),this.h()},l(e){const r=tw('[data-svelte="svelte-1phssyn"]',document.head);A=s(r,"META",{name:!0,content:!0}),r.forEach(t),fr=c(e),N=s(e,"H1",{class:!0});var $i=o(N);ce=s($i,"A",{id:!0,class:!0,href:!0});var dd=o(ce);Va=s(dd,"SPAN",{});var vd=o(Va);m(ht.$$.fragment,vd),vd.forEach(t),dd.forEach(t),Th=c($i),za=s($i,"SPAN",{});var yd=o(za);Jh=i(yd,"Advantage Actor Critic (A2C) using Robotics Simulations with PyBullet and Panda-Gym \u{1F916}"),yd.forEach(t),$i.forEach(t),cr=c(e),m(pt.$$.fragment,e),ur=c(e),ue=s(e,"P",{});var ki=o(ue);$h=i(ki,"Now that you\u2019ve studied the theory behind Advantage Actor Critic (A2C), "),Xa=s(ki,"STRONG",{});var bd=o(Xa);kh=i(bd,"you\u2019re ready to train your A2C agent"),bd.forEach(t),jh=i(ki," using Stable-Baselines3 in robotic environments. And train two robots:"),ki.forEach(t),mr=c(e),me=s(e,"UL",{});var ji=o(me);Ha=s(ji,"LI",{});var wd=o(Ha);Ih=i(wd,"A spider \u{1F577}\uFE0F to learn to move."),wd.forEach(t),Sh=c(ji),La=s(ji,"LI",{});var _d=o(La);Ch=i(_d,"A robotic arm \u{1F9BE} to move to the correct position."),_d.forEach(t),ji.forEach(t),dr=c(e),Kl=s(e,"P",{});var gd=o(Kl);Gh=i(gd,"We\u2019re going to use two Robotics environments:"),gd.forEach(t),vr=c(e),de=s(e,"UL",{});var Ii=o(de);xa=s(Ii,"LI",{});var Md=o(xa);ft=s(Md,"A",{href:!0,rel:!0});var Ed=o(ft);Bh=i(Ed,"PyBullet"),Ed.forEach(t),Md.forEach(t),Zh=c(Ii),Da=s(Ii,"LI",{});var Ud=o(Da);ct=s(Ud,"A",{href:!0,rel:!0});var Td=o(ct);Ah=i(Td,"panda-gym"),Td.forEach(t),Ud.forEach(t),Ii.forEach(t),yr=c(e),ut=s(e,"IMG",{src:!0,alt:!0}),br=c(e),ea=s(e,"P",{});var Jd=o(ea);Nh=i(Jd,"To validate this hands-on for the certification process, you need to push your two trained models to the Hub and get the following results:"),Jd.forEach(t),wr=c(e),ve=s(e,"UL",{});var Si=o(ve);ta=s(Si,"LI",{});var Am=o(ta);qa=s(Am,"CODE",{});var $d=o(qa);Wh=i($d,"AntBulletEnv-v0"),$d.forEach(t),Ph=i(Am," get a result of >= 650."),Am.forEach(t),Rh=c(Si),la=s(Si,"LI",{});var Nm=o(la);Oa=s(Nm,"CODE",{});var kd=o(Oa);Qh=i(kd,"PandaReachDense-v2"),kd.forEach(t),Yh=i(Nm," get a result of >= -3.5."),Nm.forEach(t),Si.forEach(t),_r=c(e),W=s(e,"P",{});var nr=o(W);Fh=i(nr,"To find your result, "),mt=s(nr,"A",{href:!0,rel:!0});var jd=o(mt);Vh=i(jd,"go to the leaderboard"),jd.forEach(t),zh=i(nr," and find your model, "),Ka=s(nr,"STRONG",{});var Id=o(Ka);Xh=i(Id,"the result = mean_reward - std of reward"),Id.forEach(t),nr.forEach(t),gr=c(e),aa=s(e,"P",{});var Sd=o(aa);es=s(Sd,"STRONG",{});var Cd=o(es);Hh=i(Cd,"If you don\u2019t find your model, go to the bottom of the page and click on the refresh button."),Cd.forEach(t),Sd.forEach(t),Mr=c(e),dt=s(e,"P",{});var Wm=o(dt);Lh=i(Wm,"For more information about the certification process, check this section \u{1F449} "),vt=s(Wm,"A",{href:!0,rel:!0});var Gd=o(vt);xh=i(Gd,"https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),Gd.forEach(t),Wm.forEach(t),Er=c(e),yt=s(e,"P",{});var Pm=o(yt);ts=s(Pm,"STRONG",{});var Bd=o(ts);Dh=i(Bd,"To start the hands-on click on Open In Colab button"),Bd.forEach(t),qh=i(Pm," \u{1F447} :"),Pm.forEach(t),Ur=c(e),sa=s(e,"P",{});var Zd=o(sa);bt=s(Zd,"A",{href:!0,rel:!0});var Ad=o(bt);oa=s(Ad,"IMG",{src:!0,alt:!0}),Ad.forEach(t),Zd.forEach(t),Tr=c(e),P=s(e,"H1",{class:!0});var Ci=o(P);ye=s(Ci,"A",{id:!0,class:!0,href:!0});var Nd=o(ye);ls=s(Nd,"SPAN",{});var Wd=o(ls);m(wt.$$.fragment,Wd),Wd.forEach(t),Nd.forEach(t),Oh=c(Ci),as=s(Ci,"SPAN",{});var Pd=o(as);Kh=i(Pd,"Unit 6: Advantage Actor Critic (A2C) using Robotics Simulations with PyBullet and Panda-Gym \u{1F916}"),Pd.forEach(t),Ci.forEach(t),Jr=c(e),R=s(e,"H3",{class:!0});var Gi=o(R);be=s(Gi,"A",{id:!0,class:!0,href:!0});var Rd=o(be);ss=s(Rd,"SPAN",{});var Qd=o(ss);m(_t.$$.fragment,Qd),Qd.forEach(t),Rd.forEach(t),ep=c(Gi),os=s(Gi,"SPAN",{});var Yd=o(os);tp=i(Yd,"\u{1F3AE} Environments:"),Yd.forEach(t),Gi.forEach(t),$r=c(e),we=s(e,"UL",{});var Bi=o(we);rs=s(Bi,"LI",{});var Fd=o(rs);gt=s(Fd,"A",{href:!0,rel:!0});var Vd=o(gt);lp=i(Vd,"PyBullet"),Vd.forEach(t),Fd.forEach(t),ap=c(Bi),ns=s(Bi,"LI",{});var zd=o(ns);Mt=s(zd,"A",{href:!0,rel:!0});var Xd=o(Mt);sp=i(Xd,"Panda-Gym"),Xd.forEach(t),zd.forEach(t),Bi.forEach(t),kr=c(e),Q=s(e,"H3",{class:!0});var Zi=o(Q);_e=s(Zi,"A",{id:!0,class:!0,href:!0});var Hd=o(_e);is=s(Hd,"SPAN",{});var Ld=o(is);m(Et.$$.fragment,Ld),Ld.forEach(t),Hd.forEach(t),op=c(Zi),hs=s(Zi,"SPAN",{});var xd=o(hs);rp=i(xd,"\u{1F4DA} RL-Library:"),xd.forEach(t),Zi.forEach(t),jr=c(e),ra=s(e,"UL",{});var Dd=o(ra);ps=s(Dd,"LI",{});var qd=o(ps);Ut=s(qd,"A",{href:!0,rel:!0});var Od=o(Ut);np=i(Od,"Stable-Baselines3"),Od.forEach(t),qd.forEach(t),Dd.forEach(t),Ir=c(e),k=s(e,"P",{});var Na=o(k);ip=i(Na,"We\u2019re constantly trying to improve our tutorials, so "),fs=s(Na,"STRONG",{});var Kd=o(fs);hp=i(Kd,"if you find some issues in this notebook"),Kd.forEach(t),pp=i(Na,", please "),Tt=s(Na,"A",{href:!0,rel:!0});var ev=o(Tt);fp=i(ev,"open an issue on the GitHub Repo"),ev.forEach(t),cp=i(Na,"."),Na.forEach(t),Sr=c(e),Y=s(e,"H2",{class:!0});var Ai=o(Y);ge=s(Ai,"A",{id:!0,class:!0,href:!0});var tv=o(ge);cs=s(tv,"SPAN",{});var lv=o(cs);m(Jt.$$.fragment,lv),lv.forEach(t),tv.forEach(t),up=c(Ai),us=s(Ai,"SPAN",{});var av=o(us);mp=i(av,"Objectives of this notebook \u{1F3C6}"),av.forEach(t),Ai.forEach(t),Cr=c(e),na=s(e,"P",{});var sv=o(na);dp=i(sv,"At the end of the notebook, you will:"),sv.forEach(t),Gr=c(e),T=s(e,"UL",{});var rt=o(T);F=s(rt,"LI",{});var Wa=o(F);vp=i(Wa,"Be able to use the environment librairies "),ms=s(Wa,"STRONG",{});var ov=o(ms);yp=i(ov,"PyBullet"),ov.forEach(t),bp=i(Wa," and "),ds=s(Wa,"STRONG",{});var rv=o(ds);wp=i(rv,"Panda-Gym"),rv.forEach(t),_p=i(Wa,"."),Wa.forEach(t),gp=c(rt),$t=s(rt,"LI",{});var Ni=o($t);Mp=i(Ni,"Be able to "),vs=s(Ni,"STRONG",{});var nv=o(vs);Ep=i(nv,"train robots using A2C"),nv.forEach(t),Up=i(Ni,"."),Ni.forEach(t),Tp=c(rt),kt=s(rt,"LI",{});var Wi=o(kt);Jp=i(Wi,"Understand why "),ys=s(Wi,"STRONG",{});var iv=o(ys);$p=i(iv,"we need to normalize the input"),iv.forEach(t),kp=i(Wi,"."),Wi.forEach(t),jp=c(rt),jt=s(rt,"LI",{});var Pi=o(jt);Ip=i(Pi,"Be able to "),bs=s(Pi,"STRONG",{});var hv=o(bs);Sp=i(hv,"push your trained agent and the code to the Hub"),hv.forEach(t),Cp=i(Pi," with a nice video replay and an evaluation score \u{1F525}."),Pi.forEach(t),rt.forEach(t),Br=c(e),V=s(e,"H2",{class:!0});var Ri=o(V);Me=s(Ri,"A",{id:!0,class:!0,href:!0});var pv=o(Me);ws=s(pv,"SPAN",{});var fv=o(ws);m(It.$$.fragment,fv),fv.forEach(t),pv.forEach(t),Gp=c(Ri),_s=s(Ri,"SPAN",{});var cv=o(_s);Bp=i(cv,"Prerequisites \u{1F3D7}\uFE0F"),cv.forEach(t),Ri.forEach(t),Zr=i(e,`

Before diving into the notebook, you need to:
`),Ee=s(e,"P",{});var Qi=o(Ee);Zp=i(Qi,"\u{1F532} \u{1F4DA} Study "),St=s(Qi,"A",{href:!0,rel:!0});var uv=o(St);Ap=i(uv,"Actor-Critic methods by reading Unit 6"),uv.forEach(t),Np=i(Qi," \u{1F917}"),Qi.forEach(t),Ar=c(e),z=s(e,"H1",{class:!0});var Yi=o(z);Ue=s(Yi,"A",{id:!0,class:!0,href:!0});var mv=o(Ue);gs=s(mv,"SPAN",{});var dv=o(gs);m(Ct.$$.fragment,dv),dv.forEach(t),mv.forEach(t),Wp=c(Yi),Ms=s(Yi,"SPAN",{});var vv=o(Ms);Pp=i(vv,"Let's train our first robots \u{1F916}"),vv.forEach(t),Yi.forEach(t),Nr=c(e),X=s(e,"H2",{class:!0});var Fi=o(X);Te=s(Fi,"A",{id:!0,class:!0,href:!0});var yv=o(Te);Es=s(yv,"SPAN",{});var bv=o(Es);m(Gt.$$.fragment,bv),bv.forEach(t),yv.forEach(t),Rp=c(Fi),Us=s(Fi,"SPAN",{});var wv=o(Us);Qp=i(wv,"Set the GPU \u{1F4AA}"),wv.forEach(t),Fi.forEach(t),Wr=c(e),ia=s(e,"UL",{});var _v=o(ia);Je=s(_v,"LI",{});var ir=o(Je);Yp=i(ir,"To "),Ts=s(ir,"STRONG",{});var gv=o(Ts);Fp=i(gv,"accelerate the agent\u2019s training, we\u2019ll use a GPU"),gv.forEach(t),Vp=i(ir,". To do that, go to "),Js=s(ir,"CODE",{});var Mv=o(Js);zp=i(Mv,"Runtime > Change Runtime type"),Mv.forEach(t),ir.forEach(t),_v.forEach(t),Pr=c(e),Bt=s(e,"IMG",{src:!0,alt:!0}),Rr=c(e),ha=s(e,"UL",{});var Ev=o(ha);$s=s(Ev,"LI",{});var Uv=o($s);ks=s(Uv,"CODE",{});var Tv=o(ks);Xp=i(Tv,"Hardware Accelerator > GPU"),Tv.forEach(t),Uv.forEach(t),Ev.forEach(t),Qr=c(e),Zt=s(e,"IMG",{src:!0,alt:!0}),Yr=c(e),H=s(e,"H2",{class:!0});var Vi=o(H);$e=s(Vi,"A",{id:!0,class:!0,href:!0});var Jv=o($e);js=s(Jv,"SPAN",{});var $v=o(js);m(At.$$.fragment,$v),$v.forEach(t),Jv.forEach(t),Hp=c(Vi),Is=s(Vi,"SPAN",{});var kv=o(Is);Lp=i(kv,"Create a virtual display \u{1F53D}"),kv.forEach(t),Vi.forEach(t),Fr=c(e),ke=s(e,"P",{});var zi=o(ke);xp=i(zi,"During the notebook, we\u2019ll need to generate a replay video. To do so, with colab, "),Ss=s(zi,"STRONG",{});var jv=o(Ss);Dp=i(jv,"we need to have a virtual screen to be able to render the environment"),jv.forEach(t),qp=i(zi," (and thus record the frames)."),zi.forEach(t),Vr=c(e),pa=s(e,"P",{});var Iv=o(pa);Op=i(Iv,"The following cell will install the librairies and create and run a virtual screen \u{1F5A5}"),Iv.forEach(t),zr=c(e),m(Nt.$$.fragment,e),Xr=c(e),m(Wt.$$.fragment,e),Hr=c(e),L=s(e,"H3",{class:!0});var Xi=o(L);je=s(Xi,"A",{id:!0,class:!0,href:!0});var Sv=o(je);Cs=s(Sv,"SPAN",{});var Cv=o(Cs);m(Pt.$$.fragment,Cv),Cv.forEach(t),Sv.forEach(t),Kp=c(Xi),Gs=s(Xi,"SPAN",{});var Gv=o(Gs);ef=i(Gv,"Install dependencies \u{1F53D}"),Gv.forEach(t),Xi.forEach(t),Lr=i(e,`

The first step is to install the dependencies, we\u2019ll install multiple ones:
`),E=s(e,"UL",{});var B=o(E);fa=s(B,"LI",{});var Rm=o(fa);Bs=s(Rm,"CODE",{});var Bv=o(Bs);tf=i(Bv,"pybullet"),Bv.forEach(t),lf=i(Rm,": Contains the walking robots environments."),Rm.forEach(t),af=c(B),ca=s(B,"LI",{});var Qm=o(ca);Zs=s(Qm,"CODE",{});var Zv=o(Zs);sf=i(Zv,"panda-gym"),Zv.forEach(t),of=i(Qm,": Contains the robotics arm environments."),Qm.forEach(t),rf=c(B),ua=s(B,"LI",{});var Ym=o(ua);As=s(Ym,"CODE",{});var Av=o(As);nf=i(Av,"stable-baselines3[extra]"),Av.forEach(t),hf=i(Ym,": The SB3 deep reinforcement learning library."),Ym.forEach(t),pf=c(B),ma=s(B,"LI",{});var Fm=o(ma);Ns=s(Fm,"CODE",{});var Nv=o(Ns);ff=i(Nv,"huggingface_sb3"),Nv.forEach(t),cf=i(Fm,": Additional code for Stable-baselines3 to load and upload models from the Hugging Face \u{1F917} Hub."),Fm.forEach(t),uf=c(B),da=s(B,"LI",{});var Vm=o(da);Ws=s(Vm,"CODE",{});var Wv=o(Ws);mf=i(Wv,"huggingface_hub"),Wv.forEach(t),df=i(Vm,": Library allowing anyone to work with the Hub repositories."),Vm.forEach(t),B.forEach(t),xr=c(e),m(Rt.$$.fragment,e),Dr=c(e),x=s(e,"H2",{class:!0});var Hi=o(x);Ie=s(Hi,"A",{id:!0,class:!0,href:!0});var Pv=o(Ie);Ps=s(Pv,"SPAN",{});var Rv=o(Ps);m(Qt.$$.fragment,Rv),Rv.forEach(t),Pv.forEach(t),vf=c(Hi),Rs=s(Hi,"SPAN",{});var Qv=o(Rs);yf=i(Qv,"Import the packages \u{1F4E6}"),Qv.forEach(t),Hi.forEach(t),qr=c(e),m(Yt.$$.fragment,e),Or=c(e),D=s(e,"H2",{class:!0});var Li=o(D);Se=s(Li,"A",{id:!0,class:!0,href:!0});var Yv=o(Se);Qs=s(Yv,"SPAN",{});var Fv=o(Qs);m(Ft.$$.fragment,Fv),Fv.forEach(t),Yv.forEach(t),bf=c(Li),Ys=s(Li,"SPAN",{});var Vv=o(Ys);wf=i(Vv,"Environment 1: AntBulletEnv-v0 \u{1F578}"),Vv.forEach(t),Li.forEach(t),Kr=c(e),q=s(e,"H3",{class:!0});var xi=o(q);Ce=s(xi,"A",{id:!0,class:!0,href:!0});var zv=o(Ce);Fs=s(zv,"SPAN",{});var Xv=o(Fs);m(Vt.$$.fragment,Xv),Xv.forEach(t),zv.forEach(t),_f=c(xi),Vs=s(xi,"SPAN",{});var Hv=o(Vs);gf=i(Hv,"Create the AntBulletEnv-v0"),Hv.forEach(t),xi.forEach(t),en=c(e),O=s(e,"H4",{class:!0});var Di=o(O);Ge=s(Di,"A",{id:!0,class:!0,href:!0});var Lv=o(Ge);zs=s(Lv,"SPAN",{});var xv=o(zs);m(zt.$$.fragment,xv),xv.forEach(t),Lv.forEach(t),Mf=c(Di),Xs=s(Di,"SPAN",{});var Dv=o(Xs);Ef=i(Dv,"The environment \u{1F3AE}"),Dv.forEach(t),Di.forEach(t),tn=c(e),Xt=s(e,"P",{});var zm=o(Xt);Uf=i(zm,`In this environment, the agent needs to use its different joints correctly in order to walk.
You can find a detailled explanation of this environment here: `),Ht=s(zm,"A",{href:!0,rel:!0});var qv=o(Ht);Tf=i(qv,"https://hackmd.io/@jeffreymo/SJJrSJh5_#PyBullet"),qv.forEach(t),zm.forEach(t),ln=c(e),m(Lt.$$.fragment,e),an=c(e),m(xt.$$.fragment,e),sn=c(e),Be=s(e,"P",{});var qi=o(Be);Jf=i(qi,"The observation Space (from "),Dt=s(qi,"A",{href:!0,rel:!0});var Ov=o(Dt);$f=i(Ov,"Jeffrey Y Mo"),Ov.forEach(t),kf=i(qi,`):
The difference is that our observation space is 28 not 29.`),qi.forEach(t),on=c(e),qt=s(e,"IMG",{src:!0,alt:!0}),rn=c(e),m(Ot.$$.fragment,e),nn=c(e),Ze=s(e,"P",{});var Oi=o(Ze);jf=i(Oi,"The action Space (from "),Kt=s(Oi,"A",{href:!0,rel:!0});var Kv=o(Kt);If=i(Kv,"Jeffrey Y Mo"),Kv.forEach(t),Sf=i(Oi,"):"),Oi.forEach(t),hn=c(e),el=s(e,"IMG",{src:!0,alt:!0}),pn=c(e),K=s(e,"H3",{class:!0});var Ki=o(K);Ae=s(Ki,"A",{id:!0,class:!0,href:!0});var ey=o(Ae);Hs=s(ey,"SPAN",{});var ty=o(Hs);m(tl.$$.fragment,ty),ty.forEach(t),ey.forEach(t),Cf=c(Ki),Ls=s(Ki,"SPAN",{});var ly=o(Ls);Gf=i(ly,"Normalize observation and rewards"),ly.forEach(t),Ki.forEach(t),fn=c(e),Ne=s(e,"P",{});var eh=o(Ne);Bf=i(eh,"A good practice in reinforcement learning is to "),ll=s(eh,"A",{href:!0,rel:!0});var ay=o(ll);Zf=i(ay,"normalize input features"),ay.forEach(t),Af=i(eh,"."),eh.forEach(t),cn=c(e),va=s(e,"P",{});var sy=o(va);Nf=i(sy,"For that purpose, there is a wrapper that will compute a running average and standard deviation of input features."),sy.forEach(t),un=c(e),al=s(e,"P",{});var Xm=o(al);Wf=i(Xm,"We also normalize rewards with this same wrapper by adding "),xs=s(Xm,"CODE",{});var oy=o(xs);Pf=i(oy,"norm_reward = True"),oy.forEach(t),Xm.forEach(t),mn=c(e),ya=s(e,"P",{});var ry=o(ya);sl=s(ry,"A",{href:!0,rel:!0});var ny=o(sl);Rf=i(ny,"You should check the documentation to fill this cell"),ny.forEach(t),ry.forEach(t),dn=c(e),m(ol.$$.fragment,e),vn=c(e),ee=s(e,"H4",{class:!0});var th=o(ee);We=s(th,"A",{id:!0,class:!0,href:!0});var iy=o(We);Ds=s(iy,"SPAN",{});var hy=o(Ds);m(rl.$$.fragment,hy),hy.forEach(t),iy.forEach(t),Qf=c(th),qs=s(th,"SPAN",{});var py=o(qs);Yf=i(py,"Solution"),py.forEach(t),th.forEach(t),yn=c(e),m(nl.$$.fragment,e),bn=c(e),te=s(e,"H3",{class:!0});var lh=o(te);Pe=s(lh,"A",{id:!0,class:!0,href:!0});var fy=o(Pe);Os=s(fy,"SPAN",{});var cy=o(Os);m(il.$$.fragment,cy),cy.forEach(t),fy.forEach(t),Ff=c(lh),Ks=s(lh,"SPAN",{});var uy=o(Ks);Vf=i(uy,"Create the A2C Model \u{1F916}"),uy.forEach(t),lh.forEach(t),wn=c(e),ba=s(e,"P",{});var my=o(ba);zf=i(my,"In this case, because we have a vector of 28 values as input, we\u2019ll use an MLP (multi-layer perceptron) as policy."),my.forEach(t),_n=c(e),hl=s(e,"P",{});var Hm=o(hl);Xf=i(Hm,"For more information about A2C implementation with StableBaselines3 check: "),pl=s(Hm,"A",{href:!0,rel:!0});var dy=o(pl);Hf=i(dy,"https://stable-baselines3.readthedocs.io/en/master/modules/a2c.html#notes"),dy.forEach(t),Hm.forEach(t),gn=c(e),Re=s(e,"P",{});var ah=o(Re);Lf=i(ah,"To find the best parameters I checked the "),fl=s(ah,"A",{href:!0,rel:!0});var vy=o(fl);xf=i(vy,"official trained agents by Stable-Baselines3 team"),vy.forEach(t),Df=i(ah,"."),ah.forEach(t),Mn=c(e),m(cl.$$.fragment,e),En=c(e),le=s(e,"H4",{class:!0});var sh=o(le);Qe=s(sh,"A",{id:!0,class:!0,href:!0});var yy=o(Qe);eo=s(yy,"SPAN",{});var by=o(eo);m(ul.$$.fragment,by),by.forEach(t),yy.forEach(t),qf=c(sh),to=s(sh,"SPAN",{});var wy=o(to);Of=i(wy,"Solution"),wy.forEach(t),sh.forEach(t),Un=c(e),m(ml.$$.fragment,e),Tn=c(e),ae=s(e,"H3",{class:!0});var oh=o(ae);Ye=s(oh,"A",{id:!0,class:!0,href:!0});var _y=o(Ye);lo=s(_y,"SPAN",{});var gy=o(lo);m(dl.$$.fragment,gy),gy.forEach(t),_y.forEach(t),Kf=c(oh),ao=s(oh,"SPAN",{});var My=o(ao);ec=i(My,"Train the A2C agent \u{1F3C3}"),My.forEach(t),oh.forEach(t),Jn=c(e),wa=s(e,"UL",{});var Ey=o(wa);so=s(Ey,"LI",{});var Uy=o(so);tc=i(Uy,"Let\u2019s train our agent for 2,000,000 timesteps. Don\u2019t forget to use GPU on Colab. It will take approximately ~25-40min"),Uy.forEach(t),Ey.forEach(t),$n=c(e),m(vl.$$.fragment,e),kn=c(e),m(yl.$$.fragment,e),jn=c(e),se=s(e,"H3",{class:!0});var rh=o(se);Fe=s(rh,"A",{id:!0,class:!0,href:!0});var Ty=o(Fe);oo=s(Ty,"SPAN",{});var Jy=o(oo);m(bl.$$.fragment,Jy),Jy.forEach(t),Ty.forEach(t),lc=c(rh),ro=s(rh,"SPAN",{});var $y=o(ro);ac=i($y,"Evaluate the agent \u{1F4C8}"),$y.forEach(t),rh.forEach(t),In=i(e,`

- Now that our  agent is trained, we need to **check its performance**.
- Stable-Baselines3 provides a method to do that: \`evaluate_policy\`
- In my case, I got a mean reward of \`2371.90 +/- 16.50\`

	`),m(wl.$$.fragment,e),Sn=c(e),oe=s(e,"H3",{class:!0});var nh=o(oe);Ve=s(nh,"A",{id:!0,class:!0,href:!0});var ky=o(Ve);no=s(ky,"SPAN",{});var jy=o(no);m(_l.$$.fragment,jy),jy.forEach(t),ky.forEach(t),sc=c(nh),io=s(nh,"SPAN",{});var Iy=o(io);oc=i(Iy,"Publish your trained model on the Hub \u{1F525}"),Iy.forEach(t),nh.forEach(t),Cn=i(e,`

Now that we saw we got good results after the training, we can publish our trained model on the Hub with one line of code.
`),gl=s(e,"P",{});var Lm=o(gl);rc=i(Lm,"\u{1F4DA} The libraries documentation \u{1F449} "),Ml=s(Lm,"A",{href:!0,rel:!0});var Sy=o(Ml);nc=i(Sy,"https://github.com/huggingface/huggingface_sb3/tree/main#hugging-face\u2014x-stable-baselines3-v20"),Sy.forEach(t),Lm.forEach(t),Gn=c(e),_a=s(e,"P",{});var Cy=o(_a);ic=i(Cy,"Here\u2019s an example of a Model Card (with a PyBullet environment):"),Cy.forEach(t),Bn=c(e),El=s(e,"IMG",{src:!0,alt:!0}),Zn=c(e),j=s(e,"P",{});var Pa=o(j);hc=i(Pa,"By using "),ho=s(Pa,"CODE",{});var Gy=o(ho);pc=i(Gy,"package_to_hub"),Gy.forEach(t),fc=i(Pa,", as we already mentionned in the former units, "),po=s(Pa,"STRONG",{});var By=o(po);cc=i(By,"you evaluate, record a replay, generate a model card of your agent and push it to the hub"),By.forEach(t),uc=i(Pa,"."),Pa.forEach(t),An=c(e),ga=s(e,"P",{});var Zy=o(ga);mc=i(Zy,"This way:"),Zy.forEach(t),Nn=c(e),J=s(e,"UL",{});var nt=o(J);Ul=s(nt,"LI",{});var ih=o(Ul);dc=i(ih,"You can "),fo=s(ih,"STRONG",{});var Ay=o(fo);vc=i(Ay,"showcase our work"),Ay.forEach(t),yc=i(ih," \u{1F525}"),ih.forEach(t),bc=c(nt),Tl=s(nt,"LI",{});var hh=o(Tl);wc=i(hh,"You can "),co=s(hh,"STRONG",{});var Ny=o(co);_c=i(Ny,"visualize your agent playing"),Ny.forEach(t),gc=i(hh," \u{1F440}"),hh.forEach(t),Mc=c(nt),Jl=s(nt,"LI",{});var ph=o(Jl);Ec=i(ph,"You can "),uo=s(ph,"STRONG",{});var Wy=o(uo);Uc=i(Wy,"share an agent with the community that others can use"),Wy.forEach(t),Tc=i(ph," \u{1F4BE}"),ph.forEach(t),Jc=c(nt),ze=s(nt,"LI",{});var hr=o(ze);$c=i(hr,"You can "),mo=s(hr,"STRONG",{});var Py=o(mo);kc=i(Py,"access a leaderboard \u{1F3C6} to see how well your agent is performing compared to your classmates"),Py.forEach(t),jc=i(hr," \u{1F449} "),$l=s(hr,"A",{href:!0,rel:!0});var Ry=o($l);Ic=i(Ry,"https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),Ry.forEach(t),hr.forEach(t),nt.forEach(t),Wn=c(e),Ma=s(e,"P",{});var Qy=o(Ma);Sc=i(Qy,"To be able to share your model with the community there are three more steps to follow:"),Qy.forEach(t),Pn=c(e),kl=s(e,"P",{});var xm=o(kl);Cc=i(xm,"1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),jl=s(xm,"A",{href:!0,rel:!0});var Yy=o(jl);Gc=i(Yy,"https://huggingface.co/join"),Yy.forEach(t),xm.forEach(t),Rn=c(e),Ea=s(e,"P",{});var Fy=o(Ea);Bc=i(Fy,"2\uFE0F\u20E3 Sign in and then you need to get your authentication token from the Hugging Face website."),Fy.forEach(t),Qn=c(e),Ua=s(e,"UL",{});var Vy=o(Ua);Xe=s(Vy,"LI",{});var pr=o(Xe);Zc=i(pr,"Create a new token ("),Il=s(pr,"A",{href:!0,rel:!0});var zy=o(Il);Ac=i(zy,"https://huggingface.co/settings/tokens"),zy.forEach(t),Nc=i(pr,") "),vo=s(pr,"STRONG",{});var Xy=o(vo);Wc=i(Xy,"with write role"),Xy.forEach(t),pr.forEach(t),Vy.forEach(t),Yn=c(e),Sl=s(e,"IMG",{src:!0,alt:!0}),Fn=c(e),He=s(e,"UL",{});var fh=o(He);yo=s(fh,"LI",{});var Hy=o(yo);Pc=i(Hy,"Copy the token"),Hy.forEach(t),Rc=c(fh),bo=s(fh,"LI",{});var Ly=o(bo);Qc=i(Ly,"Run the cell below and paste the token"),Ly.forEach(t),fh.forEach(t),Vn=c(e),m(Cl.$$.fragment,e),zn=c(e),Gl=s(e,"P",{});var Dm=o(Gl);Yc=i(Dm,"If you don\u2019t want to use Google Colab or a Jupyter Notebook, you need to use this command instead: "),wo=s(Dm,"CODE",{});var xy=o(wo);Fc=i(xy,"huggingface-cli login"),xy.forEach(t),Dm.forEach(t),Xn=c(e),Le=s(e,"P",{});var ch=o(Le);Vc=i(ch,"3\uFE0F\u20E3 We\u2019re now ready to push our trained agent to the \u{1F917} Hub \u{1F525} using "),_o=s(ch,"CODE",{});var Dy=o(_o);zc=i(Dy,"package_to_hub()"),Dy.forEach(t),Xc=i(ch," function"),ch.forEach(t),Hn=c(e),m(Bl.$$.fragment,e),Ln=c(e),re=s(e,"H2",{class:!0});var uh=o(re);xe=s(uh,"A",{id:!0,class:!0,href:!0});var qy=o(xe);go=s(qy,"SPAN",{});var Oy=o(go);m(Zl.$$.fragment,Oy),Oy.forEach(t),qy.forEach(t),Hc=c(uh),Mo=s(uh,"SPAN",{});var Ky=o(Mo);Lc=i(Ky,"Take a coffee break \u2615"),Ky.forEach(t),uh.forEach(t),xn=i(e,`

- You already trained your first robot that learned to move congratutlations \u{1F973}!
- It's **time to take a break**. Don't hesitate to **save this notebook** \`File > Save a copy to Drive\` to work on this second part later.
`),ne=s(e,"H2",{class:!0});var mh=o(ne);De=s(mh,"A",{id:!0,class:!0,href:!0});var eb=o(De);Eo=s(eb,"SPAN",{});var tb=o(Eo);m(Al.$$.fragment,tb),tb.forEach(t),eb.forEach(t),xc=c(mh),Uo=s(mh,"SPAN",{});var lb=o(Uo);Dc=i(lb,"Environment 2: PandaReachDense-v2 \u{1F9BE}"),lb.forEach(t),mh.forEach(t),Dn=c(e),Ta=s(e,"P",{});var ab=o(Ta);qc=i(ab,"The agent we\u2019re going to train is a robotic arm that needs to do controls (moving the arm and using the end-effector)."),ab.forEach(t),qn=c(e),qe=s(e,"P",{});var dh=o(qe);Oc=i(dh,"In robotics, the "),To=s(dh,"EM",{});var sb=o(To);Kc=i(sb,"end-effector"),sb.forEach(t),eu=i(dh," is the device at the end of a robotic arm designed to interact with the environment."),dh.forEach(t),On=c(e),Oe=s(e,"P",{});var vh=o(Oe);tu=i(vh,"In "),Jo=s(vh,"CODE",{});var ob=o(Jo);lu=i(ob,"PandaReach"),ob.forEach(t),au=i(vh,", the robot must place its end-effector at a target position (green ball)."),vh.forEach(t),Kn=c(e),U=s(e,"P",{});var Z=o(U);su=i(Z,"We\u2019re going to use the dense version of this environment. This means we\u2019ll get a "),$o=s(Z,"EM",{});var rb=o($o);ou=i(rb,"dense reward function"),rb.forEach(t),ru=i(Z," that "),ko=s(Z,"STRONG",{});var nb=o(ko);nu=i(nb,"will provide a reward at each timestep"),nb.forEach(t),iu=i(Z," (the closer the agent is to completing the task, the higher the reward). This is in contrast to a "),jo=s(Z,"EM",{});var ib=o(jo);hu=i(ib,"sparse reward function"),ib.forEach(t),pu=i(Z," where the environment "),Io=s(Z,"STRONG",{});var hb=o(Io);fu=i(hb,"return a reward if and only if the task is completed"),hb.forEach(t),cu=i(Z,"."),Z.forEach(t),ei=c(e),I=s(e,"P",{});var Ra=o(I);uu=i(Ra,"Also, we\u2019re going to use the "),So=s(Ra,"EM",{});var pb=o(So);mu=i(pb,"End-effector displacement control"),pb.forEach(t),du=i(Ra,", which means the "),Co=s(Ra,"STRONG",{});var fb=o(Co);vu=i(fb,"action corresponds to the displacement of the end-effector"),fb.forEach(t),yu=i(Ra,". We don\u2019t control the individual motion of each joint (joint control)."),Ra.forEach(t),ti=c(e),Nl=s(e,"IMG",{src:!0,alt:!0}),li=c(e),Ke=s(e,"P",{});var yh=o(Ke);bu=i(yh,"This way "),Go=s(yh,"STRONG",{});var cb=o(Go);wu=i(cb,"the training will be easier"),cb.forEach(t),_u=i(yh,"."),yh.forEach(t),ai=c(e),et=s(e,"P",{});var bh=o(et);gu=i(bh,"In "),Bo=s(bh,"CODE",{});var ub=o(Bo);Mu=i(ub,"PandaReachDense-v2"),ub.forEach(t),Eu=i(bh,", the robotic arm must place its end-effector at a target position (green ball)."),bh.forEach(t),si=c(e),m(Wl.$$.fragment,e),oi=c(e),m(Pl.$$.fragment,e),ri=c(e),tt=s(e,"P",{});var wh=o(tt);Uu=i(wh,"The observation space "),Zo=s(wh,"STRONG",{});var mb=o(Zo);Tu=i(mb,"is a dictionary with 3 different elements"),mb.forEach(t),Ju=i(wh,":"),wh.forEach(t),ni=c(e),S=s(e,"UL",{});var Qa=o(S);Ja=s(Qa,"LI",{});var qm=o(Ja);Ao=s(qm,"CODE",{});var db=o(Ao);$u=i(db,"achieved_goal"),db.forEach(t),ku=i(qm,": (x,y,z) position of the goal."),qm.forEach(t),ju=c(Qa),$a=s(Qa,"LI",{});var Om=o($a);No=s(Om,"CODE",{});var vb=o(No);Iu=i(vb,"desired_goal"),vb.forEach(t),Su=i(Om,": (x,y,z) distance between the goal position and the current object position."),Om.forEach(t),Cu=c(Qa),ka=s(Qa,"LI",{});var Km=o(ka);Wo=s(Km,"CODE",{});var yb=o(Wo);Gu=i(yb,"observation"),yb.forEach(t),Bu=i(Km,": position (x,y,z) and velocity of the end-effector (vx, vy, vz)."),Km.forEach(t),Qa.forEach(t),ii=c(e),lt=s(e,"P",{});var _h=o(lt);Zu=i(_h,"Given it\u2019s a dictionary as observation, "),Po=s(_h,"STRONG",{});var bb=o(Po);Au=i(bb,"we will need to use a MultiInputPolicy policy instead of MlpPolicy"),bb.forEach(t),Nu=i(_h,"."),_h.forEach(t),hi=c(e),m(Rl.$$.fragment,e),pi=c(e),ja=s(e,"P",{});var wb=o(ja);Wu=i(wb,"The action space is a vector with 3 values:"),wb.forEach(t),fi=c(e),Ia=s(e,"UL",{});var _b=o(Ia);Ro=s(_b,"LI",{});var gb=o(Ro);Pu=i(gb,"Control x, y, z movement"),gb.forEach(t),_b.forEach(t),ci=c(e),Sa=s(e,"P",{});var Mb=o(Sa);Ru=i(Mb,"Now it\u2019s your turn:"),Mb.forEach(t),ui=c(e),g=s(e,"OL",{});var M=o(g);Qo=s(M,"LI",{});var Eb=o(Qo);Qu=i(Eb,"Define the environment called \u201CPandaReachDense-v2\u201D."),Eb.forEach(t),Yu=c(M),Yo=s(M,"LI",{});var Ub=o(Yo);Fu=i(Ub,"Make a vectorized environment."),Ub.forEach(t),Vu=c(M),Ca=s(M,"LI",{});var ed=o(Ca);zu=i(ed,"Add a wrapper to normalize the observations and rewards. "),Ql=s(ed,"A",{href:!0,rel:!0});var Tb=o(Ql);Xu=i(Tb,"Check the documentation"),Tb.forEach(t),ed.forEach(t),Hu=c(M),Fo=s(M,"LI",{});var Jb=o(Fo);Lu=i(Jb,"Create the A2C Model (don\u2019t forget verbose=1 to print the training logs)."),Jb.forEach(t),xu=c(M),Vo=s(M,"LI",{});var $b=o(Vo);Du=i($b,"Train it for 1M Timesteps."),$b.forEach(t),qu=c(M),zo=s(M,"LI",{});var kb=o(zo);Ou=i(kb,"Save the model and  VecNormalize statistics when saving the agent."),kb.forEach(t),Ku=c(M),Xo=s(M,"LI",{});var jb=o(Xo);em=i(jb,"Evaluate your agent."),jb.forEach(t),tm=c(M),Yl=s(M,"LI",{});var gh=o(Yl);lm=i(gh,"Publish your trained model on the Hub \u{1F525} with "),Ho=s(gh,"CODE",{});var Ib=o(Ho);am=i(Ib,"package_to_hub"),Ib.forEach(t),sm=i(gh,"."),gh.forEach(t),M.forEach(t),mi=c(e),ie=s(e,"H3",{class:!0});var Mh=o(ie);at=s(Mh,"A",{id:!0,class:!0,href:!0});var Sb=o(at);Lo=s(Sb,"SPAN",{});var Cb=o(Lo);m(Fl.$$.fragment,Cb),Cb.forEach(t),Sb.forEach(t),om=c(Mh),xo=s(Mh,"SPAN",{});var Gb=o(xo);rm=i(Gb,"Solution (fill the todo)"),Gb.forEach(t),Mh.forEach(t),di=c(e),m(Vl.$$.fragment,e),vi=c(e),m(zl.$$.fragment,e),yi=c(e),he=s(e,"H2",{class:!0});var Eh=o(he);st=s(Eh,"A",{id:!0,class:!0,href:!0});var Bb=o(st);Do=s(Bb,"SPAN",{});var Zb=o(Do);m(Xl.$$.fragment,Zb),Zb.forEach(t),Bb.forEach(t),nm=c(Eh),qo=s(Eh,"SPAN",{});var Ab=o(qo);im=i(Ab,"Some additional challenges \u{1F3C6}"),Ab.forEach(t),Eh.forEach(t),bi=c(e),$=s(e,"P",{});var it=o($);hm=i(it,"The best way to learn "),Oo=s(it,"STRONG",{});var Nb=o(Oo);pm=i(Nb,"is to try things on your own"),Nb.forEach(t),fm=i(it,"! Why not try  "),Ko=s(it,"CODE",{});var Wb=o(Ko);cm=i(Wb,"HalfCheetahBulletEnv-v0"),Wb.forEach(t),um=i(it," for PyBullet and "),er=s(it,"CODE",{});var Pb=o(er);mm=i(Pb,"PandaPickAndPlace-v1"),Pb.forEach(t),dm=i(it," for Panda-Gym?"),it.forEach(t),wi=c(e),C=s(e,"P",{});var Ya=o(C);vm=i(Ya,"If you want to try more advanced tasks for panda-gym, you need to check what was done using "),tr=s(Ya,"STRONG",{});var Rb=o(tr);ym=i(Rb,"TQC or SAC"),Rb.forEach(t),bm=i(Ya," (a more sample-efficient algorithm suited for robotics tasks). In real robotics, you\u2019ll use a more sample-efficient algorithm for a simple reason: contrary to a simulation "),lr=s(Ya,"STRONG",{});var Qb=o(lr);wm=i(Qb,"if you move your robotic arm too much, you have a risk of breaking it"),Qb.forEach(t),_m=i(Ya,"."),Ya.forEach(t),_i=c(e),Hl=s(e,"P",{});var td=o(Hl);gm=i(td,"PandaPickAndPlace-v1: "),Ll=s(td,"A",{href:!0,rel:!0});var Yb=o(Ll);Mm=i(Yb,"https://huggingface.co/sb3/tqc-PandaPickAndPlace-v1"),Yb.forEach(t),td.forEach(t),gi=c(e),xl=s(e,"P",{});var ld=o(xl);Em=i(ld,"And don\u2019t hesitate to check panda-gym documentation here: "),Dl=s(ld,"A",{href:!0,rel:!0});var Fb=o(Dl);Um=i(Fb,"https://panda-gym.readthedocs.io/en/latest/usage/train_with_sb3.html"),Fb.forEach(t),ld.forEach(t),Mi=c(e),Ga=s(e,"P",{});var Vb=o(Ga);Tm=i(Vb,"Here are some ideas to go further:"),Vb.forEach(t),Ei=c(e),G=s(e,"UL",{});var Fa=o(G);ar=s(Fa,"LI",{});var zb=o(ar);Jm=i(zb,"Train more steps"),zb.forEach(t),$m=c(Fa),Ba=s(Fa,"LI",{});var ad=o(Ba);km=i(ad,"Try different hyperparameters by looking at what your classmates have done \u{1F449} "),ql=s(ad,"A",{href:!0,rel:!0});var Xb=o(ql);jm=i(Xb,"https://huggingface.co/models?other=https://huggingface.co/models?other=AntBulletEnv-v0"),Xb.forEach(t),ad.forEach(t),Im=c(Fa),Za=s(Fa,"LI",{});var sd=o(Za);sr=s(sd,"STRONG",{});var Hb=o(sr);Sm=i(Hb,"Push your new trained model"),Hb.forEach(t),Cm=i(sd," on the Hub \u{1F525}"),sd.forEach(t),Fa.forEach(t),Ui=c(e),Aa=s(e,"P",{});var Lb=o(Aa);Gm=i(Lb,"See you on Unit 7! \u{1F525}"),Lb.forEach(t),Ti=c(e),pe=s(e,"H2",{class:!0});var Uh=o(pe);ot=s(Uh,"A",{id:!0,class:!0,href:!0});var xb=o(ot);or=s(xb,"SPAN",{});var Db=o(or);m(Ol.$$.fragment,Db),Db.forEach(t),xb.forEach(t),Bm=c(Uh),rr=s(Uh,"SPAN",{});var qb=o(rr);Zm=i(qb,"Keep learning, stay awesome \u{1F917}"),qb.forEach(t),Uh.forEach(t),this.h()},h(){p(A,"name","hf:doc:metadata"),p(A,"content",JSON.stringify(rw)),p(ce,"id","hands-on"),p(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(ce,"href","#hands-on"),p(N,"class","relative group"),p(ft,"href","https://github.com/bulletphysics/bullet3"),p(ft,"rel","nofollow"),p(ct,"href","https://github.com/qgallouedec/panda-gym"),p(ct,"rel","nofollow"),fe(ut.src,rd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/environments.gif")||p(ut,"src",rd),p(ut,"alt","Environments"),p(mt,"href","https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),p(mt,"rel","nofollow"),p(vt,"href","https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),p(vt,"rel","nofollow"),fe(oa.src,nd="https://colab.research.google.com/assets/colab-badge.svg")||p(oa,"src",nd),p(oa,"alt","Open In Colab"),p(bt,"href","https://colab.research.google.com/github/huggingface/deep-rl-class/blob/master/notebooks/unit6/unit6.ipynb"),p(bt,"rel","nofollow"),p(ye,"id","unit-6-advantage-actor-critic-a2c-using-robotics-simulations-with-pybullet-and-pandagym"),p(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(ye,"href","#unit-6-advantage-actor-critic-a2c-using-robotics-simulations-with-pybullet-and-pandagym"),p(P,"class","relative group"),p(be,"id","environments"),p(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(be,"href","#environments"),p(R,"class","relative group"),p(gt,"href","https://github.com/bulletphysics/bullet3"),p(gt,"rel","nofollow"),p(Mt,"href","https://github.com/qgallouedec/panda-gym"),p(Mt,"rel","nofollow"),p(_e,"id","rllibrary"),p(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(_e,"href","#rllibrary"),p(Q,"class","relative group"),p(Ut,"href","https://stable-baselines3.readthedocs.io/"),p(Ut,"rel","nofollow"),p(Tt,"href","https://github.com/huggingface/deep-rl-class/issues"),p(Tt,"rel","nofollow"),p(ge,"id","objectives-of-this-notebook"),p(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(ge,"href","#objectives-of-this-notebook"),p(Y,"class","relative group"),p(Me,"id","prerequisites"),p(Me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Me,"href","#prerequisites"),p(V,"class","relative group"),p(St,"href","https://huggingface.co/deep-rl-course/unit6/introduction"),p(St,"rel","nofollow"),p(Ue,"id","lets-train-our-first-robots"),p(Ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Ue,"href","#lets-train-our-first-robots"),p(z,"class","relative group"),p(Te,"id","set-the-gpu"),p(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Te,"href","#set-the-gpu"),p(X,"class","relative group"),fe(Bt.src,id="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step1.jpg")||p(Bt,"src",id),p(Bt,"alt","GPU Step 1"),fe(Zt.src,hd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step2.jpg")||p(Zt,"src",hd),p(Zt,"alt","GPU Step 2"),p($e,"id","create-a-virtual-display"),p($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p($e,"href","#create-a-virtual-display"),p(H,"class","relative group"),p(je,"id","install-dependencies"),p(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(je,"href","#install-dependencies"),p(L,"class","relative group"),p(Ie,"id","import-the-packages"),p(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Ie,"href","#import-the-packages"),p(x,"class","relative group"),p(Se,"id","environment-1-antbulletenvv0"),p(Se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Se,"href","#environment-1-antbulletenvv0"),p(D,"class","relative group"),p(Ce,"id","create-the-antbulletenvv0"),p(Ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Ce,"href","#create-the-antbulletenvv0"),p(q,"class","relative group"),p(Ge,"id","the-environment"),p(Ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Ge,"href","#the-environment"),p(O,"class","relative group"),p(Ht,"href","https://hackmd.io/@jeffreymo/SJJrSJh5_#PyBullet"),p(Ht,"rel","nofollow"),p(Dt,"href","https://hackmd.io/@jeffreymo/SJJrSJh5_#PyBullet"),p(Dt,"rel","nofollow"),fe(qt.src,pd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/obs_space.png")||p(qt,"src",pd),p(qt,"alt","PyBullet Ant Obs space"),p(Kt,"href","https://hackmd.io/@jeffreymo/SJJrSJh5_#PyBullet"),p(Kt,"rel","nofollow"),fe(el.src,fd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/action_space.png")||p(el,"src",fd),p(el,"alt","PyBullet Ant Obs space"),p(Ae,"id","normalize-observation-and-rewards"),p(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Ae,"href","#normalize-observation-and-rewards"),p(K,"class","relative group"),p(ll,"href","https://stable-baselines3.readthedocs.io/en/master/guide/rl_tips.html"),p(ll,"rel","nofollow"),p(sl,"href","https://stable-baselines3.readthedocs.io/en/master/guide/vec_envs.html#vecnormalize"),p(sl,"rel","nofollow"),p(We,"id","solution"),p(We,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(We,"href","#solution"),p(ee,"class","relative group"),p(Pe,"id","create-the-a2c-model"),p(Pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Pe,"href","#create-the-a2c-model"),p(te,"class","relative group"),p(pl,"href","https://stable-baselines3.readthedocs.io/en/master/modules/a2c.html#notes"),p(pl,"rel","nofollow"),p(fl,"href","https://huggingface.co/sb3"),p(fl,"rel","nofollow"),p(Qe,"id","solution"),p(Qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Qe,"href","#solution"),p(le,"class","relative group"),p(Ye,"id","train-the-a2c-agent"),p(Ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Ye,"href","#train-the-a2c-agent"),p(ae,"class","relative group"),p(Fe,"id","evaluate-the-agent"),p(Fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Fe,"href","#evaluate-the-agent"),p(se,"class","relative group"),p(Ve,"id","publish-your-trained-model-on-the-hub"),p(Ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(Ve,"href","#publish-your-trained-model-on-the-hub"),p(oe,"class","relative group"),p(Ml,"href","https://github.com/huggingface/huggingface_sb3/tree/main#hugging-face--x-stable-baselines3-v20"),p(Ml,"rel","nofollow"),fe(El.src,cd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/modelcardpybullet.png")||p(El,"src",cd),p(El,"alt","Model Card Pybullet"),p($l,"href","https://huggingface.co/spaces/huggingface-projects/Deep-Reinforcement-Learning-Leaderboard"),p($l,"rel","nofollow"),p(jl,"href","https://huggingface.co/join"),p(jl,"rel","nofollow"),p(Il,"href","https://huggingface.co/settings/tokens"),p(Il,"rel","nofollow"),fe(Sl.src,ud="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/create-token.jpg")||p(Sl,"src",ud),p(Sl,"alt","Create HF Token"),p(xe,"id","take-a-coffee-break"),p(xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(xe,"href","#take-a-coffee-break"),p(re,"class","relative group"),p(De,"id","environment-2-pandareachdensev2"),p(De,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(De,"href","#environment-2-pandareachdensev2"),p(ne,"class","relative group"),fe(Nl.src,md="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit8/robotics.jpg")||p(Nl,"src",md),p(Nl,"alt","Robotics"),p(Ql,"href","https://stable-baselines3.readthedocs.io/en/master/guide/vec_envs.html#vecnormalize"),p(Ql,"rel","nofollow"),p(at,"id","solution-fill-the-todo"),p(at,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(at,"href","#solution-fill-the-todo"),p(ie,"class","relative group"),p(st,"id","some-additional-challenges"),p(st,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(st,"href","#some-additional-challenges"),p(he,"class","relative group"),p(Ll,"href","https://huggingface.co/sb3/tqc-PandaPickAndPlace-v1"),p(Ll,"rel","nofollow"),p(Dl,"href","https://panda-gym.readthedocs.io/en/latest/usage/train_with_sb3.html"),p(Dl,"rel","nofollow"),p(ql,"href","https://huggingface.co/models?other=https://huggingface.co/models?other=AntBulletEnv-v0"),p(ql,"rel","nofollow"),p(ot,"id","keep-learning-stay-awesome"),p(ot,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(ot,"href","#keep-learning-stay-awesome"),p(pe,"class","relative group")},m(e,r){l(document.head,A),h(e,fr,r),h(e,N,r),l(N,ce),l(ce,Va),d(ht,Va,null),l(N,Th),l(N,za),l(za,Jh),h(e,cr,r),d(pt,e,r),h(e,ur,r),h(e,ue,r),l(ue,$h),l(ue,Xa),l(Xa,kh),l(ue,jh),h(e,mr,r),h(e,me,r),l(me,Ha),l(Ha,Ih),l(me,Sh),l(me,La),l(La,Ch),h(e,dr,r),h(e,Kl,r),l(Kl,Gh),h(e,vr,r),h(e,de,r),l(de,xa),l(xa,ft),l(ft,Bh),l(de,Zh),l(de,Da),l(Da,ct),l(ct,Ah),h(e,yr,r),h(e,ut,r),h(e,br,r),h(e,ea,r),l(ea,Nh),h(e,wr,r),h(e,ve,r),l(ve,ta),l(ta,qa),l(qa,Wh),l(ta,Ph),l(ve,Rh),l(ve,la),l(la,Oa),l(Oa,Qh),l(la,Yh),h(e,_r,r),h(e,W,r),l(W,Fh),l(W,mt),l(mt,Vh),l(W,zh),l(W,Ka),l(Ka,Xh),h(e,gr,r),h(e,aa,r),l(aa,es),l(es,Hh),h(e,Mr,r),h(e,dt,r),l(dt,Lh),l(dt,vt),l(vt,xh),h(e,Er,r),h(e,yt,r),l(yt,ts),l(ts,Dh),l(yt,qh),h(e,Ur,r),h(e,sa,r),l(sa,bt),l(bt,oa),h(e,Tr,r),h(e,P,r),l(P,ye),l(ye,ls),d(wt,ls,null),l(P,Oh),l(P,as),l(as,Kh),h(e,Jr,r),h(e,R,r),l(R,be),l(be,ss),d(_t,ss,null),l(R,ep),l(R,os),l(os,tp),h(e,$r,r),h(e,we,r),l(we,rs),l(rs,gt),l(gt,lp),l(we,ap),l(we,ns),l(ns,Mt),l(Mt,sp),h(e,kr,r),h(e,Q,r),l(Q,_e),l(_e,is),d(Et,is,null),l(Q,op),l(Q,hs),l(hs,rp),h(e,jr,r),h(e,ra,r),l(ra,ps),l(ps,Ut),l(Ut,np),h(e,Ir,r),h(e,k,r),l(k,ip),l(k,fs),l(fs,hp),l(k,pp),l(k,Tt),l(Tt,fp),l(k,cp),h(e,Sr,r),h(e,Y,r),l(Y,ge),l(ge,cs),d(Jt,cs,null),l(Y,up),l(Y,us),l(us,mp),h(e,Cr,r),h(e,na,r),l(na,dp),h(e,Gr,r),h(e,T,r),l(T,F),l(F,vp),l(F,ms),l(ms,yp),l(F,bp),l(F,ds),l(ds,wp),l(F,_p),l(T,gp),l(T,$t),l($t,Mp),l($t,vs),l(vs,Ep),l($t,Up),l(T,Tp),l(T,kt),l(kt,Jp),l(kt,ys),l(ys,$p),l(kt,kp),l(T,jp),l(T,jt),l(jt,Ip),l(jt,bs),l(bs,Sp),l(jt,Cp),h(e,Br,r),h(e,V,r),l(V,Me),l(Me,ws),d(It,ws,null),l(V,Gp),l(V,_s),l(_s,Bp),h(e,Zr,r),h(e,Ee,r),l(Ee,Zp),l(Ee,St),l(St,Ap),l(Ee,Np),h(e,Ar,r),h(e,z,r),l(z,Ue),l(Ue,gs),d(Ct,gs,null),l(z,Wp),l(z,Ms),l(Ms,Pp),h(e,Nr,r),h(e,X,r),l(X,Te),l(Te,Es),d(Gt,Es,null),l(X,Rp),l(X,Us),l(Us,Qp),h(e,Wr,r),h(e,ia,r),l(ia,Je),l(Je,Yp),l(Je,Ts),l(Ts,Fp),l(Je,Vp),l(Je,Js),l(Js,zp),h(e,Pr,r),h(e,Bt,r),h(e,Rr,r),h(e,ha,r),l(ha,$s),l($s,ks),l(ks,Xp),h(e,Qr,r),h(e,Zt,r),h(e,Yr,r),h(e,H,r),l(H,$e),l($e,js),d(At,js,null),l(H,Hp),l(H,Is),l(Is,Lp),h(e,Fr,r),h(e,ke,r),l(ke,xp),l(ke,Ss),l(Ss,Dp),l(ke,qp),h(e,Vr,r),h(e,pa,r),l(pa,Op),h(e,zr,r),d(Nt,e,r),h(e,Xr,r),d(Wt,e,r),h(e,Hr,r),h(e,L,r),l(L,je),l(je,Cs),d(Pt,Cs,null),l(L,Kp),l(L,Gs),l(Gs,ef),h(e,Lr,r),h(e,E,r),l(E,fa),l(fa,Bs),l(Bs,tf),l(fa,lf),l(E,af),l(E,ca),l(ca,Zs),l(Zs,sf),l(ca,of),l(E,rf),l(E,ua),l(ua,As),l(As,nf),l(ua,hf),l(E,pf),l(E,ma),l(ma,Ns),l(Ns,ff),l(ma,cf),l(E,uf),l(E,da),l(da,Ws),l(Ws,mf),l(da,df),h(e,xr,r),d(Rt,e,r),h(e,Dr,r),h(e,x,r),l(x,Ie),l(Ie,Ps),d(Qt,Ps,null),l(x,vf),l(x,Rs),l(Rs,yf),h(e,qr,r),d(Yt,e,r),h(e,Or,r),h(e,D,r),l(D,Se),l(Se,Qs),d(Ft,Qs,null),l(D,bf),l(D,Ys),l(Ys,wf),h(e,Kr,r),h(e,q,r),l(q,Ce),l(Ce,Fs),d(Vt,Fs,null),l(q,_f),l(q,Vs),l(Vs,gf),h(e,en,r),h(e,O,r),l(O,Ge),l(Ge,zs),d(zt,zs,null),l(O,Mf),l(O,Xs),l(Xs,Ef),h(e,tn,r),h(e,Xt,r),l(Xt,Uf),l(Xt,Ht),l(Ht,Tf),h(e,ln,r),d(Lt,e,r),h(e,an,r),d(xt,e,r),h(e,sn,r),h(e,Be,r),l(Be,Jf),l(Be,Dt),l(Dt,$f),l(Be,kf),h(e,on,r),h(e,qt,r),h(e,rn,r),d(Ot,e,r),h(e,nn,r),h(e,Ze,r),l(Ze,jf),l(Ze,Kt),l(Kt,If),l(Ze,Sf),h(e,hn,r),h(e,el,r),h(e,pn,r),h(e,K,r),l(K,Ae),l(Ae,Hs),d(tl,Hs,null),l(K,Cf),l(K,Ls),l(Ls,Gf),h(e,fn,r),h(e,Ne,r),l(Ne,Bf),l(Ne,ll),l(ll,Zf),l(Ne,Af),h(e,cn,r),h(e,va,r),l(va,Nf),h(e,un,r),h(e,al,r),l(al,Wf),l(al,xs),l(xs,Pf),h(e,mn,r),h(e,ya,r),l(ya,sl),l(sl,Rf),h(e,dn,r),d(ol,e,r),h(e,vn,r),h(e,ee,r),l(ee,We),l(We,Ds),d(rl,Ds,null),l(ee,Qf),l(ee,qs),l(qs,Yf),h(e,yn,r),d(nl,e,r),h(e,bn,r),h(e,te,r),l(te,Pe),l(Pe,Os),d(il,Os,null),l(te,Ff),l(te,Ks),l(Ks,Vf),h(e,wn,r),h(e,ba,r),l(ba,zf),h(e,_n,r),h(e,hl,r),l(hl,Xf),l(hl,pl),l(pl,Hf),h(e,gn,r),h(e,Re,r),l(Re,Lf),l(Re,fl),l(fl,xf),l(Re,Df),h(e,Mn,r),d(cl,e,r),h(e,En,r),h(e,le,r),l(le,Qe),l(Qe,eo),d(ul,eo,null),l(le,qf),l(le,to),l(to,Of),h(e,Un,r),d(ml,e,r),h(e,Tn,r),h(e,ae,r),l(ae,Ye),l(Ye,lo),d(dl,lo,null),l(ae,Kf),l(ae,ao),l(ao,ec),h(e,Jn,r),h(e,wa,r),l(wa,so),l(so,tc),h(e,$n,r),d(vl,e,r),h(e,kn,r),d(yl,e,r),h(e,jn,r),h(e,se,r),l(se,Fe),l(Fe,oo),d(bl,oo,null),l(se,lc),l(se,ro),l(ro,ac),h(e,In,r),d(wl,e,r),h(e,Sn,r),h(e,oe,r),l(oe,Ve),l(Ve,no),d(_l,no,null),l(oe,sc),l(oe,io),l(io,oc),h(e,Cn,r),h(e,gl,r),l(gl,rc),l(gl,Ml),l(Ml,nc),h(e,Gn,r),h(e,_a,r),l(_a,ic),h(e,Bn,r),h(e,El,r),h(e,Zn,r),h(e,j,r),l(j,hc),l(j,ho),l(ho,pc),l(j,fc),l(j,po),l(po,cc),l(j,uc),h(e,An,r),h(e,ga,r),l(ga,mc),h(e,Nn,r),h(e,J,r),l(J,Ul),l(Ul,dc),l(Ul,fo),l(fo,vc),l(Ul,yc),l(J,bc),l(J,Tl),l(Tl,wc),l(Tl,co),l(co,_c),l(Tl,gc),l(J,Mc),l(J,Jl),l(Jl,Ec),l(Jl,uo),l(uo,Uc),l(Jl,Tc),l(J,Jc),l(J,ze),l(ze,$c),l(ze,mo),l(mo,kc),l(ze,jc),l(ze,$l),l($l,Ic),h(e,Wn,r),h(e,Ma,r),l(Ma,Sc),h(e,Pn,r),h(e,kl,r),l(kl,Cc),l(kl,jl),l(jl,Gc),h(e,Rn,r),h(e,Ea,r),l(Ea,Bc),h(e,Qn,r),h(e,Ua,r),l(Ua,Xe),l(Xe,Zc),l(Xe,Il),l(Il,Ac),l(Xe,Nc),l(Xe,vo),l(vo,Wc),h(e,Yn,r),h(e,Sl,r),h(e,Fn,r),h(e,He,r),l(He,yo),l(yo,Pc),l(He,Rc),l(He,bo),l(bo,Qc),h(e,Vn,r),d(Cl,e,r),h(e,zn,r),h(e,Gl,r),l(Gl,Yc),l(Gl,wo),l(wo,Fc),h(e,Xn,r),h(e,Le,r),l(Le,Vc),l(Le,_o),l(_o,zc),l(Le,Xc),h(e,Hn,r),d(Bl,e,r),h(e,Ln,r),h(e,re,r),l(re,xe),l(xe,go),d(Zl,go,null),l(re,Hc),l(re,Mo),l(Mo,Lc),h(e,xn,r),h(e,ne,r),l(ne,De),l(De,Eo),d(Al,Eo,null),l(ne,xc),l(ne,Uo),l(Uo,Dc),h(e,Dn,r),h(e,Ta,r),l(Ta,qc),h(e,qn,r),h(e,qe,r),l(qe,Oc),l(qe,To),l(To,Kc),l(qe,eu),h(e,On,r),h(e,Oe,r),l(Oe,tu),l(Oe,Jo),l(Jo,lu),l(Oe,au),h(e,Kn,r),h(e,U,r),l(U,su),l(U,$o),l($o,ou),l(U,ru),l(U,ko),l(ko,nu),l(U,iu),l(U,jo),l(jo,hu),l(U,pu),l(U,Io),l(Io,fu),l(U,cu),h(e,ei,r),h(e,I,r),l(I,uu),l(I,So),l(So,mu),l(I,du),l(I,Co),l(Co,vu),l(I,yu),h(e,ti,r),h(e,Nl,r),h(e,li,r),h(e,Ke,r),l(Ke,bu),l(Ke,Go),l(Go,wu),l(Ke,_u),h(e,ai,r),h(e,et,r),l(et,gu),l(et,Bo),l(Bo,Mu),l(et,Eu),h(e,si,r),d(Wl,e,r),h(e,oi,r),d(Pl,e,r),h(e,ri,r),h(e,tt,r),l(tt,Uu),l(tt,Zo),l(Zo,Tu),l(tt,Ju),h(e,ni,r),h(e,S,r),l(S,Ja),l(Ja,Ao),l(Ao,$u),l(Ja,ku),l(S,ju),l(S,$a),l($a,No),l(No,Iu),l($a,Su),l(S,Cu),l(S,ka),l(ka,Wo),l(Wo,Gu),l(ka,Bu),h(e,ii,r),h(e,lt,r),l(lt,Zu),l(lt,Po),l(Po,Au),l(lt,Nu),h(e,hi,r),d(Rl,e,r),h(e,pi,r),h(e,ja,r),l(ja,Wu),h(e,fi,r),h(e,Ia,r),l(Ia,Ro),l(Ro,Pu),h(e,ci,r),h(e,Sa,r),l(Sa,Ru),h(e,ui,r),h(e,g,r),l(g,Qo),l(Qo,Qu),l(g,Yu),l(g,Yo),l(Yo,Fu),l(g,Vu),l(g,Ca),l(Ca,zu),l(Ca,Ql),l(Ql,Xu),l(g,Hu),l(g,Fo),l(Fo,Lu),l(g,xu),l(g,Vo),l(Vo,Du),l(g,qu),l(g,zo),l(zo,Ou),l(g,Ku),l(g,Xo),l(Xo,em),l(g,tm),l(g,Yl),l(Yl,lm),l(Yl,Ho),l(Ho,am),l(Yl,sm),h(e,mi,r),h(e,ie,r),l(ie,at),l(at,Lo),d(Fl,Lo,null),l(ie,om),l(ie,xo),l(xo,rm),h(e,di,r),d(Vl,e,r),h(e,vi,r),d(zl,e,r),h(e,yi,r),h(e,he,r),l(he,st),l(st,Do),d(Xl,Do,null),l(he,nm),l(he,qo),l(qo,im),h(e,bi,r),h(e,$,r),l($,hm),l($,Oo),l(Oo,pm),l($,fm),l($,Ko),l(Ko,cm),l($,um),l($,er),l(er,mm),l($,dm),h(e,wi,r),h(e,C,r),l(C,vm),l(C,tr),l(tr,ym),l(C,bm),l(C,lr),l(lr,wm),l(C,_m),h(e,_i,r),h(e,Hl,r),l(Hl,gm),l(Hl,Ll),l(Ll,Mm),h(e,gi,r),h(e,xl,r),l(xl,Em),l(xl,Dl),l(Dl,Um),h(e,Mi,r),h(e,Ga,r),l(Ga,Tm),h(e,Ei,r),h(e,G,r),l(G,ar),l(ar,Jm),l(G,$m),l(G,Ba),l(Ba,km),l(Ba,ql),l(ql,jm),l(G,Im),l(G,Za),l(Za,sr),l(sr,Sm),l(Za,Cm),h(e,Ui,r),h(e,Aa,r),l(Aa,Gm),h(e,Ti,r),h(e,pe,r),l(pe,ot),l(ot,or),d(Ol,or,null),l(pe,Bm),l(pe,rr),l(rr,Zm),Ji=!0},p:lw,i(e){Ji||(v(ht.$$.fragment,e),v(pt.$$.fragment,e),v(wt.$$.fragment,e),v(_t.$$.fragment,e),v(Et.$$.fragment,e),v(Jt.$$.fragment,e),v(It.$$.fragment,e),v(Ct.$$.fragment,e),v(Gt.$$.fragment,e),v(At.$$.fragment,e),v(Nt.$$.fragment,e),v(Wt.$$.fragment,e),v(Pt.$$.fragment,e),v(Rt.$$.fragment,e),v(Qt.$$.fragment,e),v(Yt.$$.fragment,e),v(Ft.$$.fragment,e),v(Vt.$$.fragment,e),v(zt.$$.fragment,e),v(Lt.$$.fragment,e),v(xt.$$.fragment,e),v(Ot.$$.fragment,e),v(tl.$$.fragment,e),v(ol.$$.fragment,e),v(rl.$$.fragment,e),v(nl.$$.fragment,e),v(il.$$.fragment,e),v(cl.$$.fragment,e),v(ul.$$.fragment,e),v(ml.$$.fragment,e),v(dl.$$.fragment,e),v(vl.$$.fragment,e),v(yl.$$.fragment,e),v(bl.$$.fragment,e),v(wl.$$.fragment,e),v(_l.$$.fragment,e),v(Cl.$$.fragment,e),v(Bl.$$.fragment,e),v(Zl.$$.fragment,e),v(Al.$$.fragment,e),v(Wl.$$.fragment,e),v(Pl.$$.fragment,e),v(Rl.$$.fragment,e),v(Fl.$$.fragment,e),v(Vl.$$.fragment,e),v(zl.$$.fragment,e),v(Xl.$$.fragment,e),v(Ol.$$.fragment,e),Ji=!0)},o(e){y(ht.$$.fragment,e),y(pt.$$.fragment,e),y(wt.$$.fragment,e),y(_t.$$.fragment,e),y(Et.$$.fragment,e),y(Jt.$$.fragment,e),y(It.$$.fragment,e),y(Ct.$$.fragment,e),y(Gt.$$.fragment,e),y(At.$$.fragment,e),y(Nt.$$.fragment,e),y(Wt.$$.fragment,e),y(Pt.$$.fragment,e),y(Rt.$$.fragment,e),y(Qt.$$.fragment,e),y(Yt.$$.fragment,e),y(Ft.$$.fragment,e),y(Vt.$$.fragment,e),y(zt.$$.fragment,e),y(Lt.$$.fragment,e),y(xt.$$.fragment,e),y(Ot.$$.fragment,e),y(tl.$$.fragment,e),y(ol.$$.fragment,e),y(rl.$$.fragment,e),y(nl.$$.fragment,e),y(il.$$.fragment,e),y(cl.$$.fragment,e),y(ul.$$.fragment,e),y(ml.$$.fragment,e),y(dl.$$.fragment,e),y(vl.$$.fragment,e),y(yl.$$.fragment,e),y(bl.$$.fragment,e),y(wl.$$.fragment,e),y(_l.$$.fragment,e),y(Cl.$$.fragment,e),y(Bl.$$.fragment,e),y(Zl.$$.fragment,e),y(Al.$$.fragment,e),y(Wl.$$.fragment,e),y(Pl.$$.fragment,e),y(Rl.$$.fragment,e),y(Fl.$$.fragment,e),y(Vl.$$.fragment,e),y(zl.$$.fragment,e),y(Xl.$$.fragment,e),y(Ol.$$.fragment,e),Ji=!1},d(e){t(A),e&&t(fr),e&&t(N),b(ht),e&&t(cr),b(pt,e),e&&t(ur),e&&t(ue),e&&t(mr),e&&t(me),e&&t(dr),e&&t(Kl),e&&t(vr),e&&t(de),e&&t(yr),e&&t(ut),e&&t(br),e&&t(ea),e&&t(wr),e&&t(ve),e&&t(_r),e&&t(W),e&&t(gr),e&&t(aa),e&&t(Mr),e&&t(dt),e&&t(Er),e&&t(yt),e&&t(Ur),e&&t(sa),e&&t(Tr),e&&t(P),b(wt),e&&t(Jr),e&&t(R),b(_t),e&&t($r),e&&t(we),e&&t(kr),e&&t(Q),b(Et),e&&t(jr),e&&t(ra),e&&t(Ir),e&&t(k),e&&t(Sr),e&&t(Y),b(Jt),e&&t(Cr),e&&t(na),e&&t(Gr),e&&t(T),e&&t(Br),e&&t(V),b(It),e&&t(Zr),e&&t(Ee),e&&t(Ar),e&&t(z),b(Ct),e&&t(Nr),e&&t(X),b(Gt),e&&t(Wr),e&&t(ia),e&&t(Pr),e&&t(Bt),e&&t(Rr),e&&t(ha),e&&t(Qr),e&&t(Zt),e&&t(Yr),e&&t(H),b(At),e&&t(Fr),e&&t(ke),e&&t(Vr),e&&t(pa),e&&t(zr),b(Nt,e),e&&t(Xr),b(Wt,e),e&&t(Hr),e&&t(L),b(Pt),e&&t(Lr),e&&t(E),e&&t(xr),b(Rt,e),e&&t(Dr),e&&t(x),b(Qt),e&&t(qr),b(Yt,e),e&&t(Or),e&&t(D),b(Ft),e&&t(Kr),e&&t(q),b(Vt),e&&t(en),e&&t(O),b(zt),e&&t(tn),e&&t(Xt),e&&t(ln),b(Lt,e),e&&t(an),b(xt,e),e&&t(sn),e&&t(Be),e&&t(on),e&&t(qt),e&&t(rn),b(Ot,e),e&&t(nn),e&&t(Ze),e&&t(hn),e&&t(el),e&&t(pn),e&&t(K),b(tl),e&&t(fn),e&&t(Ne),e&&t(cn),e&&t(va),e&&t(un),e&&t(al),e&&t(mn),e&&t(ya),e&&t(dn),b(ol,e),e&&t(vn),e&&t(ee),b(rl),e&&t(yn),b(nl,e),e&&t(bn),e&&t(te),b(il),e&&t(wn),e&&t(ba),e&&t(_n),e&&t(hl),e&&t(gn),e&&t(Re),e&&t(Mn),b(cl,e),e&&t(En),e&&t(le),b(ul),e&&t(Un),b(ml,e),e&&t(Tn),e&&t(ae),b(dl),e&&t(Jn),e&&t(wa),e&&t($n),b(vl,e),e&&t(kn),b(yl,e),e&&t(jn),e&&t(se),b(bl),e&&t(In),b(wl,e),e&&t(Sn),e&&t(oe),b(_l),e&&t(Cn),e&&t(gl),e&&t(Gn),e&&t(_a),e&&t(Bn),e&&t(El),e&&t(Zn),e&&t(j),e&&t(An),e&&t(ga),e&&t(Nn),e&&t(J),e&&t(Wn),e&&t(Ma),e&&t(Pn),e&&t(kl),e&&t(Rn),e&&t(Ea),e&&t(Qn),e&&t(Ua),e&&t(Yn),e&&t(Sl),e&&t(Fn),e&&t(He),e&&t(Vn),b(Cl,e),e&&t(zn),e&&t(Gl),e&&t(Xn),e&&t(Le),e&&t(Hn),b(Bl,e),e&&t(Ln),e&&t(re),b(Zl),e&&t(xn),e&&t(ne),b(Al),e&&t(Dn),e&&t(Ta),e&&t(qn),e&&t(qe),e&&t(On),e&&t(Oe),e&&t(Kn),e&&t(U),e&&t(ei),e&&t(I),e&&t(ti),e&&t(Nl),e&&t(li),e&&t(Ke),e&&t(ai),e&&t(et),e&&t(si),b(Wl,e),e&&t(oi),b(Pl,e),e&&t(ri),e&&t(tt),e&&t(ni),e&&t(S),e&&t(ii),e&&t(lt),e&&t(hi),b(Rl,e),e&&t(pi),e&&t(ja),e&&t(fi),e&&t(Ia),e&&t(ci),e&&t(Sa),e&&t(ui),e&&t(g),e&&t(mi),e&&t(ie),b(Fl),e&&t(di),b(Vl,e),e&&t(vi),b(zl,e),e&&t(yi),e&&t(he),b(Xl),e&&t(bi),e&&t($),e&&t(wi),e&&t(C),e&&t(_i),e&&t(Hl),e&&t(gi),e&&t(xl),e&&t(Mi),e&&t(Ga),e&&t(Ei),e&&t(G),e&&t(Ui),e&&t(Aa),e&&t(Ti),e&&t(pe),b(Ol)}}}const rw={local:"lets-train-our-first-robots",sections:[{local:"set-the-gpu",title:"Set the GPU \u{1F4AA}"},{local:"create-a-virtual-display",sections:[{local:"install-dependencies",title:"Install dependencies \u{1F53D}"}],title:"Create a virtual display \u{1F53D}"},{local:"import-the-packages",title:"Import the packages \u{1F4E6}"},{local:"environment-1-antbulletenvv0",sections:[{local:"create-the-antbulletenvv0",sections:[{local:"the-environment",title:"The environment \u{1F3AE}"}],title:"Create the AntBulletEnv-v0"},{local:"normalize-observation-and-rewards",sections:[{local:"solution",title:"Solution"}],title:"Normalize observation and rewards"},{local:"create-the-a2c-model",sections:[{local:"solution",title:"Solution"}],title:"Create the A2C Model \u{1F916}"},{local:"train-the-a2c-agent",title:"Train the A2C agent \u{1F3C3}"},{local:"evaluate-the-agent",title:"Evaluate the agent \u{1F4C8}"},{local:"publish-your-trained-model-on-the-hub",title:"Publish your trained model on the Hub \u{1F525}"}],title:"Environment 1: AntBulletEnv-v0 \u{1F578}"},{local:"take-a-coffee-break",title:"Take a coffee break \u2615"},{local:"environment-2-pandareachdensev2",sections:[{local:"solution-fill-the-todo",title:"Solution (fill the todo)"}],title:"Environment 2: PandaReachDense-v2 \u{1F9BE}"},{local:"some-additional-challenges",title:"Some additional challenges \u{1F3C6}"},{local:"keep-learning-stay-awesome",title:"Keep learning, stay awesome \u{1F917}"}],title:"Let's train our first robots \u{1F916}"};function nw(od){return aw(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class cw extends Ob{constructor(A){super();Kb(this,A,nw,ow,ew,{})}}export{cw as default,rw as metadata};
