import{S as fy,i as cy,s as gy,e as o,k as u,w as c,t as n,M as my,c as s,d as t,m as p,a as r,x as g,h as i,b as f,N as _,G as l,g as h,y as m,L as yy,q as y,o as d,B as v,v as dy}from"../../chunks/vendor-hf-doc-builder.js";import{I as w}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as k}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as vy}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function wy(Bf){let G,mr,S,K,Xl,Qe,Qn,Yl,qn,yr,qe,dr,ee,Kn,Fl,ei,ti,vr,te,li,Dl,oi,si,wr,C,le,Ql,Ke,ri,ql,ai,br,et,Kl,ni,ii,_r,rl,tt,al,zf,Er,lt,Of,Mr,ot,hi,eo,ui,Tr,oe,pi,to,fi,ci,$r,st,kr,A,se,lo,rt,gi,oo,mi,Ur,nl,re,yi,at,di,vi,nt,wi,Jr,Z,ae,so,it,bi,ro,_i,jr,il,ao,ht,Ei,Hr,J,Mi,no,Ti,$i,ut,ki,Ui,Ir,N,ne,io,pt,Ji,ho,ji,Gr,hl,Hi,Sr,j,ft,Ii,uo,Gi,Si,Ci,ul,po,Ai,Zi,Ni,ct,Wi,fo,Li,Ri,Cr,W,ie,co,gt,Pi,go,xi,Ar,pl,Bi,Zr,he,zi,mo,Oi,Vi,Nr,ue,Xi,yo,Yi,Fi,Wr,L,pe,vo,mt,Di,wo,Qi,Lr,yt,Vf,Rr,fl,bo,_o,qi,Pr,dt,Xf,xr,R,fe,Eo,vt,Ki,Mo,eh,Br,cl,To,th,zr,wt,Or,bt,Vr,P,ce,$o,_t,lh,gl,oh,ko,sh,Xr,ge,Uo,rh,ah,ml,nh,Jo,ih,Yr,Et,Fr,Mt,Dr,U,hh,Tt,uh,ph,jo,fh,ch,$t,gh,Qr,kt,qr,yl,mh,Kr,Ut,ea,x,me,Ho,Jt,yh,Io,dh,ta,B,ye,Go,jt,vh,So,wh,la,dl,bh,oa,H,Co,_h,Eh,Ao,Mh,Th,Zo,$h,sa,de,kh,No,Uh,Jh,ra,ve,Yf,aa,z,we,Wo,Ht,jh,Lo,Hh,na,be,Ff,ia,O,Ro,Ih,Gh,Po,Sh,Ch,ha,V,_e,xo,It,Ah,Bo,Zh,ua,Ee,Nh,zo,Wh,Lh,pa,I,Rh,Oo,Ph,xh,Vo,Bh,zh,fa,Me,Oh,Xo,Vh,Xh,ca,vl,Yh,ga,Te,Df,ma,E,$e,Yo,Fh,Dh,Fo,Qh,qh,Kh,ke,Do,eu,tu,Qo,lu,ou,su,Ue,qo,ru,au,Ko,nu,iu,hu,Je,es,uu,pu,ts,fu,cu,ya,X,je,ls,Gt,gu,os,mu,da,b,ss,wl,yu,rs,du,vu,as,St,wu,Ct,bu,_u,Eu,ns,is,Mu,Tu,hs,bl,$u,us,ku,Uu,ps,_l,Ju,fs,ju,Hu,cs,gs,Iu,va,At,wa,He,ms,ys,Gu,Su,ds,Zt,vs,Cu,Au,ws,Zu,ba,El,Nu,_a,Y,Ie,bs,Nt,Wu,_s,Lu,Ea,Wt,Ru,Es,Pu,Ma,Ge,Qf,Ta,Ml,xu,$a,M,Tl,Ms,Bu,zu,Ou,$l,Ts,Vu,Xu,Yu,kl,$s,Fu,Du,Qu,Ul,ks,qu,Ku,ka,Se,ep,Us,tp,lp,Ua,Jl,Lt,op,Js,sp,rp,Ja,Ce,ap,js,np,ip,ja,Rt,Ha,F,Ae,Hs,Pt,hp,Is,up,Ia,jl,Hl,pp,Gs,fp,Ga,Il,cp,Sa,xt,gp,Bt,mp,Ca,Gl,yp,Aa,Sl,Ze,dp,zt,vp,wp,Ss,bp,Za,Ot,qf,Na,Ne,Cs,_p,Ep,As,Mp,Wa,Vt,La,Xt,Tp,Zs,$p,Ra,We,kp,Ns,Up,Jp,Pa,Le,Kf,xa,Cl,jp,Ba,T,Al,Ws,Hp,Ip,Gp,Zl,Ls,Sp,Cp,Ap,Yt,Rs,Zp,Np,Ps,Wp,Lp,Nl,xs,Rp,Pp,za,Ft,Oa,Wl,xp,Va,Dt,Xa,Qt,Bp,Bs,zp,Ya,Re,ec,Fa,qt,Op,zs,Vp,Da,D,Pe,Os,Kt,Xp,Vs,Yp,Qa,Ll,Fp,qa,xe,Xs,Rl,Dp,el,Qp,qp,Ys,Fs,Kp,Ka,Be,tc,en,ze,Ds,Qs,ef,tf,qs,tl,lf,Ks,of,sf,tn,Oe,er,rf,af,Pl,nf,tr,hf,ln,ll,uf,lr,pf,on,xl,ff,sn,Q,cf,or,gf,mf,sr,yf,rn,Ve,lc,an,q,Xe,rr,ol,df,ar,vf,nn;return Qe=new w({}),qe=new vy({props:{classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/deep-rl-class/blob/master/notebooks/bonus-unit1/bonus-unit1.ipynb"}],askForHelpUrl:"http://hf.co/join/discord"}}),Ke=new w({}),st=new k({props:{code:"JTI1JTI1aHRtbCUwQSUzQ3ZpZGVvJTIwY29udHJvbHMlMjBhdXRvcGxheSUzRSUzQ3NvdXJjZSUyMHNyYyUzRCUyMmh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZkYXRhc2V0cyUyRmh1Z2dpbmdmYWNlLWRlZXAtcmwtY291cnNlJTJGY291cnNlLWltYWdlcyUyRnJlc29sdmUlMkZtYWluJTJGZW4lMkZub3RlYm9va3MlMkZ1bml0LWJvbnVzMSUyRmh1Z2d5Lm1wNCUyMiUyMHR5cGUlM0QlMjJ2aWRlbyUyRm1wNCUyMiUzRSUzQyUyRnZpZGVvJTNF",highlighted:`%%html
&lt;video controls autoplay&gt;&lt;source src=<span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/huggy.mp4&quot;</span> <span class="hljs-built_in">type</span>=<span class="hljs-string">&quot;video/mp4&quot;</span>&gt;&lt;/video&gt;`}}),rt=new w({}),it=new w({}),pt=new w({}),gt=new w({}),mt=new w({}),vt=new w({}),wt=new k({props:{code:"JTIzJTIwQ2xvbmUlMjB0aGUlMjByZXBvc2l0b3J5JTIwKGNhbiUyMHRha2UlMjAzbWluKSUwQWdpdCUyMGNsb25lJTIwLS1kZXB0aCUyMDElMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZVbml0eS1UZWNobm9sb2dpZXMlMkZtbC1hZ2VudHM=",highlighted:`<span class="hljs-comment"># Clone the repository (can take 3min)</span>
git <span class="hljs-built_in">clone</span> --depth 1 https://github.com/Unity-Technologies/ml-agents`}}),bt=new k({props:{code:"JTIzJTIwR28lMjBpbnNpZGUlMjB0aGUlMjByZXBvc2l0b3J5JTIwYW5kJTIwaW5zdGFsbCUyMHRoZSUyMHBhY2thZ2UlMjAoY2FuJTIwdGFrZSUyMDNtaW4pJTBBJTI1Y2QlMjBtbC1hZ2VudHMlMEFwaXAzJTIwaW5zdGFsbCUyMC1lJTIwLiUyRm1sLWFnZW50cy1lbnZzJTBBcGlwMyUyMGluc3RhbGwlMjAtZSUyMC4lMkZtbC1hZ2VudHM=",highlighted:`<span class="hljs-comment"># Go inside the repository and install the package (can take 3min)</span>
%<span class="hljs-built_in">cd</span> ml-agents
pip3 install -e ./ml-agents-envs
pip3 install -e ./ml-agents`}}),_t=new w({}),Et=new k({props:{code:"bWtkaXIlMjAuJTJGdHJhaW5lZC1lbnZzLWV4ZWN1dGFibGVzJTBBbWtkaXIlMjAuJTJGdHJhaW5lZC1lbnZzLWV4ZWN1dGFibGVzJTJGbGludXg=",highlighted:`<span class="hljs-built_in">mkdir</span> ./trained-envs-executables
<span class="hljs-built_in">mkdir</span> ./trained-envs-executables/linux`}}),Mt=new k({props:{code:"d2dldCUyMC0tbG9hZC1jb29raWVzJTIwJTJGdG1wJTJGY29va2llcy50eHQlMjAlMjJodHRwcyUzQSUyRiUyRmRvY3MuZ29vZ2xlLmNvbSUyRnVjJTNGZXhwb3J0JTNEZG93bmxvYWQlMjZjb25maXJtJTNEJTI0KHdnZXQlMjAtLXF1aWV0JTIwLS1zYXZlLWNvb2tpZXMlMjAlMkZ0bXAlMkZjb29raWVzLnR4dCUyMC0ta2VlcC1zZXNzaW9uLWNvb2tpZXMlMjAtLW5vLWNoZWNrLWNlcnRpZmljYXRlJTIwJ2h0dHBzJTNBJTJGJTJGZG9jcy5nb29nbGUuY29tJTJGdWMlM0ZleHBvcnQlM0Rkb3dubG9hZCUyNmlkJTNEMXp2M005NVpKVFdIVVZPV1Q2Y2txX2NtOThuZnQ4Z2RGJyUyMC1PLSUyMCU3QyUyMHNlZCUyMC1ybiUyMCdzJTJGLipjb25maXJtJTNEKCU1QjAtOUEtWmEtel8lNUQlMkIpLiolMkYlNUMxJTVDbiUyRnAnKSUyNmlkJTNEMXp2M005NVpKVFdIVVZPV1Q2Y2txX2NtOThuZnQ4Z2RGJTIyJTIwLU8lMjAuJTJGdHJhaW5lZC1lbnZzLWV4ZWN1dGFibGVzJTJGbGludXglMkZIdWdneS56aXAlMjAlMjYlMjYlMjBybSUyMC1yZiUyMCUyRnRtcCUyRmNvb2tpZXMudHh0",highlighted:'wget --load-cookies /tmp/cookies.txt <span class="hljs-string">&quot;https://docs.google.com/uc?export=download&amp;confirm=<span class="hljs-subst">$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate &#x27;https://docs.google.com/uc?export=download&amp;id=1zv3M95ZJTWHUVOWT6ckq_cm98nft8gdF&#x27; -O- | sed -rn &#x27;s/.*confirm=([0-9A-Za-z_]+)</span>.*/\\1\\n/p&#x27;)&amp;id=1zv3M95ZJTWHUVOWT6ckq_cm98nft8gdF&quot;</span> -O ./trained-envs-executables/linux/Huggy.zip &amp;&amp; <span class="hljs-built_in">rm</span> -rf /tmp/cookies.txt'}}),kt=new k({props:{code:"JTI1JTI1Y2FwdHVyZSUwQXVuemlwJTIwLWQlMjAuJTJGdHJhaW5lZC1lbnZzLWV4ZWN1dGFibGVzJTJGbGludXglMkYlMjAuJTJGdHJhaW5lZC1lbnZzLWV4ZWN1dGFibGVzJTJGbGludXglMkZIdWdneS56aXA=",highlighted:`%%capture
unzip -d ./trained-envs-executables/linux/ ./trained-envs-executables/linux/Huggy.zip`}}),Ut=new k({props:{code:"Y2htb2QlMjAtUiUyMDc1NSUyMC4lMkZ0cmFpbmVkLWVudnMtZXhlY3V0YWJsZXMlMkZsaW51eCUyRkh1Z2d5",highlighted:'<span class="hljs-built_in">chmod</span> -R 755 ./trained-envs-executables/linux/Huggy'}}),Jt=new w({}),jt=new w({}),Ht=new w({}),It=new w({}),Gt=new w({}),At=new k({props:{code:"YmVoYXZpb3JzJTNBJTBBJTIwJTIwSHVnZ3klM0ElMEElMjAlMjAlMjAlMjB0cmFpbmVyX3R5cGUlM0ElMjBwcG8lMEElMjAlMjAlMjAlMjBoeXBlcnBhcmFtZXRlcnMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjBiYXRjaF9zaXplJTNBJTIwMjA0OCUwQSUyMCUyMCUyMCUyMCUyMCUyMGJ1ZmZlcl9zaXplJTNBJTIwMjA0ODAlMEElMjAlMjAlMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNBJTIwMC4wMDAzJTBBJTIwJTIwJTIwJTIwJTIwJTIwYmV0YSUzQSUyMDAuMDA1JTBBJTIwJTIwJTIwJTIwJTIwJTIwZXBzaWxvbiUzQSUyMDAuMiUwQSUyMCUyMCUyMCUyMCUyMCUyMGxhbWJkJTNBJTIwMC45NSUwQSUyMCUyMCUyMCUyMCUyMCUyMG51bV9lcG9jaCUzQSUyMDMlMEElMjAlMjAlMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlX3NjaGVkdWxlJTNBJTIwbGluZWFyJTBBJTIwJTIwJTIwJTIwbmV0d29ya19zZXR0aW5ncyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMG5vcm1hbGl6ZSUzQSUyMHRydWUlMEElMjAlMjAlMjAlMjAlMjAlMjBoaWRkZW5fdW5pdHMlM0ElMjA1MTIlMEElMjAlMjAlMjAlMjAlMjAlMjBudW1fbGF5ZXJzJTNBJTIwMyUwQSUyMCUyMCUyMCUyMCUyMCUyMHZpc19lbmNvZGVfdHlwZSUzQSUyMHNpbXBsZSUwQSUyMCUyMCUyMCUyMHJld2FyZF9zaWduYWxzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwZXh0cmluc2ljJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ2FtbWElM0ElMjAwLjk5NSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0cmVuZ3RoJTNBJTIwMS4wJTBBJTIwJTIwJTIwJTIwY2hlY2twb2ludF9pbnRlcnZhbCUzQSUyMDIwMDAwMCUwQSUyMCUyMCUyMCUyMGtlZXBfY2hlY2twb2ludHMlM0ElMjAxNSUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzQSUyMDJlNiUwQSUyMCUyMCUyMCUyMHRpbWVfaG9yaXpvbiUzQSUyMDEwMDAlMEElMjAlMjAlMjAlMjBzdW1tYXJ5X2ZyZXElM0ElMjA1MDAwMA==",highlighted:`<span class="hljs-attr">behaviors:</span>
  <span class="hljs-attr">Huggy:</span>
    <span class="hljs-attr">trainer_type:</span> <span class="hljs-string">ppo</span>
    <span class="hljs-attr">hyperparameters:</span>
      <span class="hljs-attr">batch_size:</span> <span class="hljs-number">2048</span>
      <span class="hljs-attr">buffer_size:</span> <span class="hljs-number">20480</span>
      <span class="hljs-attr">learning_rate:</span> <span class="hljs-number">0.0003</span>
      <span class="hljs-attr">beta:</span> <span class="hljs-number">0.005</span>
      <span class="hljs-attr">epsilon:</span> <span class="hljs-number">0.2</span>
      <span class="hljs-attr">lambd:</span> <span class="hljs-number">0.95</span>
      <span class="hljs-attr">num_epoch:</span> <span class="hljs-number">3</span>
      <span class="hljs-attr">learning_rate_schedule:</span> <span class="hljs-string">linear</span>
    <span class="hljs-attr">network_settings:</span>
      <span class="hljs-attr">normalize:</span> <span class="hljs-literal">true</span>
      <span class="hljs-attr">hidden_units:</span> <span class="hljs-number">512</span>
      <span class="hljs-attr">num_layers:</span> <span class="hljs-number">3</span>
      <span class="hljs-attr">vis_encode_type:</span> <span class="hljs-string">simple</span>
    <span class="hljs-attr">reward_signals:</span>
      <span class="hljs-attr">extrinsic:</span>
        <span class="hljs-attr">gamma:</span> <span class="hljs-number">0.995</span>
        <span class="hljs-attr">strength:</span> <span class="hljs-number">1.0</span>
    <span class="hljs-attr">checkpoint_interval:</span> <span class="hljs-number">200000</span>
    <span class="hljs-attr">keep_checkpoints:</span> <span class="hljs-number">15</span>
    <span class="hljs-attr">max_steps:</span> <span class="hljs-number">2e6</span>
    <span class="hljs-attr">time_horizon:</span> <span class="hljs-number">1000</span>
    <span class="hljs-attr">summary_freq:</span> <span class="hljs-number">50000</span>`}}),Nt=new w({}),Rt=new k({props:{code:"bWxhZ2VudHMtbGVhcm4lMjAuJTJGY29uZmlnJTJGcHBvJTJGSHVnZ3kueWFtbCUyMC0tZW52JTNELiUyRnRyYWluZWQtZW52cy1leGVjdXRhYmxlcyUyRmxpbnV4JTJGSHVnZ3klMkZIdWdneSUyMC0tcnVuLWlkJTNEJTIySHVnZ3klMjIlMjAtLW5vLWdyYXBoaWNz",highlighted:'mlagents-learn ./config/ppo/Huggy.yaml --<span class="hljs-built_in">env</span>=./trained-envs-executables/linux/Huggy/Huggy --run-id=<span class="hljs-string">&quot;Huggy&quot;</span> --no-graphics'}}),Pt=new w({}),Vt=new k({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),Ft=new k({props:{code:"bWxhZ2VudHMtcHVzaC10by1oZiUyMC0tcnVuLWlkJTNEJTIySHVnZ3lUcmFpbmluZyUyMiUyMC0tbG9jYWwtZGlyJTNEJTIyLiUyRnJlc3VsdHMlMkZIdWdneSUyMiUyMC0tcmVwby1pZCUzRCUyMlRob21hc1NpbW9uaW5pJTJGcHBvLUh1Z2d5JTIyJTIwLS1jb21taXQtbWVzc2FnZSUzRCUyMkh1Z2d5JTIy",highlighted:'mlagents-push-to-hf --run-id=<span class="hljs-string">&quot;HuggyTraining&quot;</span> --local-dir=<span class="hljs-string">&quot;./results/Huggy&quot;</span> --repo-id=<span class="hljs-string">&quot;ThomasSimonini/ppo-Huggy&quot;</span> --commit-message=<span class="hljs-string">&quot;Huggy&quot;</span>'}}),Dt=new k({props:{code:"WW91ciUyMG1vZGVsJTIwaXMlMjBwdXNoZWQlMjB0byUyMHRoZSUyMGh1Yi4lMjBZb3UlMjBjYW4lMjB2aWV3JTIweW91ciUyMG1vZGVsJTIwaGVyZSUzQSUyMGh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZUaG9tYXNTaW1vbmluaSUyRnBwby1IdWdneQ==",highlighted:'Your model is pushed to the hub. You can view your model here: https:<span class="hljs-regexp">//</span>huggingface.co<span class="hljs-regexp">/ThomasSimonini/</span>ppo-Huggy'}}),Kt=new w({}),ol=new w({}),{c(){G=o("meta"),mr=u(),S=o("h1"),K=o("a"),Xl=o("span"),c(Qe.$$.fragment),Qn=u(),Yl=o("span"),qn=n("Let's train and play with Huggy \u{1F436}"),yr=u(),c(qe.$$.fragment),dr=u(),ee=o("p"),Kn=n("We strongly "),Fl=o("strong"),ei=n("recommend students use Google Colab for the hands-on exercises"),ti=n(" instead of running them on their personal computers."),vr=u(),te=o("p"),li=n("By using Google Colab, "),Dl=o("strong"),oi=n("you can focus on learning and experimenting without worrying about the technical aspects"),si=n(" of setting up your environments."),wr=u(),C=o("h2"),le=o("a"),Ql=o("span"),c(Ke.$$.fragment),ri=u(),ql=o("span"),ai=n("Let's train Huggy \u{1F436}"),br=u(),et=o("p"),Kl=o("strong"),ni=n("To start to train Huggy, click on Open In Colab button"),ii=n(" \u{1F447} :"),_r=u(),rl=o("p"),tt=o("a"),al=o("img"),Er=u(),lt=o("img"),Mr=u(),ot=o("p"),hi=n("In this notebook, we\u2019ll reinforce what we learned in the first Unit by "),eo=o("strong"),ui=n("teaching Huggy the Dog to fetch the stick and then play with it directly in your browser"),Tr=u(),oe=o("p"),pi=n("\u2B07\uFE0F Here is an example of what "),to=o("strong"),fi=n("you will achieve at the end of the unit."),ci=n(" \u2B07\uFE0F (launch \u25B6 to see)"),$r=u(),c(st.$$.fragment),kr=u(),A=o("h3"),se=o("a"),lo=o("span"),c(rt.$$.fragment),gi=u(),oo=o("span"),mi=n("The environment \u{1F3AE}"),Ur=u(),nl=o("ul"),re=o("li"),yi=n("Huggy the Dog, an environment created by "),at=o("a"),di=n("Thomas Simonini"),vi=n(" based on "),nt=o("a"),wi=n("Puppo The Corgi"),Jr=u(),Z=o("h3"),ae=o("a"),so=o("span"),c(it.$$.fragment),bi=u(),ro=o("span"),_i=n("The library used \u{1F4DA}"),jr=u(),il=o("ul"),ao=o("li"),ht=o("a"),Ei=n("MLAgents"),Hr=u(),J=o("p"),Mi=n("We\u2019re constantly trying to improve our tutorials, so "),no=o("strong"),Ti=n("if you find some issues in this notebook"),$i=n(", please "),ut=o("a"),ki=n("open an issue on the Github Repo"),Ui=n("."),Ir=u(),N=o("h2"),ne=o("a"),io=o("span"),c(pt.$$.fragment),Ji=u(),ho=o("span"),ji=n("Objectives of this notebook \u{1F3C6}"),Gr=u(),hl=o("p"),Hi=n("At the end of the notebook, you will:"),Sr=u(),j=o("ul"),ft=o("li"),Ii=n("Understand "),uo=o("strong"),Gi=n("the state space, action space, and reward function used to train Huggy"),Si=n("."),Ci=u(),ul=o("li"),po=o("strong"),Ai=n("Train your own Huggy"),Zi=n(" to fetch the stick."),Ni=u(),ct=o("li"),Wi=n("Be able to play "),fo=o("strong"),Li=n("with your trained Huggy directly in your browser"),Ri=n("."),Cr=u(),W=o("h2"),ie=o("a"),co=o("span"),c(gt.$$.fragment),Pi=u(),go=o("span"),xi=n("Prerequisites \u{1F3D7}\uFE0F"),Ar=u(),pl=o("p"),Bi=n("Before diving into the notebook, you need to:"),Zr=u(),he=o("p"),zi=n("\u{1F532} \u{1F4DA} "),mo=o("strong"),Oi=n("Develop an understanding of the foundations of Reinforcement learning"),Vi=n(" (MC, TD, Rewards hypothesis\u2026) by doing Unit 1"),Nr=u(),ue=o("p"),Xi=n("\u{1F532} \u{1F4DA} "),yo=o("strong"),Yi=n("Read the introduction to Huggy"),Fi=n(" by doing Bonus Unit 1"),Wr=u(),L=o("h2"),pe=o("a"),vo=o("span"),c(mt.$$.fragment),Di=u(),wo=o("span"),Qi=n("Set the GPU \u{1F4AA}"),Lr=n(`

- To **accelerate the agent's training, we'll use a GPU**. To do that, go to \`Runtime > Change Runtime type\`
`),yt=o("img"),Rr=u(),fl=o("ul"),bo=o("li"),_o=o("code"),qi=n("Hardware Accelerator > GPU"),Pr=u(),dt=o("img"),xr=u(),R=o("h2"),fe=o("a"),Eo=o("span"),c(vt.$$.fragment),Ki=u(),Mo=o("span"),eh=n("Clone the repository and install the dependencies \u{1F53D}"),Br=u(),cl=o("ul"),To=o("li"),th=n("We need to clone the repository, that contains ML-Agents."),zr=u(),c(wt.$$.fragment),Or=u(),c(bt.$$.fragment),Vr=u(),P=o("h2"),ce=o("a"),$o=o("span"),c(_t.$$.fragment),lh=u(),gl=o("span"),oh=n("Download and move the environment zip file in "),ko=o("code"),sh=n("./trained-envs-executables/linux/"),Xr=u(),ge=o("ul"),Uo=o("li"),rh=n("Our environment executable is in a zip file."),ah=u(),ml=o("li"),nh=n("We need to download it and place it to "),Jo=o("code"),ih=n("./trained-envs-executables/linux/"),Yr=u(),c(Et.$$.fragment),Fr=u(),c(Mt.$$.fragment),Dr=u(),U=o("p"),hh=n("Download the file Huggy.zip from "),Tt=o("a"),uh=n("https://drive.google.com/uc?export=download&id=1zv3M95ZJTWHUVOWT6ckq_cm98nft8gdF"),ph=n(" using "),jo=o("code"),fh=n("wget"),ch=n(". Check out the full solution to download large files from GDrive "),$t=o("a"),gh=n("here"),Qr=u(),c(kt.$$.fragment),qr=u(),yl=o("p"),mh=n("Make sure your file is accessible"),Kr=u(),c(Ut.$$.fragment),ea=u(),x=o("h2"),me=o("a"),Ho=o("span"),c(Jt.$$.fragment),yh=u(),Io=o("span"),dh=n("Let's recap how this environment works"),ta=u(),B=o("h3"),ye=o("a"),Go=o("span"),c(jt.$$.fragment),vh=u(),So=o("span"),wh=n("The State Space: what Huggy perceives."),la=u(),dl=o("p"),bh=n("Huggy doesn\u2019t \u201Csee\u201D his environment. Instead, we provide him information about the environment:"),oa=u(),H=o("ul"),Co=o("li"),_h=n("The target (stick) position"),Eh=u(),Ao=o("li"),Mh=n("The relative position between himself and the target"),Th=u(),Zo=o("li"),$h=n("The orientation of his legs."),sa=u(),de=o("p"),kh=n("Given all this information, Huggy "),No=o("strong"),Uh=n("can decide which action to take next to fulfill his goal"),Jh=n("."),ra=u(),ve=o("img"),aa=u(),z=o("h3"),we=o("a"),Wo=o("span"),c(Ht.$$.fragment),jh=u(),Lo=o("span"),Hh=n("The Action Space: what moves Huggy can do"),na=u(),be=o("img"),ia=u(),O=o("p"),Ro=o("strong"),Ih=n("Joint motors drive huggy legs"),Gh=n(". This means that to get the target, Huggy needs to "),Po=o("strong"),Sh=n("learn to rotate the joint motors of each of his legs correctly so he can move"),Ch=n("."),ha=u(),V=o("h3"),_e=o("a"),xo=o("span"),c(It.$$.fragment),Ah=u(),Bo=o("span"),Zh=n("The Reward Function"),ua=u(),Ee=o("p"),Nh=n("The reward function is designed so that "),zo=o("strong"),Wh=n("Huggy will fulfill his goal"),Lh=n(" : fetch the stick."),pa=u(),I=o("p"),Rh=n("Remember that one of the foundations of Reinforcement Learning is the "),Oo=o("em"),Ph=n("reward hypothesis"),xh=n(": a goal can be described as the "),Vo=o("strong"),Bh=n("maximization of the expected cumulative reward"),zh=n("."),fa=u(),Me=o("p"),Oh=n("Here, our goal is that Huggy "),Xo=o("strong"),Vh=n("goes towards the stick but without spinning too much"),Xh=n(". Hence, our reward function must translate this goal."),ca=u(),vl=o("p"),Yh=n("Our reward function:"),ga=u(),Te=o("img"),ma=u(),E=o("ul"),$e=o("li"),Yo=o("em"),Fh=n("Orientation bonus"),Dh=n(": we "),Fo=o("strong"),Qh=n("reward him for getting close to the target"),qh=n("."),Kh=u(),ke=o("li"),Do=o("em"),eu=n("Time penalty"),tu=n(": a fixed-time penalty given at every action to "),Qo=o("strong"),lu=n("force him to get to the stick as fast as possible"),ou=n("."),su=u(),Ue=o("li"),qo=o("em"),ru=n("Rotation penalty"),au=n(": we penalize Huggy if "),Ko=o("strong"),nu=n("he spins too much and turns too quickly"),iu=n("."),hu=u(),Je=o("li"),es=o("em"),uu=n("Getting to the target reward"),pu=n(": we reward Huggy for "),ts=o("strong"),fu=n("reaching the target"),cu=n("."),ya=u(),X=o("h2"),je=o("a"),ls=o("span"),c(Gt.$$.fragment),gu=u(),os=o("span"),mu=n("Check the Huggy config file"),da=u(),b=o("ul"),ss=o("li"),wl=o("p"),yu=n("In ML-Agents, you define the "),rs=o("strong"),du=n("training hyperparameters in config.yaml files."),vu=u(),as=o("li"),St=o("p"),wu=n("For the scope of this notebook, we\u2019re not going to modify the hyperparameters, but if you want to try as an experiment, Unity provides very "),Ct=o("a"),bu=n("good documentation explaining each of them here"),_u=n("."),Eu=u(),ns=o("li"),is=o("p"),Mu=n("We need to create a config file for Huggy."),Tu=u(),hs=o("li"),bl=o("p"),$u=n("Go to "),us=o("code"),ku=n("/content/ml-agents/config/ppo"),Uu=u(),ps=o("li"),_l=o("p"),Ju=n("Create a new file called "),fs=o("code"),ju=n("Huggy.yaml"),Hu=u(),cs=o("li"),gs=o("p"),Iu=n("Copy and paste the content below \u{1F53D}"),va=u(),c(At.$$.fragment),wa=u(),He=o("ul"),ms=o("li"),ys=o("p"),Gu=n("Don\u2019t forget to save the file!"),Su=u(),ds=o("li"),Zt=o("p"),vs=o("strong"),Cu=n("In the case you want to modify the hyperparameters"),Au=n(", in Google Colab notebook, you can click here to open the config.yaml: "),ws=o("code"),Zu=n("/content/ml-agents/config/ppo/Huggy.yaml"),ba=u(),El=o("p"),Nu=n("We\u2019re now ready to train our agent \u{1F525}."),_a=u(),Y=o("h2"),Ie=o("a"),bs=o("span"),c(Nt.$$.fragment),Wu=u(),_s=o("span"),Lu=n("Train our agent"),Ea=u(),Wt=o("p"),Ru=n("To train our agent, we just need to "),Es=o("strong"),Pu=n("launch mlagents-learn and select the executable containing the environment."),Ma=u(),Ge=o("img"),Ta=u(),Ml=o("p"),xu=n("With ML Agents, we run a training script. We define four parameters:"),$a=u(),M=o("ol"),Tl=o("li"),Ms=o("code"),Bu=n("mlagents-learn <config>"),zu=n(": the path where the hyperparameter config file is."),Ou=u(),$l=o("li"),Ts=o("code"),Vu=n("--env"),Xu=n(": where the environment executable is."),Yu=u(),kl=o("li"),$s=o("code"),Fu=n("--run_id"),Du=n(": the name you want to give to your training run id."),Qu=u(),Ul=o("li"),ks=o("code"),qu=n("--no-graphics"),Ku=n(": to not launch the visualization during the training."),ka=u(),Se=o("p"),ep=n("Train the model and use the "),Us=o("code"),tp=n("--resume"),lp=n(" flag to continue training in case of interruption."),Ua=u(),Jl=o("blockquote"),Lt=o("p"),op=n("It will fail first time when you use "),Js=o("code"),sp=n("--resume"),rp=n(", try running the block again to bypass the error."),Ja=u(),Ce=o("p"),ap=n("The training will take 30 to 45min depending on your machine (don\u2019t forget to "),js=o("strong"),np=n("set up a GPU"),ip=n("), go take a \u2615\uFE0F you deserve it \u{1F917}."),ja=u(),c(Rt.$$.fragment),Ha=u(),F=o("h2"),Ae=o("a"),Hs=o("span"),c(Pt.$$.fragment),hp=u(),Is=o("span"),up=n("Push the agent to the \u{1F917} Hub"),Ia=u(),jl=o("ul"),Hl=o("li"),pp=n("Now that we trained our agent, we\u2019re "),Gs=o("strong"),fp=n("ready to push it to the Hub to be able to play with Huggy on your browser\u{1F525}."),Ga=u(),Il=o("p"),cp=n("To be able to share your model with the community there are three more steps to follow:"),Sa=u(),xt=o("p"),gp=n("1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),Bt=o("a"),mp=n("https://huggingface.co/join"),Ca=u(),Gl=o("p"),yp=n("2\uFE0F\u20E3 Sign in and then get your token from the Hugging Face website."),Aa=u(),Sl=o("ul"),Ze=o("li"),dp=n("Create a new token ("),zt=o("a"),vp=n("https://huggingface.co/settings/tokens"),wp=n(") "),Ss=o("strong"),bp=n("with write role"),Za=u(),Ot=o("img"),Na=u(),Ne=o("ul"),Cs=o("li"),_p=n("Copy the token"),Ep=u(),As=o("li"),Mp=n("Run the cell below and paste the token"),Wa=u(),c(Vt.$$.fragment),La=u(),Xt=o("p"),Tp=n("If you don\u2019t want to use Google Colab or a Jupyter Notebook, you need to use this command instead: "),Zs=o("code"),$p=n("huggingface-cli login"),Ra=u(),We=o("p"),kp=n("Then, we simply need to run "),Ns=o("code"),Up=n("mlagents-push-to-hf"),Jp=n("."),Pa=u(),Le=o("img"),xa=u(),Cl=o("p"),jp=n("And we define 4 parameters:"),Ba=u(),T=o("ol"),Al=o("li"),Ws=o("code"),Hp=n("--run-id"),Ip=n(": the name of the training run id."),Gp=u(),Zl=o("li"),Ls=o("code"),Sp=n("--local-dir"),Cp=n(": where the agent was saved, it\u2019s results/<run_id name>, so in my case results/First Training."),Ap=u(),Yt=o("li"),Rs=o("code"),Zp=n("--repo-id"),Np=n(`: the name of the Hugging Face repo you want to create or update. It\u2019s always <your huggingface username>/<the repo name>
If the repo does not exist `),Ps=o("strong"),Wp=n("it will be created automatically"),Lp=u(),Nl=o("li"),xs=o("code"),Rp=n("--commit-message"),Pp=n(": since HF repos are git repositories you need to give a commit message."),za=u(),c(Ft.$$.fragment),Oa=u(),Wl=o("p"),xp=n("If everything worked you should see this at the end of the process (but with a different url \u{1F606}) :"),Va=u(),c(Dt.$$.fragment),Xa=u(),Qt=o("p"),Bp=n("It\u2019s the link to your model repository. The repository contains a model card that explains how to use the model, your Tensorboard logs and your config file. "),Bs=o("strong"),zp=n("What\u2019s awesome is that it\u2019s a git repository, which means you can have different commits, update your repository with a new push, open Pull Requests, etc."),Ya=u(),Re=o("img"),Fa=u(),qt=o("p"),Op=n("But now comes the best part: "),zs=o("strong"),Vp=n("being able to play with Huggy online \u{1F440}."),Da=u(),D=o("h2"),Pe=o("a"),Os=o("span"),c(Kt.$$.fragment),Xp=u(),Vs=o("span"),Yp=n("Play with your Huggy \u{1F415}"),Qa=u(),Ll=o("p"),Fp=n("This step is the simplest:"),qa=u(),xe=o("ul"),Xs=o("li"),Rl=o("p"),Dp=n("Open the Huggy game in your browser: "),el=o("a"),Qp=n("https://huggingface.co/spaces/ThomasSimonini/Huggy"),qp=u(),Ys=o("li"),Fs=o("p"),Kp=n("Click on Play with my Huggy model"),Ka=u(),Be=o("img"),en=u(),ze=o("ol"),Ds=o("li"),Qs=o("p"),ef=n("In step 1, choose your model repository, which is the model id (in my case ThomasSimonini/ppo-Huggy)."),tf=u(),qs=o("li"),tl=o("p"),lf=n("In step 2, "),Ks=o("strong"),of=n("choose which model you want to replay"),sf=n(":"),tn=u(),Oe=o("ul"),er=o("li"),rf=n("I have multiple ones, since we saved a model every 500000 timesteps."),af=u(),Pl=o("li"),nf=n("But since I want the most recent one, I choose "),tr=o("code"),hf=n("Huggy.onnx"),ln=u(),ll=o("p"),uf=n("\u{1F449} It\u2019s good "),lr=o("strong"),pf=n("to try with different models steps to see the improvement of the agent."),on=u(),xl=o("p"),ff=n("Congrats on finishing this bonus unit!"),sn=u(),Q=o("p"),cf=n("You can now sit and enjoy playing with your Huggy \u{1F436}. And don\u2019t "),or=o("strong"),gf=n("forget to spread the love by sharing Huggy with your friends \u{1F917}"),mf=n(". And if you share about it on social media, "),sr=o("strong"),yf=n("please tag us @huggingface and me @simoninithomas"),rn=u(),Ve=o("img"),an=u(),q=o("h2"),Xe=o("a"),rr=o("span"),c(ol.$$.fragment),df=u(),ar=o("span"),vf=n("Keep Learning, Stay awesome \u{1F917}"),this.h()},l(e){const a=my('[data-svelte="svelte-1phssyn"]',document.head);G=s(a,"META",{name:!0,content:!0}),a.forEach(t),mr=p(e),S=s(e,"H1",{class:!0});var hn=r(S);K=s(hn,"A",{id:!0,class:!0,href:!0});var oc=r(K);Xl=s(oc,"SPAN",{});var sc=r(Xl);g(Qe.$$.fragment,sc),sc.forEach(t),oc.forEach(t),Qn=p(hn),Yl=s(hn,"SPAN",{});var rc=r(Yl);qn=i(rc,"Let's train and play with Huggy \u{1F436}"),rc.forEach(t),hn.forEach(t),yr=p(e),g(qe.$$.fragment,e),dr=p(e),ee=s(e,"P",{});var un=r(ee);Kn=i(un,"We strongly "),Fl=s(un,"STRONG",{});var ac=r(Fl);ei=i(ac,"recommend students use Google Colab for the hands-on exercises"),ac.forEach(t),ti=i(un," instead of running them on their personal computers."),un.forEach(t),vr=p(e),te=s(e,"P",{});var pn=r(te);li=i(pn,"By using Google Colab, "),Dl=s(pn,"STRONG",{});var nc=r(Dl);oi=i(nc,"you can focus on learning and experimenting without worrying about the technical aspects"),nc.forEach(t),si=i(pn," of setting up your environments."),pn.forEach(t),wr=p(e),C=s(e,"H2",{class:!0});var fn=r(C);le=s(fn,"A",{id:!0,class:!0,href:!0});var ic=r(le);Ql=s(ic,"SPAN",{});var hc=r(Ql);g(Ke.$$.fragment,hc),hc.forEach(t),ic.forEach(t),ri=p(fn),ql=s(fn,"SPAN",{});var uc=r(ql);ai=i(uc,"Let's train Huggy \u{1F436}"),uc.forEach(t),fn.forEach(t),br=p(e),et=s(e,"P",{});var wf=r(et);Kl=s(wf,"STRONG",{});var pc=r(Kl);ni=i(pc,"To start to train Huggy, click on Open In Colab button"),pc.forEach(t),ii=i(wf," \u{1F447} :"),wf.forEach(t),_r=p(e),rl=s(e,"P",{});var fc=r(rl);tt=s(fc,"A",{href:!0,rel:!0});var cc=r(tt);al=s(cc,"IMG",{src:!0,alt:!0}),cc.forEach(t),fc.forEach(t),Er=p(e),lt=s(e,"IMG",{src:!0,alt:!0}),Mr=p(e),ot=s(e,"P",{});var bf=r(ot);hi=i(bf,"In this notebook, we\u2019ll reinforce what we learned in the first Unit by "),eo=s(bf,"STRONG",{});var gc=r(eo);ui=i(gc,"teaching Huggy the Dog to fetch the stick and then play with it directly in your browser"),gc.forEach(t),bf.forEach(t),Tr=p(e),oe=s(e,"P",{});var cn=r(oe);pi=i(cn,"\u2B07\uFE0F Here is an example of what "),to=s(cn,"STRONG",{});var mc=r(to);fi=i(mc,"you will achieve at the end of the unit."),mc.forEach(t),ci=i(cn," \u2B07\uFE0F (launch \u25B6 to see)"),cn.forEach(t),$r=p(e),g(st.$$.fragment,e),kr=p(e),A=s(e,"H3",{class:!0});var gn=r(A);se=s(gn,"A",{id:!0,class:!0,href:!0});var yc=r(se);lo=s(yc,"SPAN",{});var dc=r(lo);g(rt.$$.fragment,dc),dc.forEach(t),yc.forEach(t),gi=p(gn),oo=s(gn,"SPAN",{});var vc=r(oo);mi=i(vc,"The environment \u{1F3AE}"),vc.forEach(t),gn.forEach(t),Ur=p(e),nl=s(e,"UL",{});var wc=r(nl);re=s(wc,"LI",{});var nr=r(re);yi=i(nr,"Huggy the Dog, an environment created by "),at=s(nr,"A",{href:!0,rel:!0});var bc=r(at);di=i(bc,"Thomas Simonini"),bc.forEach(t),vi=i(nr," based on "),nt=s(nr,"A",{href:!0,rel:!0});var _c=r(nt);wi=i(_c,"Puppo The Corgi"),_c.forEach(t),nr.forEach(t),wc.forEach(t),Jr=p(e),Z=s(e,"H3",{class:!0});var mn=r(Z);ae=s(mn,"A",{id:!0,class:!0,href:!0});var Ec=r(ae);so=s(Ec,"SPAN",{});var Mc=r(so);g(it.$$.fragment,Mc),Mc.forEach(t),Ec.forEach(t),bi=p(mn),ro=s(mn,"SPAN",{});var Tc=r(ro);_i=i(Tc,"The library used \u{1F4DA}"),Tc.forEach(t),mn.forEach(t),jr=p(e),il=s(e,"UL",{});var $c=r(il);ao=s($c,"LI",{});var kc=r(ao);ht=s(kc,"A",{href:!0,rel:!0});var Uc=r(ht);Ei=i(Uc,"MLAgents"),Uc.forEach(t),kc.forEach(t),$c.forEach(t),Hr=p(e),J=s(e,"P",{});var Bl=r(J);Mi=i(Bl,"We\u2019re constantly trying to improve our tutorials, so "),no=s(Bl,"STRONG",{});var Jc=r(no);Ti=i(Jc,"if you find some issues in this notebook"),Jc.forEach(t),$i=i(Bl,", please "),ut=s(Bl,"A",{href:!0,rel:!0});var jc=r(ut);ki=i(jc,"open an issue on the Github Repo"),jc.forEach(t),Ui=i(Bl,"."),Bl.forEach(t),Ir=p(e),N=s(e,"H2",{class:!0});var yn=r(N);ne=s(yn,"A",{id:!0,class:!0,href:!0});var Hc=r(ne);io=s(Hc,"SPAN",{});var Ic=r(io);g(pt.$$.fragment,Ic),Ic.forEach(t),Hc.forEach(t),Ji=p(yn),ho=s(yn,"SPAN",{});var Gc=r(ho);ji=i(Gc,"Objectives of this notebook \u{1F3C6}"),Gc.forEach(t),yn.forEach(t),Gr=p(e),hl=s(e,"P",{});var Sc=r(hl);Hi=i(Sc,"At the end of the notebook, you will:"),Sc.forEach(t),Sr=p(e),j=s(e,"UL",{});var zl=r(j);ft=s(zl,"LI",{});var dn=r(ft);Ii=i(dn,"Understand "),uo=s(dn,"STRONG",{});var Cc=r(uo);Gi=i(Cc,"the state space, action space, and reward function used to train Huggy"),Cc.forEach(t),Si=i(dn,"."),dn.forEach(t),Ci=p(zl),ul=s(zl,"LI",{});var _f=r(ul);po=s(_f,"STRONG",{});var Ac=r(po);Ai=i(Ac,"Train your own Huggy"),Ac.forEach(t),Zi=i(_f," to fetch the stick."),_f.forEach(t),Ni=p(zl),ct=s(zl,"LI",{});var vn=r(ct);Wi=i(vn,"Be able to play "),fo=s(vn,"STRONG",{});var Zc=r(fo);Li=i(Zc,"with your trained Huggy directly in your browser"),Zc.forEach(t),Ri=i(vn,"."),vn.forEach(t),zl.forEach(t),Cr=p(e),W=s(e,"H2",{class:!0});var wn=r(W);ie=s(wn,"A",{id:!0,class:!0,href:!0});var Nc=r(ie);co=s(Nc,"SPAN",{});var Wc=r(co);g(gt.$$.fragment,Wc),Wc.forEach(t),Nc.forEach(t),Pi=p(wn),go=s(wn,"SPAN",{});var Lc=r(go);xi=i(Lc,"Prerequisites \u{1F3D7}\uFE0F"),Lc.forEach(t),wn.forEach(t),Ar=p(e),pl=s(e,"P",{});var Rc=r(pl);Bi=i(Rc,"Before diving into the notebook, you need to:"),Rc.forEach(t),Zr=p(e),he=s(e,"P",{});var bn=r(he);zi=i(bn,"\u{1F532} \u{1F4DA} "),mo=s(bn,"STRONG",{});var Pc=r(mo);Oi=i(Pc,"Develop an understanding of the foundations of Reinforcement learning"),Pc.forEach(t),Vi=i(bn," (MC, TD, Rewards hypothesis\u2026) by doing Unit 1"),bn.forEach(t),Nr=p(e),ue=s(e,"P",{});var _n=r(ue);Xi=i(_n,"\u{1F532} \u{1F4DA} "),yo=s(_n,"STRONG",{});var xc=r(yo);Yi=i(xc,"Read the introduction to Huggy"),xc.forEach(t),Fi=i(_n," by doing Bonus Unit 1"),_n.forEach(t),Wr=p(e),L=s(e,"H2",{class:!0});var En=r(L);pe=s(En,"A",{id:!0,class:!0,href:!0});var Bc=r(pe);vo=s(Bc,"SPAN",{});var zc=r(vo);g(mt.$$.fragment,zc),zc.forEach(t),Bc.forEach(t),Di=p(En),wo=s(En,"SPAN",{});var Oc=r(wo);Qi=i(Oc,"Set the GPU \u{1F4AA}"),Oc.forEach(t),En.forEach(t),Lr=i(e,`

- To **accelerate the agent's training, we'll use a GPU**. To do that, go to \`Runtime > Change Runtime type\`
`),yt=s(e,"IMG",{src:!0,alt:!0}),Rr=p(e),fl=s(e,"UL",{});var Vc=r(fl);bo=s(Vc,"LI",{});var Xc=r(bo);_o=s(Xc,"CODE",{});var Yc=r(_o);qi=i(Yc,"Hardware Accelerator > GPU"),Yc.forEach(t),Xc.forEach(t),Vc.forEach(t),Pr=p(e),dt=s(e,"IMG",{src:!0,alt:!0}),xr=p(e),R=s(e,"H2",{class:!0});var Mn=r(R);fe=s(Mn,"A",{id:!0,class:!0,href:!0});var Fc=r(fe);Eo=s(Fc,"SPAN",{});var Dc=r(Eo);g(vt.$$.fragment,Dc),Dc.forEach(t),Fc.forEach(t),Ki=p(Mn),Mo=s(Mn,"SPAN",{});var Qc=r(Mo);eh=i(Qc,"Clone the repository and install the dependencies \u{1F53D}"),Qc.forEach(t),Mn.forEach(t),Br=p(e),cl=s(e,"UL",{});var qc=r(cl);To=s(qc,"LI",{});var Kc=r(To);th=i(Kc,"We need to clone the repository, that contains ML-Agents."),Kc.forEach(t),qc.forEach(t),zr=p(e),g(wt.$$.fragment,e),Or=p(e),g(bt.$$.fragment,e),Vr=p(e),P=s(e,"H2",{class:!0});var Tn=r(P);ce=s(Tn,"A",{id:!0,class:!0,href:!0});var eg=r(ce);$o=s(eg,"SPAN",{});var tg=r($o);g(_t.$$.fragment,tg),tg.forEach(t),eg.forEach(t),lh=p(Tn),gl=s(Tn,"SPAN",{});var Ef=r(gl);oh=i(Ef,"Download and move the environment zip file in "),ko=s(Ef,"CODE",{});var lg=r(ko);sh=i(lg,"./trained-envs-executables/linux/"),lg.forEach(t),Ef.forEach(t),Tn.forEach(t),Xr=p(e),ge=s(e,"UL",{});var $n=r(ge);Uo=s($n,"LI",{});var og=r(Uo);rh=i(og,"Our environment executable is in a zip file."),og.forEach(t),ah=p($n),ml=s($n,"LI",{});var Mf=r(ml);nh=i(Mf,"We need to download it and place it to "),Jo=s(Mf,"CODE",{});var sg=r(Jo);ih=i(sg,"./trained-envs-executables/linux/"),sg.forEach(t),Mf.forEach(t),$n.forEach(t),Yr=p(e),g(Et.$$.fragment,e),Fr=p(e),g(Mt.$$.fragment,e),Dr=p(e),U=s(e,"P",{});var sl=r(U);hh=i(sl,"Download the file Huggy.zip from "),Tt=s(sl,"A",{href:!0,rel:!0});var rg=r(Tt);uh=i(rg,"https://drive.google.com/uc?export=download&id=1zv3M95ZJTWHUVOWT6ckq_cm98nft8gdF"),rg.forEach(t),ph=i(sl," using "),jo=s(sl,"CODE",{});var ag=r(jo);fh=i(ag,"wget"),ag.forEach(t),ch=i(sl,". Check out the full solution to download large files from GDrive "),$t=s(sl,"A",{href:!0,rel:!0});var ng=r($t);gh=i(ng,"here"),ng.forEach(t),sl.forEach(t),Qr=p(e),g(kt.$$.fragment,e),qr=p(e),yl=s(e,"P",{});var ig=r(yl);mh=i(ig,"Make sure your file is accessible"),ig.forEach(t),Kr=p(e),g(Ut.$$.fragment,e),ea=p(e),x=s(e,"H2",{class:!0});var kn=r(x);me=s(kn,"A",{id:!0,class:!0,href:!0});var hg=r(me);Ho=s(hg,"SPAN",{});var ug=r(Ho);g(Jt.$$.fragment,ug),ug.forEach(t),hg.forEach(t),yh=p(kn),Io=s(kn,"SPAN",{});var pg=r(Io);dh=i(pg,"Let's recap how this environment works"),pg.forEach(t),kn.forEach(t),ta=p(e),B=s(e,"H3",{class:!0});var Un=r(B);ye=s(Un,"A",{id:!0,class:!0,href:!0});var fg=r(ye);Go=s(fg,"SPAN",{});var cg=r(Go);g(jt.$$.fragment,cg),cg.forEach(t),fg.forEach(t),vh=p(Un),So=s(Un,"SPAN",{});var gg=r(So);wh=i(gg,"The State Space: what Huggy perceives."),gg.forEach(t),Un.forEach(t),la=p(e),dl=s(e,"P",{});var mg=r(dl);bh=i(mg,"Huggy doesn\u2019t \u201Csee\u201D his environment. Instead, we provide him information about the environment:"),mg.forEach(t),oa=p(e),H=s(e,"UL",{});var Ol=r(H);Co=s(Ol,"LI",{});var yg=r(Co);_h=i(yg,"The target (stick) position"),yg.forEach(t),Eh=p(Ol),Ao=s(Ol,"LI",{});var dg=r(Ao);Mh=i(dg,"The relative position between himself and the target"),dg.forEach(t),Th=p(Ol),Zo=s(Ol,"LI",{});var vg=r(Zo);$h=i(vg,"The orientation of his legs."),vg.forEach(t),Ol.forEach(t),sa=p(e),de=s(e,"P",{});var Jn=r(de);kh=i(Jn,"Given all this information, Huggy "),No=s(Jn,"STRONG",{});var wg=r(No);Uh=i(wg,"can decide which action to take next to fulfill his goal"),wg.forEach(t),Jh=i(Jn,"."),Jn.forEach(t),ra=p(e),ve=s(e,"IMG",{src:!0,alt:!0,width:!0}),aa=p(e),z=s(e,"H3",{class:!0});var jn=r(z);we=s(jn,"A",{id:!0,class:!0,href:!0});var bg=r(we);Wo=s(bg,"SPAN",{});var _g=r(Wo);g(Ht.$$.fragment,_g),_g.forEach(t),bg.forEach(t),jh=p(jn),Lo=s(jn,"SPAN",{});var Eg=r(Lo);Hh=i(Eg,"The Action Space: what moves Huggy can do"),Eg.forEach(t),jn.forEach(t),na=p(e),be=s(e,"IMG",{src:!0,alt:!0,width:!0}),ia=p(e),O=s(e,"P",{});var ir=r(O);Ro=s(ir,"STRONG",{});var Mg=r(Ro);Ih=i(Mg,"Joint motors drive huggy legs"),Mg.forEach(t),Gh=i(ir,". This means that to get the target, Huggy needs to "),Po=s(ir,"STRONG",{});var Tg=r(Po);Sh=i(Tg,"learn to rotate the joint motors of each of his legs correctly so he can move"),Tg.forEach(t),Ch=i(ir,"."),ir.forEach(t),ha=p(e),V=s(e,"H3",{class:!0});var Hn=r(V);_e=s(Hn,"A",{id:!0,class:!0,href:!0});var $g=r(_e);xo=s($g,"SPAN",{});var kg=r(xo);g(It.$$.fragment,kg),kg.forEach(t),$g.forEach(t),Ah=p(Hn),Bo=s(Hn,"SPAN",{});var Ug=r(Bo);Zh=i(Ug,"The Reward Function"),Ug.forEach(t),Hn.forEach(t),ua=p(e),Ee=s(e,"P",{});var In=r(Ee);Nh=i(In,"The reward function is designed so that "),zo=s(In,"STRONG",{});var Jg=r(zo);Wh=i(Jg,"Huggy will fulfill his goal"),Jg.forEach(t),Lh=i(In," : fetch the stick."),In.forEach(t),pa=p(e),I=s(e,"P",{});var Vl=r(I);Rh=i(Vl,"Remember that one of the foundations of Reinforcement Learning is the "),Oo=s(Vl,"EM",{});var jg=r(Oo);Ph=i(jg,"reward hypothesis"),jg.forEach(t),xh=i(Vl,": a goal can be described as the "),Vo=s(Vl,"STRONG",{});var Hg=r(Vo);Bh=i(Hg,"maximization of the expected cumulative reward"),Hg.forEach(t),zh=i(Vl,"."),Vl.forEach(t),fa=p(e),Me=s(e,"P",{});var Gn=r(Me);Oh=i(Gn,"Here, our goal is that Huggy "),Xo=s(Gn,"STRONG",{});var Ig=r(Xo);Vh=i(Ig,"goes towards the stick but without spinning too much"),Ig.forEach(t),Xh=i(Gn,". Hence, our reward function must translate this goal."),Gn.forEach(t),ca=p(e),vl=s(e,"P",{});var Gg=r(vl);Yh=i(Gg,"Our reward function:"),Gg.forEach(t),ga=p(e),Te=s(e,"IMG",{src:!0,alt:!0,width:!0}),ma=p(e),E=s(e,"UL",{});var Ye=r(E);$e=s(Ye,"LI",{});var hr=r($e);Yo=s(hr,"EM",{});var Sg=r(Yo);Fh=i(Sg,"Orientation bonus"),Sg.forEach(t),Dh=i(hr,": we "),Fo=s(hr,"STRONG",{});var Cg=r(Fo);Qh=i(Cg,"reward him for getting close to the target"),Cg.forEach(t),qh=i(hr,"."),hr.forEach(t),Kh=p(Ye),ke=s(Ye,"LI",{});var ur=r(ke);Do=s(ur,"EM",{});var Ag=r(Do);eu=i(Ag,"Time penalty"),Ag.forEach(t),tu=i(ur,": a fixed-time penalty given at every action to "),Qo=s(ur,"STRONG",{});var Zg=r(Qo);lu=i(Zg,"force him to get to the stick as fast as possible"),Zg.forEach(t),ou=i(ur,"."),ur.forEach(t),su=p(Ye),Ue=s(Ye,"LI",{});var pr=r(Ue);qo=s(pr,"EM",{});var Ng=r(qo);ru=i(Ng,"Rotation penalty"),Ng.forEach(t),au=i(pr,": we penalize Huggy if "),Ko=s(pr,"STRONG",{});var Wg=r(Ko);nu=i(Wg,"he spins too much and turns too quickly"),Wg.forEach(t),iu=i(pr,"."),pr.forEach(t),hu=p(Ye),Je=s(Ye,"LI",{});var fr=r(Je);es=s(fr,"EM",{});var Lg=r(es);uu=i(Lg,"Getting to the target reward"),Lg.forEach(t),pu=i(fr,": we reward Huggy for "),ts=s(fr,"STRONG",{});var Rg=r(ts);fu=i(Rg,"reaching the target"),Rg.forEach(t),cu=i(fr,"."),fr.forEach(t),Ye.forEach(t),ya=p(e),X=s(e,"H2",{class:!0});var Sn=r(X);je=s(Sn,"A",{id:!0,class:!0,href:!0});var Pg=r(je);ls=s(Pg,"SPAN",{});var xg=r(ls);g(Gt.$$.fragment,xg),xg.forEach(t),Pg.forEach(t),gu=p(Sn),os=s(Sn,"SPAN",{});var Bg=r(os);mu=i(Bg,"Check the Huggy config file"),Bg.forEach(t),Sn.forEach(t),da=p(e),b=s(e,"UL",{});var $=r(b);ss=s($,"LI",{});var zg=r(ss);wl=s(zg,"P",{});var Tf=r(wl);yu=i(Tf,"In ML-Agents, you define the "),rs=s(Tf,"STRONG",{});var Og=r(rs);du=i(Og,"training hyperparameters in config.yaml files."),Og.forEach(t),Tf.forEach(t),zg.forEach(t),vu=p($),as=s($,"LI",{});var Vg=r(as);St=s(Vg,"P",{});var Cn=r(St);wu=i(Cn,"For the scope of this notebook, we\u2019re not going to modify the hyperparameters, but if you want to try as an experiment, Unity provides very "),Ct=s(Cn,"A",{href:!0,rel:!0});var Xg=r(Ct);bu=i(Xg,"good documentation explaining each of them here"),Xg.forEach(t),_u=i(Cn,"."),Cn.forEach(t),Vg.forEach(t),Eu=p($),ns=s($,"LI",{});var Yg=r(ns);is=s(Yg,"P",{});var Fg=r(is);Mu=i(Fg,"We need to create a config file for Huggy."),Fg.forEach(t),Yg.forEach(t),Tu=p($),hs=s($,"LI",{});var Dg=r(hs);bl=s(Dg,"P",{});var $f=r(bl);$u=i($f,"Go to "),us=s($f,"CODE",{});var Qg=r(us);ku=i(Qg,"/content/ml-agents/config/ppo"),Qg.forEach(t),$f.forEach(t),Dg.forEach(t),Uu=p($),ps=s($,"LI",{});var qg=r(ps);_l=s(qg,"P",{});var kf=r(_l);Ju=i(kf,"Create a new file called "),fs=s(kf,"CODE",{});var Kg=r(fs);ju=i(Kg,"Huggy.yaml"),Kg.forEach(t),kf.forEach(t),qg.forEach(t),Hu=p($),cs=s($,"LI",{});var em=r(cs);gs=s(em,"P",{});var tm=r(gs);Iu=i(tm,"Copy and paste the content below \u{1F53D}"),tm.forEach(t),em.forEach(t),$.forEach(t),va=p(e),g(At.$$.fragment,e),wa=p(e),He=s(e,"UL",{});var An=r(He);ms=s(An,"LI",{});var lm=r(ms);ys=s(lm,"P",{});var om=r(ys);Gu=i(om,"Don\u2019t forget to save the file!"),om.forEach(t),lm.forEach(t),Su=p(An),ds=s(An,"LI",{});var sm=r(ds);Zt=s(sm,"P",{});var Zn=r(Zt);vs=s(Zn,"STRONG",{});var rm=r(vs);Cu=i(rm,"In the case you want to modify the hyperparameters"),rm.forEach(t),Au=i(Zn,", in Google Colab notebook, you can click here to open the config.yaml: "),ws=s(Zn,"CODE",{});var am=r(ws);Zu=i(am,"/content/ml-agents/config/ppo/Huggy.yaml"),am.forEach(t),Zn.forEach(t),sm.forEach(t),An.forEach(t),ba=p(e),El=s(e,"P",{});var nm=r(El);Nu=i(nm,"We\u2019re now ready to train our agent \u{1F525}."),nm.forEach(t),_a=p(e),Y=s(e,"H2",{class:!0});var Nn=r(Y);Ie=s(Nn,"A",{id:!0,class:!0,href:!0});var im=r(Ie);bs=s(im,"SPAN",{});var hm=r(bs);g(Nt.$$.fragment,hm),hm.forEach(t),im.forEach(t),Wu=p(Nn),_s=s(Nn,"SPAN",{});var um=r(_s);Lu=i(um,"Train our agent"),um.forEach(t),Nn.forEach(t),Ea=p(e),Wt=s(e,"P",{});var Uf=r(Wt);Ru=i(Uf,"To train our agent, we just need to "),Es=s(Uf,"STRONG",{});var pm=r(Es);Pu=i(pm,"launch mlagents-learn and select the executable containing the environment."),pm.forEach(t),Uf.forEach(t),Ma=p(e),Ge=s(e,"IMG",{src:!0,alt:!0,width:!0}),Ta=p(e),Ml=s(e,"P",{});var fm=r(Ml);xu=i(fm,"With ML Agents, we run a training script. We define four parameters:"),fm.forEach(t),$a=p(e),M=s(e,"OL",{});var Fe=r(M);Tl=s(Fe,"LI",{});var Jf=r(Tl);Ms=s(Jf,"CODE",{});var cm=r(Ms);Bu=i(cm,"mlagents-learn <config>"),cm.forEach(t),zu=i(Jf,": the path where the hyperparameter config file is."),Jf.forEach(t),Ou=p(Fe),$l=s(Fe,"LI",{});var jf=r($l);Ts=s(jf,"CODE",{});var gm=r(Ts);Vu=i(gm,"--env"),gm.forEach(t),Xu=i(jf,": where the environment executable is."),jf.forEach(t),Yu=p(Fe),kl=s(Fe,"LI",{});var Hf=r(kl);$s=s(Hf,"CODE",{});var mm=r($s);Fu=i(mm,"--run_id"),mm.forEach(t),Du=i(Hf,": the name you want to give to your training run id."),Hf.forEach(t),Qu=p(Fe),Ul=s(Fe,"LI",{});var If=r(Ul);ks=s(If,"CODE",{});var ym=r(ks);qu=i(ym,"--no-graphics"),ym.forEach(t),Ku=i(If,": to not launch the visualization during the training."),If.forEach(t),Fe.forEach(t),ka=p(e),Se=s(e,"P",{});var Wn=r(Se);ep=i(Wn,"Train the model and use the "),Us=s(Wn,"CODE",{});var dm=r(Us);tp=i(dm,"--resume"),dm.forEach(t),lp=i(Wn," flag to continue training in case of interruption."),Wn.forEach(t),Ua=p(e),Jl=s(e,"BLOCKQUOTE",{});var vm=r(Jl);Lt=s(vm,"P",{});var Ln=r(Lt);op=i(Ln,"It will fail first time when you use "),Js=s(Ln,"CODE",{});var wm=r(Js);sp=i(wm,"--resume"),wm.forEach(t),rp=i(Ln,", try running the block again to bypass the error."),Ln.forEach(t),vm.forEach(t),Ja=p(e),Ce=s(e,"P",{});var Rn=r(Ce);ap=i(Rn,"The training will take 30 to 45min depending on your machine (don\u2019t forget to "),js=s(Rn,"STRONG",{});var bm=r(js);np=i(bm,"set up a GPU"),bm.forEach(t),ip=i(Rn,"), go take a \u2615\uFE0F you deserve it \u{1F917}."),Rn.forEach(t),ja=p(e),g(Rt.$$.fragment,e),Ha=p(e),F=s(e,"H2",{class:!0});var Pn=r(F);Ae=s(Pn,"A",{id:!0,class:!0,href:!0});var _m=r(Ae);Hs=s(_m,"SPAN",{});var Em=r(Hs);g(Pt.$$.fragment,Em),Em.forEach(t),_m.forEach(t),hp=p(Pn),Is=s(Pn,"SPAN",{});var Mm=r(Is);up=i(Mm,"Push the agent to the \u{1F917} Hub"),Mm.forEach(t),Pn.forEach(t),Ia=p(e),jl=s(e,"UL",{});var Tm=r(jl);Hl=s(Tm,"LI",{});var Gf=r(Hl);pp=i(Gf,"Now that we trained our agent, we\u2019re "),Gs=s(Gf,"STRONG",{});var $m=r(Gs);fp=i($m,"ready to push it to the Hub to be able to play with Huggy on your browser\u{1F525}."),$m.forEach(t),Gf.forEach(t),Tm.forEach(t),Ga=p(e),Il=s(e,"P",{});var km=r(Il);cp=i(km,"To be able to share your model with the community there are three more steps to follow:"),km.forEach(t),Sa=p(e),xt=s(e,"P",{});var Sf=r(xt);gp=i(Sf,"1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),Bt=s(Sf,"A",{href:!0,rel:!0});var Um=r(Bt);mp=i(Um,"https://huggingface.co/join"),Um.forEach(t),Sf.forEach(t),Ca=p(e),Gl=s(e,"P",{});var Jm=r(Gl);yp=i(Jm,"2\uFE0F\u20E3 Sign in and then get your token from the Hugging Face website."),Jm.forEach(t),Aa=p(e),Sl=s(e,"UL",{});var jm=r(Sl);Ze=s(jm,"LI",{});var cr=r(Ze);dp=i(cr,"Create a new token ("),zt=s(cr,"A",{href:!0,rel:!0});var Hm=r(zt);vp=i(Hm,"https://huggingface.co/settings/tokens"),Hm.forEach(t),wp=i(cr,") "),Ss=s(cr,"STRONG",{});var Im=r(Ss);bp=i(Im,"with write role"),Im.forEach(t),cr.forEach(t),jm.forEach(t),Za=p(e),Ot=s(e,"IMG",{src:!0,alt:!0}),Na=p(e),Ne=s(e,"UL",{});var xn=r(Ne);Cs=s(xn,"LI",{});var Gm=r(Cs);_p=i(Gm,"Copy the token"),Gm.forEach(t),Ep=p(xn),As=s(xn,"LI",{});var Sm=r(As);Mp=i(Sm,"Run the cell below and paste the token"),Sm.forEach(t),xn.forEach(t),Wa=p(e),g(Vt.$$.fragment,e),La=p(e),Xt=s(e,"P",{});var Cf=r(Xt);Tp=i(Cf,"If you don\u2019t want to use Google Colab or a Jupyter Notebook, you need to use this command instead: "),Zs=s(Cf,"CODE",{});var Cm=r(Zs);$p=i(Cm,"huggingface-cli login"),Cm.forEach(t),Cf.forEach(t),Ra=p(e),We=s(e,"P",{});var Bn=r(We);kp=i(Bn,"Then, we simply need to run "),Ns=s(Bn,"CODE",{});var Am=r(Ns);Up=i(Am,"mlagents-push-to-hf"),Am.forEach(t),Jp=i(Bn,"."),Bn.forEach(t),Pa=p(e),Le=s(e,"IMG",{src:!0,alt:!0,width:!0}),xa=p(e),Cl=s(e,"P",{});var Zm=r(Cl);jp=i(Zm,"And we define 4 parameters:"),Zm.forEach(t),Ba=p(e),T=s(e,"OL",{});var De=r(T);Al=s(De,"LI",{});var Af=r(Al);Ws=s(Af,"CODE",{});var Nm=r(Ws);Hp=i(Nm,"--run-id"),Nm.forEach(t),Ip=i(Af,": the name of the training run id."),Af.forEach(t),Gp=p(De),Zl=s(De,"LI",{});var Zf=r(Zl);Ls=s(Zf,"CODE",{});var Wm=r(Ls);Sp=i(Wm,"--local-dir"),Wm.forEach(t),Cp=i(Zf,": where the agent was saved, it\u2019s results/<run_id name>, so in my case results/First Training."),Zf.forEach(t),Ap=p(De),Yt=s(De,"LI",{});var zn=r(Yt);Rs=s(zn,"CODE",{});var Lm=r(Rs);Zp=i(Lm,"--repo-id"),Lm.forEach(t),Np=i(zn,`: the name of the Hugging Face repo you want to create or update. It\u2019s always <your huggingface username>/<the repo name>
If the repo does not exist `),Ps=s(zn,"STRONG",{});var Rm=r(Ps);Wp=i(Rm,"it will be created automatically"),Rm.forEach(t),zn.forEach(t),Lp=p(De),Nl=s(De,"LI",{});var Nf=r(Nl);xs=s(Nf,"CODE",{});var Pm=r(xs);Rp=i(Pm,"--commit-message"),Pm.forEach(t),Pp=i(Nf,": since HF repos are git repositories you need to give a commit message."),Nf.forEach(t),De.forEach(t),za=p(e),g(Ft.$$.fragment,e),Oa=p(e),Wl=s(e,"P",{});var xm=r(Wl);xp=i(xm,"If everything worked you should see this at the end of the process (but with a different url \u{1F606}) :"),xm.forEach(t),Va=p(e),g(Dt.$$.fragment,e),Xa=p(e),Qt=s(e,"P",{});var Wf=r(Qt);Bp=i(Wf,"It\u2019s the link to your model repository. The repository contains a model card that explains how to use the model, your Tensorboard logs and your config file. "),Bs=s(Wf,"STRONG",{});var Bm=r(Bs);zp=i(Bm,"What\u2019s awesome is that it\u2019s a git repository, which means you can have different commits, update your repository with a new push, open Pull Requests, etc."),Bm.forEach(t),Wf.forEach(t),Ya=p(e),Re=s(e,"IMG",{src:!0,alt:!0,width:!0}),Fa=p(e),qt=s(e,"P",{});var Lf=r(qt);Op=i(Lf,"But now comes the best part: "),zs=s(Lf,"STRONG",{});var zm=r(zs);Vp=i(zm,"being able to play with Huggy online \u{1F440}."),zm.forEach(t),Lf.forEach(t),Da=p(e),D=s(e,"H2",{class:!0});var On=r(D);Pe=s(On,"A",{id:!0,class:!0,href:!0});var Om=r(Pe);Os=s(Om,"SPAN",{});var Vm=r(Os);g(Kt.$$.fragment,Vm),Vm.forEach(t),Om.forEach(t),Xp=p(On),Vs=s(On,"SPAN",{});var Xm=r(Vs);Yp=i(Xm,"Play with your Huggy \u{1F415}"),Xm.forEach(t),On.forEach(t),Qa=p(e),Ll=s(e,"P",{});var Ym=r(Ll);Fp=i(Ym,"This step is the simplest:"),Ym.forEach(t),qa=p(e),xe=s(e,"UL",{});var Vn=r(xe);Xs=s(Vn,"LI",{});var Fm=r(Xs);Rl=s(Fm,"P",{});var Rf=r(Rl);Dp=i(Rf,"Open the Huggy game in your browser: "),el=s(Rf,"A",{href:!0,rel:!0});var Dm=r(el);Qp=i(Dm,"https://huggingface.co/spaces/ThomasSimonini/Huggy"),Dm.forEach(t),Rf.forEach(t),Fm.forEach(t),qp=p(Vn),Ys=s(Vn,"LI",{});var Qm=r(Ys);Fs=s(Qm,"P",{});var qm=r(Fs);Kp=i(qm,"Click on Play with my Huggy model"),qm.forEach(t),Qm.forEach(t),Vn.forEach(t),Ka=p(e),Be=s(e,"IMG",{src:!0,alt:!0,width:!0}),en=p(e),ze=s(e,"OL",{});var Xn=r(ze);Ds=s(Xn,"LI",{});var Km=r(Ds);Qs=s(Km,"P",{});var ey=r(Qs);ef=i(ey,"In step 1, choose your model repository, which is the model id (in my case ThomasSimonini/ppo-Huggy)."),ey.forEach(t),Km.forEach(t),tf=p(Xn),qs=s(Xn,"LI",{});var ty=r(qs);tl=s(ty,"P",{});var Yn=r(tl);lf=i(Yn,"In step 2, "),Ks=s(Yn,"STRONG",{});var ly=r(Ks);of=i(ly,"choose which model you want to replay"),ly.forEach(t),sf=i(Yn,":"),Yn.forEach(t),ty.forEach(t),Xn.forEach(t),tn=p(e),Oe=s(e,"UL",{});var Fn=r(Oe);er=s(Fn,"LI",{});var oy=r(er);rf=i(oy,"I have multiple ones, since we saved a model every 500000 timesteps."),oy.forEach(t),af=p(Fn),Pl=s(Fn,"LI",{});var Pf=r(Pl);nf=i(Pf,"But since I want the most recent one, I choose "),tr=s(Pf,"CODE",{});var sy=r(tr);hf=i(sy,"Huggy.onnx"),sy.forEach(t),Pf.forEach(t),Fn.forEach(t),ln=p(e),ll=s(e,"P",{});var xf=r(ll);uf=i(xf,"\u{1F449} It\u2019s good "),lr=s(xf,"STRONG",{});var ry=r(lr);pf=i(ry,"to try with different models steps to see the improvement of the agent."),ry.forEach(t),xf.forEach(t),on=p(e),xl=s(e,"P",{});var ay=r(xl);ff=i(ay,"Congrats on finishing this bonus unit!"),ay.forEach(t),sn=p(e),Q=s(e,"P",{});var gr=r(Q);cf=i(gr,"You can now sit and enjoy playing with your Huggy \u{1F436}. And don\u2019t "),or=s(gr,"STRONG",{});var ny=r(or);gf=i(ny,"forget to spread the love by sharing Huggy with your friends \u{1F917}"),ny.forEach(t),mf=i(gr,". And if you share about it on social media, "),sr=s(gr,"STRONG",{});var iy=r(sr);yf=i(iy,"please tag us @huggingface and me @simoninithomas"),iy.forEach(t),gr.forEach(t),rn=p(e),Ve=s(e,"IMG",{src:!0,alt:!0,width:!0}),an=p(e),q=s(e,"H2",{class:!0});var Dn=r(q);Xe=s(Dn,"A",{id:!0,class:!0,href:!0});var hy=r(Xe);rr=s(hy,"SPAN",{});var uy=r(rr);g(ol.$$.fragment,uy),uy.forEach(t),hy.forEach(t),df=p(Dn),ar=s(Dn,"SPAN",{});var py=r(ar);vf=i(py,"Keep Learning, Stay awesome \u{1F917}"),py.forEach(t),Dn.forEach(t),this.h()},h(){f(G,"name","hf:doc:metadata"),f(G,"content",JSON.stringify(by)),f(K,"id","train"),f(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(K,"href","#train"),f(S,"class","relative group"),f(le,"id","lets-train-huggy"),f(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(le,"href","#lets-train-huggy"),f(C,"class","relative group"),_(al.src,zf="https://colab.research.google.com/assets/colab-badge.svg")||f(al,"src",zf),f(al,"alt","Open In Colab"),f(tt,"href","https://colab.research.google.com/github/huggingface/deep-rl-class/blob/master/notebooks/bonus-unit1/bonus-unit1.ipynb"),f(tt,"rel","nofollow"),_(lt.src,Of="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit2/thumbnail.png")||f(lt,"src",Of),f(lt,"alt","Bonus Unit 1Thumbnail"),f(se,"id","the-environment"),f(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(se,"href","#the-environment"),f(A,"class","relative group"),f(at,"href","https://twitter.com/ThomasSimonini"),f(at,"rel","nofollow"),f(nt,"href","https://blog.unity.com/technology/puppo-the-corgi-cuteness-overload-with-the-unity-ml-agents-toolkit"),f(nt,"rel","nofollow"),f(ae,"id","the-library-used"),f(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ae,"href","#the-library-used"),f(Z,"class","relative group"),f(ht,"href","https://github.com/Unity-Technologies/ml-agents"),f(ht,"rel","nofollow"),f(ut,"href","https://github.com/huggingface/deep-rl-class/issues"),f(ut,"rel","nofollow"),f(ne,"id","objectives-of-this-notebook"),f(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ne,"href","#objectives-of-this-notebook"),f(N,"class","relative group"),f(ie,"id","prerequisites"),f(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ie,"href","#prerequisites"),f(W,"class","relative group"),f(pe,"id","set-the-gpu"),f(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(pe,"href","#set-the-gpu"),f(L,"class","relative group"),_(yt.src,Vf="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step1.jpg")||f(yt,"src",Vf),f(yt,"alt","GPU Step 1"),_(dt.src,Xf="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step2.jpg")||f(dt,"src",Xf),f(dt,"alt","GPU Step 2"),f(fe,"id","clone-the-repository-and-install-the-dependencies"),f(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(fe,"href","#clone-the-repository-and-install-the-dependencies"),f(R,"class","relative group"),f(ce,"id","download-and-move-the-environment-zip-file-in-trainedenvsexecutableslinux"),f(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ce,"href","#download-and-move-the-environment-zip-file-in-trainedenvsexecutableslinux"),f(P,"class","relative group"),f(Tt,"href","https://drive.google.com/uc?export=download&id=1zv3M95ZJTWHUVOWT6ckq_cm98nft8gdF"),f(Tt,"rel","nofollow"),f($t,"href","https://bcrf.biochem.wisc.edu/2021/02/05/download-google-drive-files-using-wget/"),f($t,"rel","nofollow"),f(me,"id","lets-recap-how-this-environment-works"),f(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(me,"href","#lets-recap-how-this-environment-works"),f(x,"class","relative group"),f(ye,"id","the-state-space-what-huggy-perceives"),f(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ye,"href","#the-state-space-what-huggy-perceives"),f(B,"class","relative group"),_(ve.src,Yf="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/huggy.jpg")||f(ve,"src",Yf),f(ve,"alt","Huggy"),f(ve,"width","100%"),f(we,"id","the-action-space-what-moves-huggy-can-do"),f(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(we,"href","#the-action-space-what-moves-huggy-can-do"),f(z,"class","relative group"),_(be.src,Ff="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/huggy-action.jpg")||f(be,"src",Ff),f(be,"alt","Huggy action"),f(be,"width","100%"),f(_e,"id","the-reward-function"),f(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(_e,"href","#the-reward-function"),f(V,"class","relative group"),_(Te.src,Df="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/reward.jpg")||f(Te,"src",Df),f(Te,"alt","Huggy reward function"),f(Te,"width","100%"),f(je,"id","check-the-huggy-config-file"),f(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(je,"href","#check-the-huggy-config-file"),f(X,"class","relative group"),f(Ct,"href","https://github.com/Unity-Technologies/ml-agents/blob/main/docs/Training-Configuration-File.md"),f(Ct,"rel","nofollow"),f(Ie,"id","train-our-agent"),f(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ie,"href","#train-our-agent"),f(Y,"class","relative group"),_(Ge.src,Qf="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/mllearn.png")||f(Ge,"src",Qf),f(Ge,"alt","ml learn function"),f(Ge,"width","100%"),f(Ae,"id","push-the-agent-to-the-hub"),f(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ae,"href","#push-the-agent-to-the-hub"),f(F,"class","relative group"),f(Bt,"href","https://huggingface.co/join"),f(Bt,"rel","nofollow"),f(zt,"href","https://huggingface.co/settings/tokens"),f(zt,"rel","nofollow"),_(Ot.src,qf="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/create-token.jpg")||f(Ot,"src",qf),f(Ot,"alt","Create HF Token"),_(Le.src,Kf="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/mlpush.png")||f(Le,"src",Kf),f(Le,"alt","ml learn function"),f(Le,"width","100%"),_(Re.src,ec="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/modelcard.png")||f(Re,"src",ec),f(Re,"alt","ml learn function"),f(Re,"width","100%"),f(Pe,"id","play-with-your-huggy"),f(Pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Pe,"href","#play-with-your-huggy"),f(D,"class","relative group"),f(el,"href","https://huggingface.co/spaces/ThomasSimonini/Huggy"),f(el,"rel","nofollow"),_(Be.src,tc="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/load-huggy.jpg")||f(Be,"src",tc),f(Be,"alt","load-huggy"),f(Be,"width","100%"),_(Ve.src,lc="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit-bonus1/huggy-cover.jpeg")||f(Ve,"src",lc),f(Ve,"alt","Huggy cover"),f(Ve,"width","100%"),f(Xe,"id","keep-learning-stay-awesome"),f(Xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Xe,"href","#keep-learning-stay-awesome"),f(q,"class","relative group")},m(e,a){l(document.head,G),h(e,mr,a),h(e,S,a),l(S,K),l(K,Xl),m(Qe,Xl,null),l(S,Qn),l(S,Yl),l(Yl,qn),h(e,yr,a),m(qe,e,a),h(e,dr,a),h(e,ee,a),l(ee,Kn),l(ee,Fl),l(Fl,ei),l(ee,ti),h(e,vr,a),h(e,te,a),l(te,li),l(te,Dl),l(Dl,oi),l(te,si),h(e,wr,a),h(e,C,a),l(C,le),l(le,Ql),m(Ke,Ql,null),l(C,ri),l(C,ql),l(ql,ai),h(e,br,a),h(e,et,a),l(et,Kl),l(Kl,ni),l(et,ii),h(e,_r,a),h(e,rl,a),l(rl,tt),l(tt,al),h(e,Er,a),h(e,lt,a),h(e,Mr,a),h(e,ot,a),l(ot,hi),l(ot,eo),l(eo,ui),h(e,Tr,a),h(e,oe,a),l(oe,pi),l(oe,to),l(to,fi),l(oe,ci),h(e,$r,a),m(st,e,a),h(e,kr,a),h(e,A,a),l(A,se),l(se,lo),m(rt,lo,null),l(A,gi),l(A,oo),l(oo,mi),h(e,Ur,a),h(e,nl,a),l(nl,re),l(re,yi),l(re,at),l(at,di),l(re,vi),l(re,nt),l(nt,wi),h(e,Jr,a),h(e,Z,a),l(Z,ae),l(ae,so),m(it,so,null),l(Z,bi),l(Z,ro),l(ro,_i),h(e,jr,a),h(e,il,a),l(il,ao),l(ao,ht),l(ht,Ei),h(e,Hr,a),h(e,J,a),l(J,Mi),l(J,no),l(no,Ti),l(J,$i),l(J,ut),l(ut,ki),l(J,Ui),h(e,Ir,a),h(e,N,a),l(N,ne),l(ne,io),m(pt,io,null),l(N,Ji),l(N,ho),l(ho,ji),h(e,Gr,a),h(e,hl,a),l(hl,Hi),h(e,Sr,a),h(e,j,a),l(j,ft),l(ft,Ii),l(ft,uo),l(uo,Gi),l(ft,Si),l(j,Ci),l(j,ul),l(ul,po),l(po,Ai),l(ul,Zi),l(j,Ni),l(j,ct),l(ct,Wi),l(ct,fo),l(fo,Li),l(ct,Ri),h(e,Cr,a),h(e,W,a),l(W,ie),l(ie,co),m(gt,co,null),l(W,Pi),l(W,go),l(go,xi),h(e,Ar,a),h(e,pl,a),l(pl,Bi),h(e,Zr,a),h(e,he,a),l(he,zi),l(he,mo),l(mo,Oi),l(he,Vi),h(e,Nr,a),h(e,ue,a),l(ue,Xi),l(ue,yo),l(yo,Yi),l(ue,Fi),h(e,Wr,a),h(e,L,a),l(L,pe),l(pe,vo),m(mt,vo,null),l(L,Di),l(L,wo),l(wo,Qi),h(e,Lr,a),h(e,yt,a),h(e,Rr,a),h(e,fl,a),l(fl,bo),l(bo,_o),l(_o,qi),h(e,Pr,a),h(e,dt,a),h(e,xr,a),h(e,R,a),l(R,fe),l(fe,Eo),m(vt,Eo,null),l(R,Ki),l(R,Mo),l(Mo,eh),h(e,Br,a),h(e,cl,a),l(cl,To),l(To,th),h(e,zr,a),m(wt,e,a),h(e,Or,a),m(bt,e,a),h(e,Vr,a),h(e,P,a),l(P,ce),l(ce,$o),m(_t,$o,null),l(P,lh),l(P,gl),l(gl,oh),l(gl,ko),l(ko,sh),h(e,Xr,a),h(e,ge,a),l(ge,Uo),l(Uo,rh),l(ge,ah),l(ge,ml),l(ml,nh),l(ml,Jo),l(Jo,ih),h(e,Yr,a),m(Et,e,a),h(e,Fr,a),m(Mt,e,a),h(e,Dr,a),h(e,U,a),l(U,hh),l(U,Tt),l(Tt,uh),l(U,ph),l(U,jo),l(jo,fh),l(U,ch),l(U,$t),l($t,gh),h(e,Qr,a),m(kt,e,a),h(e,qr,a),h(e,yl,a),l(yl,mh),h(e,Kr,a),m(Ut,e,a),h(e,ea,a),h(e,x,a),l(x,me),l(me,Ho),m(Jt,Ho,null),l(x,yh),l(x,Io),l(Io,dh),h(e,ta,a),h(e,B,a),l(B,ye),l(ye,Go),m(jt,Go,null),l(B,vh),l(B,So),l(So,wh),h(e,la,a),h(e,dl,a),l(dl,bh),h(e,oa,a),h(e,H,a),l(H,Co),l(Co,_h),l(H,Eh),l(H,Ao),l(Ao,Mh),l(H,Th),l(H,Zo),l(Zo,$h),h(e,sa,a),h(e,de,a),l(de,kh),l(de,No),l(No,Uh),l(de,Jh),h(e,ra,a),h(e,ve,a),h(e,aa,a),h(e,z,a),l(z,we),l(we,Wo),m(Ht,Wo,null),l(z,jh),l(z,Lo),l(Lo,Hh),h(e,na,a),h(e,be,a),h(e,ia,a),h(e,O,a),l(O,Ro),l(Ro,Ih),l(O,Gh),l(O,Po),l(Po,Sh),l(O,Ch),h(e,ha,a),h(e,V,a),l(V,_e),l(_e,xo),m(It,xo,null),l(V,Ah),l(V,Bo),l(Bo,Zh),h(e,ua,a),h(e,Ee,a),l(Ee,Nh),l(Ee,zo),l(zo,Wh),l(Ee,Lh),h(e,pa,a),h(e,I,a),l(I,Rh),l(I,Oo),l(Oo,Ph),l(I,xh),l(I,Vo),l(Vo,Bh),l(I,zh),h(e,fa,a),h(e,Me,a),l(Me,Oh),l(Me,Xo),l(Xo,Vh),l(Me,Xh),h(e,ca,a),h(e,vl,a),l(vl,Yh),h(e,ga,a),h(e,Te,a),h(e,ma,a),h(e,E,a),l(E,$e),l($e,Yo),l(Yo,Fh),l($e,Dh),l($e,Fo),l(Fo,Qh),l($e,qh),l(E,Kh),l(E,ke),l(ke,Do),l(Do,eu),l(ke,tu),l(ke,Qo),l(Qo,lu),l(ke,ou),l(E,su),l(E,Ue),l(Ue,qo),l(qo,ru),l(Ue,au),l(Ue,Ko),l(Ko,nu),l(Ue,iu),l(E,hu),l(E,Je),l(Je,es),l(es,uu),l(Je,pu),l(Je,ts),l(ts,fu),l(Je,cu),h(e,ya,a),h(e,X,a),l(X,je),l(je,ls),m(Gt,ls,null),l(X,gu),l(X,os),l(os,mu),h(e,da,a),h(e,b,a),l(b,ss),l(ss,wl),l(wl,yu),l(wl,rs),l(rs,du),l(b,vu),l(b,as),l(as,St),l(St,wu),l(St,Ct),l(Ct,bu),l(St,_u),l(b,Eu),l(b,ns),l(ns,is),l(is,Mu),l(b,Tu),l(b,hs),l(hs,bl),l(bl,$u),l(bl,us),l(us,ku),l(b,Uu),l(b,ps),l(ps,_l),l(_l,Ju),l(_l,fs),l(fs,ju),l(b,Hu),l(b,cs),l(cs,gs),l(gs,Iu),h(e,va,a),m(At,e,a),h(e,wa,a),h(e,He,a),l(He,ms),l(ms,ys),l(ys,Gu),l(He,Su),l(He,ds),l(ds,Zt),l(Zt,vs),l(vs,Cu),l(Zt,Au),l(Zt,ws),l(ws,Zu),h(e,ba,a),h(e,El,a),l(El,Nu),h(e,_a,a),h(e,Y,a),l(Y,Ie),l(Ie,bs),m(Nt,bs,null),l(Y,Wu),l(Y,_s),l(_s,Lu),h(e,Ea,a),h(e,Wt,a),l(Wt,Ru),l(Wt,Es),l(Es,Pu),h(e,Ma,a),h(e,Ge,a),h(e,Ta,a),h(e,Ml,a),l(Ml,xu),h(e,$a,a),h(e,M,a),l(M,Tl),l(Tl,Ms),l(Ms,Bu),l(Tl,zu),l(M,Ou),l(M,$l),l($l,Ts),l(Ts,Vu),l($l,Xu),l(M,Yu),l(M,kl),l(kl,$s),l($s,Fu),l(kl,Du),l(M,Qu),l(M,Ul),l(Ul,ks),l(ks,qu),l(Ul,Ku),h(e,ka,a),h(e,Se,a),l(Se,ep),l(Se,Us),l(Us,tp),l(Se,lp),h(e,Ua,a),h(e,Jl,a),l(Jl,Lt),l(Lt,op),l(Lt,Js),l(Js,sp),l(Lt,rp),h(e,Ja,a),h(e,Ce,a),l(Ce,ap),l(Ce,js),l(js,np),l(Ce,ip),h(e,ja,a),m(Rt,e,a),h(e,Ha,a),h(e,F,a),l(F,Ae),l(Ae,Hs),m(Pt,Hs,null),l(F,hp),l(F,Is),l(Is,up),h(e,Ia,a),h(e,jl,a),l(jl,Hl),l(Hl,pp),l(Hl,Gs),l(Gs,fp),h(e,Ga,a),h(e,Il,a),l(Il,cp),h(e,Sa,a),h(e,xt,a),l(xt,gp),l(xt,Bt),l(Bt,mp),h(e,Ca,a),h(e,Gl,a),l(Gl,yp),h(e,Aa,a),h(e,Sl,a),l(Sl,Ze),l(Ze,dp),l(Ze,zt),l(zt,vp),l(Ze,wp),l(Ze,Ss),l(Ss,bp),h(e,Za,a),h(e,Ot,a),h(e,Na,a),h(e,Ne,a),l(Ne,Cs),l(Cs,_p),l(Ne,Ep),l(Ne,As),l(As,Mp),h(e,Wa,a),m(Vt,e,a),h(e,La,a),h(e,Xt,a),l(Xt,Tp),l(Xt,Zs),l(Zs,$p),h(e,Ra,a),h(e,We,a),l(We,kp),l(We,Ns),l(Ns,Up),l(We,Jp),h(e,Pa,a),h(e,Le,a),h(e,xa,a),h(e,Cl,a),l(Cl,jp),h(e,Ba,a),h(e,T,a),l(T,Al),l(Al,Ws),l(Ws,Hp),l(Al,Ip),l(T,Gp),l(T,Zl),l(Zl,Ls),l(Ls,Sp),l(Zl,Cp),l(T,Ap),l(T,Yt),l(Yt,Rs),l(Rs,Zp),l(Yt,Np),l(Yt,Ps),l(Ps,Wp),l(T,Lp),l(T,Nl),l(Nl,xs),l(xs,Rp),l(Nl,Pp),h(e,za,a),m(Ft,e,a),h(e,Oa,a),h(e,Wl,a),l(Wl,xp),h(e,Va,a),m(Dt,e,a),h(e,Xa,a),h(e,Qt,a),l(Qt,Bp),l(Qt,Bs),l(Bs,zp),h(e,Ya,a),h(e,Re,a),h(e,Fa,a),h(e,qt,a),l(qt,Op),l(qt,zs),l(zs,Vp),h(e,Da,a),h(e,D,a),l(D,Pe),l(Pe,Os),m(Kt,Os,null),l(D,Xp),l(D,Vs),l(Vs,Yp),h(e,Qa,a),h(e,Ll,a),l(Ll,Fp),h(e,qa,a),h(e,xe,a),l(xe,Xs),l(Xs,Rl),l(Rl,Dp),l(Rl,el),l(el,Qp),l(xe,qp),l(xe,Ys),l(Ys,Fs),l(Fs,Kp),h(e,Ka,a),h(e,Be,a),h(e,en,a),h(e,ze,a),l(ze,Ds),l(Ds,Qs),l(Qs,ef),l(ze,tf),l(ze,qs),l(qs,tl),l(tl,lf),l(tl,Ks),l(Ks,of),l(tl,sf),h(e,tn,a),h(e,Oe,a),l(Oe,er),l(er,rf),l(Oe,af),l(Oe,Pl),l(Pl,nf),l(Pl,tr),l(tr,hf),h(e,ln,a),h(e,ll,a),l(ll,uf),l(ll,lr),l(lr,pf),h(e,on,a),h(e,xl,a),l(xl,ff),h(e,sn,a),h(e,Q,a),l(Q,cf),l(Q,or),l(or,gf),l(Q,mf),l(Q,sr),l(sr,yf),h(e,rn,a),h(e,Ve,a),h(e,an,a),h(e,q,a),l(q,Xe),l(Xe,rr),m(ol,rr,null),l(q,df),l(q,ar),l(ar,vf),nn=!0},p:yy,i(e){nn||(y(Qe.$$.fragment,e),y(qe.$$.fragment,e),y(Ke.$$.fragment,e),y(st.$$.fragment,e),y(rt.$$.fragment,e),y(it.$$.fragment,e),y(pt.$$.fragment,e),y(gt.$$.fragment,e),y(mt.$$.fragment,e),y(vt.$$.fragment,e),y(wt.$$.fragment,e),y(bt.$$.fragment,e),y(_t.$$.fragment,e),y(Et.$$.fragment,e),y(Mt.$$.fragment,e),y(kt.$$.fragment,e),y(Ut.$$.fragment,e),y(Jt.$$.fragment,e),y(jt.$$.fragment,e),y(Ht.$$.fragment,e),y(It.$$.fragment,e),y(Gt.$$.fragment,e),y(At.$$.fragment,e),y(Nt.$$.fragment,e),y(Rt.$$.fragment,e),y(Pt.$$.fragment,e),y(Vt.$$.fragment,e),y(Ft.$$.fragment,e),y(Dt.$$.fragment,e),y(Kt.$$.fragment,e),y(ol.$$.fragment,e),nn=!0)},o(e){d(Qe.$$.fragment,e),d(qe.$$.fragment,e),d(Ke.$$.fragment,e),d(st.$$.fragment,e),d(rt.$$.fragment,e),d(it.$$.fragment,e),d(pt.$$.fragment,e),d(gt.$$.fragment,e),d(mt.$$.fragment,e),d(vt.$$.fragment,e),d(wt.$$.fragment,e),d(bt.$$.fragment,e),d(_t.$$.fragment,e),d(Et.$$.fragment,e),d(Mt.$$.fragment,e),d(kt.$$.fragment,e),d(Ut.$$.fragment,e),d(Jt.$$.fragment,e),d(jt.$$.fragment,e),d(Ht.$$.fragment,e),d(It.$$.fragment,e),d(Gt.$$.fragment,e),d(At.$$.fragment,e),d(Nt.$$.fragment,e),d(Rt.$$.fragment,e),d(Pt.$$.fragment,e),d(Vt.$$.fragment,e),d(Ft.$$.fragment,e),d(Dt.$$.fragment,e),d(Kt.$$.fragment,e),d(ol.$$.fragment,e),nn=!1},d(e){t(G),e&&t(mr),e&&t(S),v(Qe),e&&t(yr),v(qe,e),e&&t(dr),e&&t(ee),e&&t(vr),e&&t(te),e&&t(wr),e&&t(C),v(Ke),e&&t(br),e&&t(et),e&&t(_r),e&&t(rl),e&&t(Er),e&&t(lt),e&&t(Mr),e&&t(ot),e&&t(Tr),e&&t(oe),e&&t($r),v(st,e),e&&t(kr),e&&t(A),v(rt),e&&t(Ur),e&&t(nl),e&&t(Jr),e&&t(Z),v(it),e&&t(jr),e&&t(il),e&&t(Hr),e&&t(J),e&&t(Ir),e&&t(N),v(pt),e&&t(Gr),e&&t(hl),e&&t(Sr),e&&t(j),e&&t(Cr),e&&t(W),v(gt),e&&t(Ar),e&&t(pl),e&&t(Zr),e&&t(he),e&&t(Nr),e&&t(ue),e&&t(Wr),e&&t(L),v(mt),e&&t(Lr),e&&t(yt),e&&t(Rr),e&&t(fl),e&&t(Pr),e&&t(dt),e&&t(xr),e&&t(R),v(vt),e&&t(Br),e&&t(cl),e&&t(zr),v(wt,e),e&&t(Or),v(bt,e),e&&t(Vr),e&&t(P),v(_t),e&&t(Xr),e&&t(ge),e&&t(Yr),v(Et,e),e&&t(Fr),v(Mt,e),e&&t(Dr),e&&t(U),e&&t(Qr),v(kt,e),e&&t(qr),e&&t(yl),e&&t(Kr),v(Ut,e),e&&t(ea),e&&t(x),v(Jt),e&&t(ta),e&&t(B),v(jt),e&&t(la),e&&t(dl),e&&t(oa),e&&t(H),e&&t(sa),e&&t(de),e&&t(ra),e&&t(ve),e&&t(aa),e&&t(z),v(Ht),e&&t(na),e&&t(be),e&&t(ia),e&&t(O),e&&t(ha),e&&t(V),v(It),e&&t(ua),e&&t(Ee),e&&t(pa),e&&t(I),e&&t(fa),e&&t(Me),e&&t(ca),e&&t(vl),e&&t(ga),e&&t(Te),e&&t(ma),e&&t(E),e&&t(ya),e&&t(X),v(Gt),e&&t(da),e&&t(b),e&&t(va),v(At,e),e&&t(wa),e&&t(He),e&&t(ba),e&&t(El),e&&t(_a),e&&t(Y),v(Nt),e&&t(Ea),e&&t(Wt),e&&t(Ma),e&&t(Ge),e&&t(Ta),e&&t(Ml),e&&t($a),e&&t(M),e&&t(ka),e&&t(Se),e&&t(Ua),e&&t(Jl),e&&t(Ja),e&&t(Ce),e&&t(ja),v(Rt,e),e&&t(Ha),e&&t(F),v(Pt),e&&t(Ia),e&&t(jl),e&&t(Ga),e&&t(Il),e&&t(Sa),e&&t(xt),e&&t(Ca),e&&t(Gl),e&&t(Aa),e&&t(Sl),e&&t(Za),e&&t(Ot),e&&t(Na),e&&t(Ne),e&&t(Wa),v(Vt,e),e&&t(La),e&&t(Xt),e&&t(Ra),e&&t(We),e&&t(Pa),e&&t(Le),e&&t(xa),e&&t(Cl),e&&t(Ba),e&&t(T),e&&t(za),v(Ft,e),e&&t(Oa),e&&t(Wl),e&&t(Va),v(Dt,e),e&&t(Xa),e&&t(Qt),e&&t(Ya),e&&t(Re),e&&t(Fa),e&&t(qt),e&&t(Da),e&&t(D),v(Kt),e&&t(Qa),e&&t(Ll),e&&t(qa),e&&t(xe),e&&t(Ka),e&&t(Be),e&&t(en),e&&t(ze),e&&t(tn),e&&t(Oe),e&&t(ln),e&&t(ll),e&&t(on),e&&t(xl),e&&t(sn),e&&t(Q),e&&t(rn),e&&t(Ve),e&&t(an),e&&t(q),v(ol)}}}const by={local:"train",sections:[{local:"lets-train-huggy",sections:[{local:"the-environment",title:"The environment \u{1F3AE}"},{local:"the-library-used",title:"The library used \u{1F4DA}"}],title:"Let's train Huggy \u{1F436}"},{local:"objectives-of-this-notebook",title:"Objectives of this notebook \u{1F3C6}"},{local:"prerequisites",title:"Prerequisites \u{1F3D7}\uFE0F"},{local:"set-the-gpu",title:"Set the GPU \u{1F4AA}"},{local:"clone-the-repository-and-install-the-dependencies",title:"Clone the repository and install the dependencies \u{1F53D}"},{local:"download-and-move-the-environment-zip-file-in-trainedenvsexecutableslinux",title:"Download and move the environment zip file in `./trained-envs-executables/linux/`"},{local:"lets-recap-how-this-environment-works",sections:[{local:"the-state-space-what-huggy-perceives",title:"The State Space: what Huggy perceives."},{local:"the-action-space-what-moves-huggy-can-do",title:"The Action Space: what moves Huggy can do"},{local:"the-reward-function",title:"The Reward Function"}],title:"Let's recap how this environment works"},{local:"check-the-huggy-config-file",title:"Check the Huggy config file"},{local:"train-our-agent",title:"Train our agent"},{local:"push-the-agent-to-the-hub",title:"Push the agent to the \u{1F917} Hub"},{local:"play-with-your-huggy",title:"Play with your Huggy \u{1F415}"},{local:"keep-learning-stay-awesome",title:"Keep Learning, Stay awesome \u{1F917}"}],title:"Let's train and play with Huggy \u{1F436} "};function _y(Bf){return dy(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ky extends fy{constructor(G){super();cy(this,G,_y,wy,gy,{})}}export{ky as default,by as metadata};
