import{S as dw,i as mw,s as vw,e as l,k as u,w as p,t as r,M as ww,c as a,d as t,m as f,a as s,x as d,h as i,b as c,N as Ge,G as o,g as h,y as m,L as gw,q as v,o as w,B as g,v as yw}from"../../chunks/vendor-hf-doc-builder.js";import{I as b}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as ee}from"../../chunks/CodeBlock-hf-doc-builder.js";function bw(Kp){let P,ps,N,te,yl,Be,rn,bl,nn,ds,oe,hn,El,un,fn,ms,Oe,cn,_l,pn,vs,He,dn,Tl,mn,ws,Re,vn,Ze,wn,gs,le,gn,Ml,yn,bn,ys,co,En,bs,C,ae,Al,ze,_n,Il,Tn,Es,po,Mn,_s,Ye,An,Sl,In,Ts,mo,Sn,Ms,I,De,$n,$l,kn,jn,Jn,se,Un,kl,Pn,Nn,Fe,Cn,xn,re,Ln,jl,Wn,Gn,Ve,Bn,As,Xe,On,qe,Hn,Is,ie,Rn,Qe,Zn,zn,Ss,E,vo,Yn,Jl,Dn,Fn,wo,Vn,Ul,Xn,qn,Ke,Qn,Pl,Kn,eh,th,go,oh,Nl,lh,$s,x,ne,Cl,et,ah,xl,sh,ks,_,rh,Ll,ih,nh,Wl,hh,uh,Gl,fh,ch,js,yo,ph,Js,S,bo,Bl,dh,mh,vh,tt,wh,Ol,gh,yh,bh,ot,Eh,Hl,_h,Th,Us,he,Mh,Rl,Ah,Ih,Ps,$,Sh,Zl,$h,kh,lt,jh,Jh,Ns,L,ue,zl,at,Uh,Yl,Ph,Cs,fe,st,Nh,Dl,Ch,xh,Lh,Eo,Wh,rt,Gh,xs,W,ce,Fl,it,Bh,Vl,Oh,Ls,pe,Hh,nt,Rh,Zh,Ws,de,zh,Xl,Yh,Dh,Gs,ht,Bs,_o,Fh,Os,ut,Hs,To,Vh,Rs,ft,Zs,Mo,Xh,zs,ct,Ys,pt,qh,dt,Qh,Ds,G,Kh,ql,eu,tu,Ql,ou,Fs,mt,lu,Kl,au,Vs,vt,su,wt,ru,Xs,gt,iu,yt,nu,qs,k,hu,bt,uu,fu,ea,cu,pu,Qs,B,me,ta,Et,du,oa,mu,Ks,O,vu,la,wu,gu,_t,yu,er,Tt,bu,aa,Eu,tr,ve,Ao,ed,_u,Io,Tu,So,Mu,or,H,we,sa,Mt,Au,ra,Iu,lr,$o,Su,ar,At,td,sr,R,ge,ia,It,$u,na,ku,rr,ko,ju,ir,j,ha,Ju,Uu,ua,Pu,Nu,jo,Cu,y,fa,xu,Lu,ca,Wu,Gu,pa,Bu,Ou,da,Hu,Ru,ma,Zu,zu,va,Yu,nr,Z,ye,wa,St,Du,ga,Fu,hr,Jo,Vu,ur,$t,od,fr,z,be,ya,kt,Xu,ba,qu,cr,Ee,Qu,Ea,Ku,ef,pr,jt,tf,_a,of,dr,J,lf,Jt,af,sf,Ta,rf,nf,mr,_e,hf,Ma,uf,ff,vr,Te,cf,Aa,pf,df,wr,U,mf,Ia,vf,wf,Sa,gf,yf,gr,Me,Uo,ld,bf,Po,Ef,No,_f,yr,Co,Tf,br,Ae,Mf,Ut,Af,If,Er,Y,Ie,$a,Pt,Sf,ka,$f,_r,D,kf,Nt,jf,Jf,Ct,Uf,ja,Pf,Nf,Tr,xt,Cf,Ja,Lt,xf,Mr,Se,Lf,Ua,Wf,Gf,Ar,Wt,Ir,xo,Bf,Sr,Gt,Of,Lo,Bt,Hf,Rf,$r,Wo,Zf,kr,F,$e,Pa,Ot,zf,Na,Yf,jr,Ht,Df,Ca,Ff,Jr,Go,Vf,Ur,T,Bo,xa,Xf,qf,Qf,Oo,La,Kf,ec,tc,Ho,Wa,oc,lc,ac,Ro,Ga,sc,rc,Pr,Zo,ic,Nr,zo,nc,Cr,Rt,xr,Yo,hc,Lr,Do,uc,Wr,Fo,fc,Gr,V,ke,Ba,Zt,cc,Oa,pc,Br,zt,dc,Ha,mc,Or,Vo,vc,Hr,Yt,wc,Dt,gc,Rr,Xo,yc,Zr,X,bc,Ft,Ec,_c,Ra,Tc,zr,Vt,ad,Yr,qo,Mc,Dr,Xt,Fr,je,Ac,Za,Ic,Sc,Vr,Qo,$c,Xr,M,Ko,za,kc,jc,Jc,el,Ya,Uc,Pc,Nc,qt,Da,Cc,xc,Fa,Lc,Wc,tl,Va,Gc,Bc,qr,ol,Oc,Qr,Qt,Kr,Kt,ei,ll,Hc,ti,eo,Rc,to,Zc,oi,oo,zc,Xa,Yc,li,q,Je,qa,lo,Dc,Qa,Fc,ai,Ue,Vc,Ka,Xc,qc,si,al,Qc,ri,sl,es,Kc,ii,ao,sd,ni,rl,ep,hi,so,rd,ui,ro,io,tp,ts,op,lp,fi,no,id,ci,il,ap,pi,Q,Pe,os,ho,sp,ls,rp,di,K,ip,as,np,hp,uo,up,mi,nl,fp,vi,hl,ul,cp,fo,pp,wi,Ne,dp,ss,mp,vp,gi,fl,wp,yi;return Be=new b({}),ze=new b({}),et=new b({}),at=new b({}),it=new b({}),ht=new ee({props:{code:"Y29uZGElMjBjcmVhdGUlMjAtLW5hbWUlMjBybCUyMHB5dGhvbiUzRDMuOSUwQWNvbmRhJTIwYWN0aXZhdGUlMjBybA==",highlighted:`conda create --name rl python=3.9
conda activate rl`}}),ut=new ee({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZVbml0eS1UZWNobm9sb2dpZXMlMkZtbC1hZ2VudHM=",highlighted:'git <span class="hljs-built_in">clone</span> https://github.com/Unity-Technologies/ml-agents'}}),ft=new ee({props:{code:"Y2QlMjBtbC1hZ2VudHMlMEFwaXAlMjBpbnN0YWxsJTIwLWUlMjAuJTJGbWwtYWdlbnRzLWVudnMlMEFwaXAlMjBpbnN0YWxsJTIwLWUlMjAuJTJGbWwtYWdlbnRz",highlighted:`<span class="hljs-built_in">cd</span> ml-agents
pip install -e ./ml-agents-envs
pip install -e ./ml-agents`}}),ct=new ee({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRvcmNo",highlighted:"pip install torch"}}),Et=new b({}),Mt=new b({}),It=new b({}),St=new b({}),kt=new b({}),Pt=new b({}),Wt=new ee({props:{code:"YmVoYXZpb3JzJTNBJTBBJTIwJTIwU29jY2VyVHdvcyUzQSUwQSUyMCUyMCUyMCUyMHRyYWluZXJfdHlwZSUzQSUyMHBvY2ElMEElMjAlMjAlMjAlMjBoeXBlcnBhcmFtZXRlcnMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjBiYXRjaF9zaXplJTNBJTIwMjA0OCUwQSUyMCUyMCUyMCUyMCUyMCUyMGJ1ZmZlcl9zaXplJTNBJTIwMjA0ODAlMEElMjAlMjAlMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNBJTIwMC4wMDAzJTBBJTIwJTIwJTIwJTIwJTIwJTIwYmV0YSUzQSUyMDAuMDA1JTBBJTIwJTIwJTIwJTIwJTIwJTIwZXBzaWxvbiUzQSUyMDAuMiUwQSUyMCUyMCUyMCUyMCUyMCUyMGxhbWJkJTNBJTIwMC45NSUwQSUyMCUyMCUyMCUyMCUyMCUyMG51bV9lcG9jaCUzQSUyMDMlMEElMjAlMjAlMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlX3NjaGVkdWxlJTNBJTIwY29uc3RhbnQlMEElMjAlMjAlMjAlMjBuZXR3b3JrX3NldHRpbmdzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwbm9ybWFsaXplJTNBJTIwZmFsc2UlMEElMjAlMjAlMjAlMjAlMjAlMjBoaWRkZW5fdW5pdHMlM0ElMjA1MTIlMEElMjAlMjAlMjAlMjAlMjAlMjBudW1fbGF5ZXJzJTNBJTIwMiUwQSUyMCUyMCUyMCUyMCUyMCUyMHZpc19lbmNvZGVfdHlwZSUzQSUyMHNpbXBsZSUwQSUyMCUyMCUyMCUyMHJld2FyZF9zaWduYWxzJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwZXh0cmluc2ljJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ2FtbWElM0ElMjAwLjk5JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3RyZW5ndGglM0ElMjAxLjAlMEElMjAlMjAlMjAlMjBrZWVwX2NoZWNrcG9pbnRzJTNBJTIwNSUwQSUyMCUyMCUyMCUyMG1heF9zdGVwcyUzQSUyMDUwMDAwMDAlMEElMjAlMjAlMjAlMjB0aW1lX2hvcml6b24lM0ElMjAxMDAwJTBBJTIwJTIwJTIwJTIwc3VtbWFyeV9mcmVxJTNBJTIwMTAwMDAlMEElMjAlMjAlMjAlMjBzZWxmX3BsYXklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjBzYXZlX3N0ZXBzJTNBJTIwNTAwMDAlMEElMjAlMjAlMjAlMjAlMjAlMjB0ZWFtX2NoYW5nZSUzQSUyMDIwMDAwMCUwQSUyMCUyMCUyMCUyMCUyMCUyMHN3YXBfc3RlcHMlM0ElMjAyMDAwJTBBJTIwJTIwJTIwJTIwJTIwJTIwd2luZG93JTNBJTIwMTAlMEElMjAlMjAlMjAlMjAlMjAlMjBwbGF5X2FnYWluc3RfbGF0ZXN0X21vZGVsX3JhdGlvJTNBJTIwMC41JTBBJTIwJTIwJTIwJTIwJTIwJTIwaW5pdGlhbF9lbG8lM0ElMjAxMjAwLjA=",highlighted:`behaviors:
  SoccerTwos:
    trainer_type: poca
    hyperparameters:
      batch_size: <span class="hljs-number">2048</span>
      buffer_size: <span class="hljs-number">20480</span>
      learning_rate: <span class="hljs-number">0.0003</span>
      beta: <span class="hljs-number">0.005</span>
      epsilon: <span class="hljs-number">0.2</span>
      lambd: <span class="hljs-number">0.95</span>
      num_epoch: <span class="hljs-number">3</span>
      learning_rate_schedule: constant
    network_settings:
      normalize: <span class="hljs-literal">false</span>
      hidden_units: <span class="hljs-number">512</span>
      num_layers: <span class="hljs-number">2</span>
      vis_encode_type: simple
    reward_signals:
      extrinsic:
        gamma: <span class="hljs-number">0.99</span>
        strength: <span class="hljs-number">1.0</span>
    keep_checkpoints: <span class="hljs-number">5</span>
    max_steps: <span class="hljs-number">5000000</span>
    time_horizon: <span class="hljs-number">1000</span>
    summary_freq: <span class="hljs-number">10000</span>
    self_play:
      save_steps: <span class="hljs-number">50000</span>
      team_change: <span class="hljs-number">200000</span>
      swap_steps: <span class="hljs-number">2000</span>
      window: <span class="hljs-number">10</span>
      play_against_latest_model_ratio: <span class="hljs-number">0.5</span>
      initial_elo: <span class="hljs-number">1200.0</span>`}}),Ot=new b({}),Rt=new ee({props:{code:"bWxhZ2VudHMtbGVhcm4lMjAuJTJGY29uZmlnJTJGcG9jYSUyRlNvY2NlclR3b3MueWFtbCUyMC0tZW52JTNELiUyRnRyYWluaW5nLWVudnMtZXhlY3V0YWJsZXMlMkZTb2NjZXJUd29zLmV4ZSUyMC0tcnVuLWlkJTNEJTIyU29jY2VyVHdvcyUyMiUyMC0tbm8tZ3JhcGhpY3M=",highlighted:'mlagents-learn ./config/poca/SoccerTwos.yaml --<span class="hljs-built_in">env</span>=./training-envs-executables/SoccerTwos.exe --run-id=<span class="hljs-string">&quot;SoccerTwos&quot;</span> --no-graphics'}}),Zt=new b({}),Xt=new ee({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:"huggingface-cli login"}}),Qt=new ee({props:{code:"bWxhZ2VudHMtcHVzaC10by1oZiUyMCUyMC0tcnVuLWlkJTNEJTIyU29jY2VyVHdvcyUyMiUyMC0tbG9jYWwtZGlyJTNEJTIyLiUyRnJlc3VsdHMlMkZTb2NjZXJUd29zJTIyJTIwLS1yZXBvLWlkJTNEJTIyVGhvbWFzU2ltb25pbmklMkZwb2NhLVNvY2NlclR3b3MlMjIlMjAtLWNvbW1pdC1tZXNzYWdlJTNEJTIyRmlyc3QlMjBQdXNoJTIyJTYw",highlighted:'mlagents-push-to-hf  --run-id=<span class="hljs-string">&quot;SoccerTwos&quot;</span> --local-dir=<span class="hljs-string">&quot;./results/SoccerTwos&quot;</span> --repo-id=<span class="hljs-string">&quot;ThomasSimonini/poca-SoccerTwos&quot;</span> --commit-message=<span class="hljs-string">&quot;First Push&quot;</span>`'}}),Kt=new ee({props:{code:"bWxhZ2VudHMtcHVzaC10by1oZiUyMCUyMC0tcnVuLWlkJTNEJTIwJTIzJTIwQWRkJTIweW91ciUyMHJ1biUyMGlkJTIwJTIwLS1sb2NhbC1kaXIlM0QlMjAlMjMlMjBZb3VyJTIwbG9jYWwlMjBkaXIlMjAlMjAtLXJlcG8taWQlM0QlMjAlMjMlMjBZb3VyJTIwcmVwbyUyMGlkJTIwLS1jb21taXQtbWVzc2FnZSUzRCUyMkZpcnN0JTIwUHVzaCUyMg==",highlighted:'mlagents-push-to-hf  --run-id= <span class="hljs-comment"># Add your run id  --local-dir= # Your local dir  --repo-id= # Your repo id --commit-message=&quot;First Push&quot;</span>'}}),lo=new b({}),ho=new b({}),{c(){P=l("meta"),ps=u(),N=l("h1"),te=l("a"),yl=l("span"),p(Be.$$.fragment),rn=u(),bl=l("span"),nn=r("Hands-on"),ds=u(),oe=l("p"),hn=r("Now that you learned the basics of multi-agents, you\u2019re ready to train your first agents in a multi-agent system: "),El=l("strong"),un=r("a 2vs2 soccer team that needs to beat the opponent team"),fn=r("."),ms=u(),Oe=l("p"),cn=r("And you\u2019re going to participate in AI vs. AI challenges where your trained agent will compete against other classmates\u2019 "),_l=l("strong"),pn=r("agents every day and be ranked on a new leaderboard."),vs=u(),He=l("p"),dn=r("To validate this hands-on for the certification process, you just need to push a trained model. There "),Tl=l("strong"),mn=r("are no minimal results to attain to validate it."),ws=u(),Re=l("p"),vn=r("For more information about the certification process, check this section \u{1F449} "),Ze=l("a"),wn=r("https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),gs=u(),le=l("p"),gn=r("This hands-on will be different since to get correct results "),Ml=l("strong"),yn=r("you need to train your agents from 4 hours to 8 hours"),bn=r(". And given the risk of timeout in Colab, we advise you to train on your computer. You don\u2019t need a supercomputer: a simple laptop is good enough for this exercise."),ys=u(),co=l("p"),En=r("Let\u2019s get started! \u{1F525}"),bs=u(),C=l("h2"),ae=l("a"),Al=l("span"),p(ze.$$.fragment),_n=u(),Il=l("span"),Tn=r("What is AI vs. AI?"),Es=u(),po=l("p"),Mn=r("AI vs. AI is an open-source tool we developed at Hugging Face to compete agents on the Hub against one another in a multi-agent setting. These models are then ranked in a leaderboard."),_s=u(),Ye=l("p"),An=r("The idea of this tool is to have a robust evaluation tool: "),Sl=l("strong"),In=r("by evaluating your agent with a lot of others, you\u2019ll get a good idea of the quality of your policy."),Ts=u(),mo=l("p"),Sn=r("More precisely, AI vs. AI is three tools:"),Ms=u(),I=l("ul"),De=l("li"),$n=r("A "),$l=l("em"),kn=r("matchmaking process"),jn=r(" defining the matches (which model against which) and running the model fights using a background task in the Space."),Jn=u(),se=l("li"),Un=r("A "),kl=l("em"),Pn=r("leaderboard"),Nn=r(" getting the match history results and displaying the models\u2019 ELO ratings: "),Fe=l("a"),Cn=r("https://huggingface.co/spaces/huggingface-projects/AIvsAI-SoccerTwos"),xn=u(),re=l("li"),Ln=r("A "),jl=l("em"),Wn=r("Space demo"),Gn=r(" to visualize your agents playing against others: "),Ve=l("a"),Bn=r("https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos"),As=u(),Xe=l("p"),On=r("In addition to these three tools, your classmate cyllum created a \u{1F917} SoccerTwos Challenge Analytics where you can check the detailed match results of a model: "),qe=l("a"),Hn=r("https://huggingface.co/spaces/cyllum/soccertwos-analytics"),Is=u(),ie=l("p"),Rn=r("We\u2019re "),Qe=l("a"),Zn=r("wrote a blog post to explain this AI vs. AI tool in detail"),zn=r(", but to give you the big picture it works this way:"),Ss=u(),E=l("ul"),vo=l("li"),Yn=r("Every four hours, our algorithm "),Jl=l("strong"),Dn=r("fetches all the available models for a given environment (in our case ML-Agents-SoccerTwos)."),Fn=u(),wo=l("li"),Vn=r("It creates a "),Ul=l("strong"),Xn=r("queue of matches with the matchmaking algorithm."),qn=u(),Ke=l("li"),Qn=r("We simulate the match in a Unity headless process and "),Pl=l("strong"),Kn=r("gather the match result"),eh=r(" (1 if the first model won, 0.5 if it\u2019s a draw, 0 if the second model won) in a Dataset."),th=u(),go=l("li"),oh=r("Then, when all matches from the matches queue are done, "),Nl=l("strong"),lh=r("we update the ELO score for each model and update the leaderboard."),$s=u(),x=l("h3"),ne=l("a"),Cl=l("span"),p(et.$$.fragment),ah=u(),xl=l("span"),sh=r("Competition Rules"),ks=u(),_=l("p"),rh=r("This first AI vs. AI competition "),Ll=l("strong"),ih=r("is an experiment"),nh=r(": the goal is to improve the tool in the future with your feedback. So some "),Wl=l("strong"),hh=r("breakups can happen during the challenge"),uh=r(`. But don\u2019t worry
`),Gl=l("strong"),fh=r("all the results are saved in a dataset so we can always restart the calculation correctly without losing information"),ch=r("."),js=u(),yo=l("p"),ph=r("In order for your model to get correctly evaluated against others you need to follow these rules:"),Js=u(),S=l("ol"),bo=l("li"),Bl=l("strong"),dh=r("You can\u2019t change the observation space or action space of the agent."),mh=r(" By doing that your model will not work during evaluation."),vh=u(),tt=l("li"),wh=r("You "),Ol=l("strong"),gh=r("can\u2019t use a custom trainer for now,"),yh=r(" you need to use the Unity MLAgents ones."),bh=u(),ot=l("li"),Eh=r("We provide executables to train your agents. You can also use the Unity Editor if you prefer "),Hl=l("strong"),_h=r(", but to avoid bugs, we advise that you use our executables"),Th=r("."),Us=u(),he=l("p"),Mh=r("What will make the difference during this challenge are "),Rl=l("strong"),Ah=r("the hyperparameters you choose"),Ih=r("."),Ps=u(),$=l("p"),Sh=r("We\u2019re constantly trying to improve our tutorials, so\xA0"),Zl=l("strong"),$h=r("if you find some issues in this notebook"),kh=r(", please\xA0"),lt=l("a"),jh=r("open an issue on the GitHub Repo"),Jh=r("."),Ns=u(),L=l("h3"),ue=l("a"),zl=l("span"),p(at.$$.fragment),Uh=u(),Yl=l("span"),Ph=r("Chat with your classmates, share advice and ask questions on Discord"),Cs=u(),fe=l("ul"),st=l("li"),Nh=r("We created a new channel called "),Dl=l("code"),Ch=r("ai-vs-ai-challenge"),xh=r(" to exchange advice and ask questions."),Lh=u(),Eo=l("li"),Wh=r("If you didn\u2019t join the discord server yet, you can "),rt=l("a"),Gh=r("join here"),xs=u(),W=l("h2"),ce=l("a"),Fl=l("span"),p(it.$$.fragment),Bh=u(),Vl=l("span"),Oh=r("Step 0: Install MLAgents and download the correct executable"),Ls=u(),pe=l("p"),Hh=r("We advise you to use "),nt=l("a"),Rh=r("conda"),Zh=r(" as a package manager and create a new environment."),Ws=u(),de=l("p"),zh=r("With conda, we create a new environment called rl with "),Xl=l("strong"),Yh=r("Python 3.9"),Dh=r(":"),Gs=u(),p(ht.$$.fragment),Bs=u(),_o=l("p"),Fh=r("To be able to train our agents correctly and push to the Hub, we need to install ML-Agents"),Os=u(),p(ut.$$.fragment),Hs=u(),To=l("p"),Vh=r("When the cloning is done (it takes 2.63 GB), we go inside the repository and install the package"),Rs=u(),p(ft.$$.fragment),Zs=u(),Mo=l("p"),Xh=r("We also need to install pytorch with:"),zs=u(),p(ct.$$.fragment),Ys=u(),pt=l("p"),qh=r("Finally, you need to install git-lfs: "),dt=l("a"),Qh=r("https://git-lfs.com/"),Ds=u(),G=l("p"),Kh=r("Now that it\u2019s installed, we need to add the environment training executable. Based on your operating system you need to download one of them, unzip it and place it in a new folder inside "),ql=l("code"),eu=r("ml-agents"),tu=r(" that you call "),Ql=l("code"),ou=r("training-envs-executables"),Fs=u(),mt=l("p"),lu=r("At the end your executable should be in "),Kl=l("code"),au=r("ml-agents/training-envs-executables/SoccerTwos"),Vs=u(),vt=l("p"),su=r("Windows: Download "),wt=l("a"),ru=r("this executable"),Xs=u(),gt=l("p"),iu=r("Linux (Ubuntu): Download "),yt=l("a"),nu=r("this executable"),qs=u(),k=l("p"),hu=r("Mac: Download "),bt=l("a"),uu=r("this executable"),fu=r(`
\u26A0 For Mac you need also to call this `),ea=l("code"),cu=r("xattr -cr training-envs-executables/SoccerTwos/SoccerTwos.app"),pu=r(" to be able to run SoccerTwos"),Qs=u(),B=l("h2"),me=l("a"),ta=l("span"),p(Et.$$.fragment),du=u(),oa=l("span"),mu=r("Step 1: Understand the environment"),Ks=u(),O=l("p"),vu=r("The environment is called "),la=l("code"),wu=r("SoccerTwos"),gu=r(". The Unity MLAgents Team made it. You can find its documentation "),_t=l("a"),yu=r("here"),er=u(),Tt=l("p"),bu=r("The goal in this environment "),aa=l("strong"),Eu=r("is to get the ball into the opponent\u2019s goal while preventing the ball from entering your own goal."),tr=u(),ve=l("figure"),Ao=l("img"),_u=u(),Io=l("figcaption"),Tu=r("This environment was made by the "),So=l("a"),Mu=r("Unity MLAgents Team"),or=u(),H=l("h3"),we=l("a"),sa=l("span"),p(Mt.$$.fragment),Au=u(),ra=l("span"),Iu=r("The reward function"),lr=u(),$o=l("p"),Su=r("The reward function is:"),ar=u(),At=l("img"),sr=u(),R=l("h3"),ge=l("a"),ia=l("span"),p(It.$$.fragment),$u=u(),na=l("span"),ku=r("The observation space"),rr=u(),ko=l("p"),ju=r("The observation space is composed of vectors of size 336:"),ir=u(),j=l("ul"),ha=l("li"),Ju=r("11 ray-casts forward distributed over 120 degrees (264 state dimensions)"),Uu=u(),ua=l("li"),Pu=r("3 ray-casts backward distributed over 90 degrees (72 state dimensions)"),Nu=u(),jo=l("li"),Cu=r("Both of these ray-casts can detect 6 objects:"),y=l("ul"),fa=l("li"),xu=r("Ball"),Lu=u(),ca=l("li"),Wu=r("Blue Goal"),Gu=u(),pa=l("li"),Bu=r("Purple Goal"),Ou=u(),da=l("li"),Hu=r("Wall"),Ru=u(),ma=l("li"),Zu=r("Blue Agent"),zu=u(),va=l("li"),Yu=r("Purple Agent"),nr=u(),Z=l("h3"),ye=l("a"),wa=l("span"),p(St.$$.fragment),Du=u(),ga=l("span"),Fu=r("The action space"),hr=u(),Jo=l("p"),Vu=r("The action space is three discrete branches:"),ur=u(),$t=l("img"),fr=u(),z=l("h2"),be=l("a"),ya=l("span"),p(kt.$$.fragment),Xu=u(),ba=l("span"),qu=r("Step 2: Understand MA-POCA"),cr=u(),Ee=l("p"),Qu=r("We know how to train agents to play against others: "),Ea=l("strong"),Ku=r("we can use self-play."),ef=r(" This is a perfect technique for a 1vs1."),pr=u(),jt=l("p"),tf=r("But in our case we\u2019re 2vs2, and each team has 2 agents. How then can we "),_a=l("strong"),of=r("train cooperative behavior for groups of agents?"),dr=u(),J=l("p"),lf=r("As explained in the "),Jt=l("a"),af=r("Unity Blog"),sf=r(", agents typically receive a reward as a group (+1 - penalty) when the team scores a goal. This implies that "),Ta=l("strong"),rf=r("every agent on the team is rewarded even if each agent didn\u2019t contribute the same to the win"),nf=r(", which makes it difficult to learn what to do independently."),mr=u(),_e=l("p"),hf=r("The Unity MLAgents team developed the solution in a new multi-agent trainer called "),Ma=l("em"),uf=r("MA-POCA (Multi-Agent POsthumous Credit Assignment)"),ff=r("."),vr=u(),Te=l("p"),cf=r("The idea is simple but powerful: a centralized critic "),Aa=l("strong"),pf=r("processes the states of all agents in the team to estimate how well each agent is doing"),df=r(". Think of this critic as a coach."),wr=u(),U=l("p"),mf=r("This allows each agent to "),Ia=l("strong"),vf=r("make decisions based only on what it perceives locally"),wf=r(", and "),Sa=l("strong"),gf=r("simultaneously evaluate how good its behavior is in the context of the whole group"),yf=r("."),gr=u(),Me=l("figure"),Uo=l("img"),bf=u(),Po=l("figcaption"),Ef=r("This illustrates MA-POCA\u2019s centralized learning and decentralized execution. Source: "),No=l("a"),_f=r("MLAgents Plays Dodgeball"),yr=u(),Co=l("p"),Tf=r("The solution then is to use Self-Play with an MA-POCA trainer (called poca). The poca trainer will help us to train cooperative behavior and self-play to win against an opponent team."),br=u(),Ae=l("p"),Mf=r("If you want to dive deeper into this MA-POCA algorithm, you need to read the paper they published "),Ut=l("a"),Af=r("here"),If=r(" and the sources we put on the additional readings section."),Er=u(),Y=l("h2"),Ie=l("a"),$a=l("span"),p(Pt.$$.fragment),Sf=u(),ka=l("span"),$f=r("Step 3: Define the config file"),_r=u(),D=l("p"),kf=r("We already learned in "),Nt=l("a"),jf=r("Unit 5"),Jf=r(" that in ML-Agents, you define "),Ct=l("strong"),Uf=r("the training hyperparameters in "),ja=l("code"),Pf=r("config.yaml"),Nf=r(" files."),Tr=u(),xt=l("p"),Cf=r("There are multiple hyperparameters. To understand them better, you should read the explanations for each of them in\xA0"),Ja=l("strong"),Lt=l("a"),xf=r("the documentation"),Mr=u(),Se=l("p"),Lf=r("The config file we\u2019re going to use here is in  "),Ua=l("code"),Wf=r("./config/poca/SoccerTwos.yaml"),Gf=r(". It looks like this:"),Ar=u(),p(Wt.$$.fragment),Ir=u(),xo=l("p"),Bf=r("Compared to Pyramids or SnowballTarget, we have new hyperparameters with a self-play part. How you modify them can be critical in getting good results."),Sr=u(),Gt=l("p"),Of=r("The advice I can give you here is to check the explanation and recommended value for each parameters (especially self-play ones) against\xA0"),Lo=l("strong"),Bt=l("a"),Hf=r("the documentation"),Rf=r("."),$r=u(),Wo=l("p"),Zf=r("Now that you\u2019ve modified our config file, you\u2019re ready to train your agents."),kr=u(),F=l("h2"),$e=l("a"),Pa=l("span"),p(Ot.$$.fragment),zf=u(),Na=l("span"),Yf=r("Step 4: Start the training"),jr=u(),Ht=l("p"),Df=r("To train the agents, we need to\xA0"),Ca=l("strong"),Ff=r("launch mlagents-learn and select the executable containing the environment."),Jr=u(),Go=l("p"),Vf=r("We define four parameters:"),Ur=u(),T=l("ol"),Bo=l("li"),xa=l("code"),Xf=r("mlagents-learn <config>"),qf=r(": the path where the hyperparameter config file is."),Qf=u(),Oo=l("li"),La=l("code"),Kf=r("-env"),ec=r(": where the environment executable is."),tc=u(),Ho=l("li"),Wa=l("code"),oc=r("-run_id"),lc=r(": the name you want to give to your training run id."),ac=u(),Ro=l("li"),Ga=l("code"),sc=r("-no-graphics"),rc=r(": to not launch the visualization during the training."),Pr=u(),Zo=l("p"),ic=r("Depending on your hardware, 5M timesteps (the recommended value, but you can also try 10M) will take 5 to 8 hours of training. You can continue using your computer in the meantime, but I advise deactivating the computer standby mode to prevent the training from being stopped."),Nr=u(),zo=l("p"),nc=r("Depending on the executable you use (windows, ubuntu, mac) the training command will look like this (your executable path can be different so don\u2019t hesitate to check before running)."),Cr=u(),p(Rt.$$.fragment),xr=u(),Yo=l("p"),hc=r("The executable contains 8 copies of SoccerTwos."),Lr=u(),Do=l("p"),uc=r("\u26A0\uFE0F It\u2019s normal if you don\u2019t see a big increase of ELO score (and even a decrease below 1200) before 2M timesteps, since your agents will spend most of their time moving randomly on the field before being able to goal."),Wr=u(),Fo=l("p"),fc=r("\u26A0\uFE0F You can stop the training with Ctrl + C but beware of typing this command only once to stop the training since MLAgents needs to generate a final .onnx file before closing the run."),Gr=u(),V=l("h2"),ke=l("a"),Ba=l("span"),p(Zt.$$.fragment),cc=u(),Oa=l("span"),pc=r("Step 5: **Push the agent to the Hugging Face Hub**"),Br=u(),zt=l("p"),dc=r("Now that we trained our agents, we\u2019re\xA0"),Ha=l("strong"),mc=r("ready to push them to the Hub to be able to participate in the AI vs. AI challenge and visualize them playing on your browser\u{1F525}."),Or=u(),Vo=l("p"),vc=r("To be able to share your model with the community, there are three more steps to follow:"),Hr=u(),Yt=l("p"),wc=r("1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1\xA0"),Dt=l("a"),gc=r("https://huggingface.co/join"),Rr=u(),Xo=l("p"),yc=r("2\uFE0F\u20E3 Sign in and store your authentication token from the Hugging Face website."),Zr=u(),X=l("p"),bc=r("Create a new token ("),Ft=l("a"),Ec=r("https://huggingface.co/settings/tokens"),_c=r(")\xA0"),Ra=l("strong"),Tc=r("with write role"),zr=u(),Vt=l("img"),Yr=u(),qo=l("p"),Mc=r("Copy the token, run this, and paste the token"),Dr=u(),p(Xt.$$.fragment),Fr=u(),je=l("p"),Ac=r("Then, we need to run "),Za=l("code"),Ic=r("mlagents-push-to-hf"),Sc=r("."),Vr=u(),Qo=l("p"),$c=r("And we define four parameters:"),Xr=u(),M=l("ol"),Ko=l("li"),za=l("code"),kc=r("-run-id"),jc=r(": the name of the training run id."),Jc=u(),el=l("li"),Ya=l("code"),Uc=r("-local-dir"),Pc=r(": where the agent was saved, it\u2019s results/<run_id name>, so in my case results/First Training."),Nc=u(),qt=l("li"),Da=l("code"),Cc=r("-repo-id"),xc=r(`: the name of the Hugging Face repo you want to create or update. It\u2019s always <your huggingface username>/<the repo name>
If the repo does not exist `),Fa=l("strong"),Lc=r("it will be created automatically"),Wc=u(),tl=l("li"),Va=l("code"),Gc=r("--commit-message"),Bc=r(": since HF repos are git repositories you need to give a commit message."),qr=u(),ol=l("p"),Oc=r("In my case"),Qr=u(),p(Qt.$$.fragment),Kr=u(),p(Kt.$$.fragment),ei=u(),ll=l("p"),Hc=r("If everything worked you should see this at the end of the process (but with a different url \u{1F606}) :"),ti=u(),eo=l("p"),Rc=r("Your model is pushed to the Hub. You can view your model here: "),to=l("a"),Zc=r("https://huggingface.co/ThomasSimonini/poca-SoccerTwos"),oi=u(),oo=l("p"),zc=r("It\u2019s the link to your model. It contains a model card that explains how to use it, your Tensorboard, and your config file. "),Xa=l("strong"),Yc=r("What\u2019s awesome is that it\u2019s a git repository, which means you can have different commits, update your repository with a new push, etc."),li=u(),q=l("h2"),Je=l("a"),qa=l("span"),p(lo.$$.fragment),Dc=u(),Qa=l("span"),Fc=r("Step 6: Verify that your model is ready for AI vs AI Challenge"),ai=u(),Ue=l("p"),Vc=r("Now that your model is pushed to the Hub, "),Ka=l("strong"),Xc=r("it\u2019s going to be added automatically to the AI vs AI Challenge model pool."),qc=r(" It can take a little bit of time before your model is added to the leaderboard given we do a run of matches every 4h."),si=u(),al=l("p"),Qc=r("But to ensure that everything works perfectly you need to check:"),ri=u(),sl=l("ol"),es=l("li"),Kc=r("That you have this tag in your model: ML-Agents-SoccerTwos. This is the tag we use to select models to be added to the challenge pool. To do that go to your model and check the tags"),ii=u(),ao=l("img"),ni=u(),rl=l("p"),ep=r("If it\u2019s not the case you just need to modify the readme and add it"),hi=u(),so=l("img"),ui=u(),ro=l("ol"),io=l("li"),tp=r("That you have a "),ts=l("code"),op=r("SoccerTwos.onnx"),lp=r(" file"),fi=u(),no=l("img"),ci=u(),il=l("p"),ap=r("We strongly suggest that you create a new model when you push to the Hub if you want to train it again or train a new version."),pi=u(),Q=l("h2"),Pe=l("a"),os=l("span"),p(ho.$$.fragment),sp=u(),ls=l("span"),rp=r("Step 7: Visualize some match in our demo"),di=u(),K=l("p"),ip=r("Now that your model is part of AI vs AI Challenge, "),as=l("strong"),np=r("you can visualize how good it is compared to others"),hp=r(": "),uo=l("a"),up=r("https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos"),mi=u(),nl=l("p"),fp=r("In order to do that, you just need to go to this demo:"),vi=u(),hl=l("ul"),ul=l("li"),cp=r("Select your model as team blue (or team purple if you prefer) and another model to compete against. The best opponents to compare your model to are either whoever is on top of the leaderboard or the "),fo=l("a"),pp=r("baseline model"),wi=u(),Ne=l("p"),dp=r("The matches you see live are not used in the calculation of your result "),ss=l("strong"),mp=r("but they are a good way to visualize how good your agent is"),vp=r("."),gi=u(),fl=l("p"),wp=r("And don\u2019t hesitate to share the best score your agent gets on discord in the #rl-i-made-this channel \u{1F525}"),this.h()},l(e){const n=ww('[data-svelte="svelte-1phssyn"]',document.head);P=a(n,"META",{name:!0,content:!0}),n.forEach(t),ps=f(e),N=a(e,"H1",{class:!0});var bi=s(N);te=a(bi,"A",{id:!0,class:!0,href:!0});var nd=s(te);yl=a(nd,"SPAN",{});var hd=s(yl);d(Be.$$.fragment,hd),hd.forEach(t),nd.forEach(t),rn=f(bi),bl=a(bi,"SPAN",{});var ud=s(bl);nn=i(ud,"Hands-on"),ud.forEach(t),bi.forEach(t),ds=f(e),oe=a(e,"P",{});var Ei=s(oe);hn=i(Ei,"Now that you learned the basics of multi-agents, you\u2019re ready to train your first agents in a multi-agent system: "),El=a(Ei,"STRONG",{});var fd=s(El);un=i(fd,"a 2vs2 soccer team that needs to beat the opponent team"),fd.forEach(t),fn=i(Ei,"."),Ei.forEach(t),ms=f(e),Oe=a(e,"P",{});var gp=s(Oe);cn=i(gp,"And you\u2019re going to participate in AI vs. AI challenges where your trained agent will compete against other classmates\u2019 "),_l=a(gp,"STRONG",{});var cd=s(_l);pn=i(cd,"agents every day and be ranked on a new leaderboard."),cd.forEach(t),gp.forEach(t),vs=f(e),He=a(e,"P",{});var yp=s(He);dn=i(yp,"To validate this hands-on for the certification process, you just need to push a trained model. There "),Tl=a(yp,"STRONG",{});var pd=s(Tl);mn=i(pd,"are no minimal results to attain to validate it."),pd.forEach(t),yp.forEach(t),ws=f(e),Re=a(e,"P",{});var bp=s(Re);vn=i(bp,"For more information about the certification process, check this section \u{1F449} "),Ze=a(bp,"A",{href:!0,rel:!0});var dd=s(Ze);wn=i(dd,"https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),dd.forEach(t),bp.forEach(t),gs=f(e),le=a(e,"P",{});var _i=s(le);gn=i(_i,"This hands-on will be different since to get correct results "),Ml=a(_i,"STRONG",{});var md=s(Ml);yn=i(md,"you need to train your agents from 4 hours to 8 hours"),md.forEach(t),bn=i(_i,". And given the risk of timeout in Colab, we advise you to train on your computer. You don\u2019t need a supercomputer: a simple laptop is good enough for this exercise."),_i.forEach(t),ys=f(e),co=a(e,"P",{});var vd=s(co);En=i(vd,"Let\u2019s get started! \u{1F525}"),vd.forEach(t),bs=f(e),C=a(e,"H2",{class:!0});var Ti=s(C);ae=a(Ti,"A",{id:!0,class:!0,href:!0});var wd=s(ae);Al=a(wd,"SPAN",{});var gd=s(Al);d(ze.$$.fragment,gd),gd.forEach(t),wd.forEach(t),_n=f(Ti),Il=a(Ti,"SPAN",{});var yd=s(Il);Tn=i(yd,"What is AI vs. AI?"),yd.forEach(t),Ti.forEach(t),Es=f(e),po=a(e,"P",{});var bd=s(po);Mn=i(bd,"AI vs. AI is an open-source tool we developed at Hugging Face to compete agents on the Hub against one another in a multi-agent setting. These models are then ranked in a leaderboard."),bd.forEach(t),_s=f(e),Ye=a(e,"P",{});var Ep=s(Ye);An=i(Ep,"The idea of this tool is to have a robust evaluation tool: "),Sl=a(Ep,"STRONG",{});var Ed=s(Sl);In=i(Ed,"by evaluating your agent with a lot of others, you\u2019ll get a good idea of the quality of your policy."),Ed.forEach(t),Ep.forEach(t),Ts=f(e),mo=a(e,"P",{});var _d=s(mo);Sn=i(_d,"More precisely, AI vs. AI is three tools:"),_d.forEach(t),Ms=f(e),I=a(e,"UL",{});var cl=s(I);De=a(cl,"LI",{});var Mi=s(De);$n=i(Mi,"A "),$l=a(Mi,"EM",{});var Td=s($l);kn=i(Td,"matchmaking process"),Td.forEach(t),jn=i(Mi," defining the matches (which model against which) and running the model fights using a background task in the Space."),Mi.forEach(t),Jn=f(cl),se=a(cl,"LI",{});var rs=s(se);Un=i(rs,"A "),kl=a(rs,"EM",{});var Md=s(kl);Pn=i(Md,"leaderboard"),Md.forEach(t),Nn=i(rs," getting the match history results and displaying the models\u2019 ELO ratings: "),Fe=a(rs,"A",{href:!0,rel:!0});var Ad=s(Fe);Cn=i(Ad,"https://huggingface.co/spaces/huggingface-projects/AIvsAI-SoccerTwos"),Ad.forEach(t),rs.forEach(t),xn=f(cl),re=a(cl,"LI",{});var is=s(re);Ln=i(is,"A "),jl=a(is,"EM",{});var Id=s(jl);Wn=i(Id,"Space demo"),Id.forEach(t),Gn=i(is," to visualize your agents playing against others: "),Ve=a(is,"A",{href:!0,rel:!0});var Sd=s(Ve);Bn=i(Sd,"https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos"),Sd.forEach(t),is.forEach(t),cl.forEach(t),As=f(e),Xe=a(e,"P",{});var _p=s(Xe);On=i(_p,"In addition to these three tools, your classmate cyllum created a \u{1F917} SoccerTwos Challenge Analytics where you can check the detailed match results of a model: "),qe=a(_p,"A",{href:!0,rel:!0});var $d=s(qe);Hn=i($d,"https://huggingface.co/spaces/cyllum/soccertwos-analytics"),$d.forEach(t),_p.forEach(t),Is=f(e),ie=a(e,"P",{});var Ai=s(ie);Rn=i(Ai,"We\u2019re "),Qe=a(Ai,"A",{href:!0,rel:!0});var kd=s(Qe);Zn=i(kd,"wrote a blog post to explain this AI vs. AI tool in detail"),kd.forEach(t),zn=i(Ai,", but to give you the big picture it works this way:"),Ai.forEach(t),Ss=f(e),E=a(e,"UL",{});var Ce=s(E);vo=a(Ce,"LI",{});var Tp=s(vo);Yn=i(Tp,"Every four hours, our algorithm "),Jl=a(Tp,"STRONG",{});var jd=s(Jl);Dn=i(jd,"fetches all the available models for a given environment (in our case ML-Agents-SoccerTwos)."),jd.forEach(t),Tp.forEach(t),Fn=f(Ce),wo=a(Ce,"LI",{});var Mp=s(wo);Vn=i(Mp,"It creates a "),Ul=a(Mp,"STRONG",{});var Jd=s(Ul);Xn=i(Jd,"queue of matches with the matchmaking algorithm."),Jd.forEach(t),Mp.forEach(t),qn=f(Ce),Ke=a(Ce,"LI",{});var Ii=s(Ke);Qn=i(Ii,"We simulate the match in a Unity headless process and "),Pl=a(Ii,"STRONG",{});var Ud=s(Pl);Kn=i(Ud,"gather the match result"),Ud.forEach(t),eh=i(Ii," (1 if the first model won, 0.5 if it\u2019s a draw, 0 if the second model won) in a Dataset."),Ii.forEach(t),th=f(Ce),go=a(Ce,"LI",{});var Ap=s(go);oh=i(Ap,"Then, when all matches from the matches queue are done, "),Nl=a(Ap,"STRONG",{});var Pd=s(Nl);lh=i(Pd,"we update the ELO score for each model and update the leaderboard."),Pd.forEach(t),Ap.forEach(t),Ce.forEach(t),$s=f(e),x=a(e,"H3",{class:!0});var Si=s(x);ne=a(Si,"A",{id:!0,class:!0,href:!0});var Nd=s(ne);Cl=a(Nd,"SPAN",{});var Cd=s(Cl);d(et.$$.fragment,Cd),Cd.forEach(t),Nd.forEach(t),ah=f(Si),xl=a(Si,"SPAN",{});var xd=s(xl);sh=i(xd,"Competition Rules"),xd.forEach(t),Si.forEach(t),ks=f(e),_=a(e,"P",{});var xe=s(_);rh=i(xe,"This first AI vs. AI competition "),Ll=a(xe,"STRONG",{});var Ld=s(Ll);ih=i(Ld,"is an experiment"),Ld.forEach(t),nh=i(xe,": the goal is to improve the tool in the future with your feedback. So some "),Wl=a(xe,"STRONG",{});var Wd=s(Wl);hh=i(Wd,"breakups can happen during the challenge"),Wd.forEach(t),uh=i(xe,`. But don\u2019t worry
`),Gl=a(xe,"STRONG",{});var Gd=s(Gl);fh=i(Gd,"all the results are saved in a dataset so we can always restart the calculation correctly without losing information"),Gd.forEach(t),ch=i(xe,"."),xe.forEach(t),js=f(e),yo=a(e,"P",{});var Bd=s(yo);ph=i(Bd,"In order for your model to get correctly evaluated against others you need to follow these rules:"),Bd.forEach(t),Js=f(e),S=a(e,"OL",{});var pl=s(S);bo=a(pl,"LI",{});var Ip=s(bo);Bl=a(Ip,"STRONG",{});var Od=s(Bl);dh=i(Od,"You can\u2019t change the observation space or action space of the agent."),Od.forEach(t),mh=i(Ip," By doing that your model will not work during evaluation."),Ip.forEach(t),vh=f(pl),tt=a(pl,"LI",{});var $i=s(tt);wh=i($i,"You "),Ol=a($i,"STRONG",{});var Hd=s(Ol);gh=i(Hd,"can\u2019t use a custom trainer for now,"),Hd.forEach(t),yh=i($i," you need to use the Unity MLAgents ones."),$i.forEach(t),bh=f(pl),ot=a(pl,"LI",{});var ki=s(ot);Eh=i(ki,"We provide executables to train your agents. You can also use the Unity Editor if you prefer "),Hl=a(ki,"STRONG",{});var Rd=s(Hl);_h=i(Rd,", but to avoid bugs, we advise that you use our executables"),Rd.forEach(t),Th=i(ki,"."),ki.forEach(t),pl.forEach(t),Us=f(e),he=a(e,"P",{});var ji=s(he);Mh=i(ji,"What will make the difference during this challenge are "),Rl=a(ji,"STRONG",{});var Zd=s(Rl);Ah=i(Zd,"the hyperparameters you choose"),Zd.forEach(t),Ih=i(ji,"."),ji.forEach(t),Ps=f(e),$=a(e,"P",{});var dl=s($);Sh=i(dl,"We\u2019re constantly trying to improve our tutorials, so\xA0"),Zl=a(dl,"STRONG",{});var zd=s(Zl);$h=i(zd,"if you find some issues in this notebook"),zd.forEach(t),kh=i(dl,", please\xA0"),lt=a(dl,"A",{href:!0,rel:!0});var Yd=s(lt);jh=i(Yd,"open an issue on the GitHub Repo"),Yd.forEach(t),Jh=i(dl,"."),dl.forEach(t),Ns=f(e),L=a(e,"H3",{class:!0});var Ji=s(L);ue=a(Ji,"A",{id:!0,class:!0,href:!0});var Dd=s(ue);zl=a(Dd,"SPAN",{});var Fd=s(zl);d(at.$$.fragment,Fd),Fd.forEach(t),Dd.forEach(t),Uh=f(Ji),Yl=a(Ji,"SPAN",{});var Vd=s(Yl);Ph=i(Vd,"Chat with your classmates, share advice and ask questions on Discord"),Vd.forEach(t),Ji.forEach(t),Cs=f(e),fe=a(e,"UL",{});var Ui=s(fe);st=a(Ui,"LI",{});var Pi=s(st);Nh=i(Pi,"We created a new channel called "),Dl=a(Pi,"CODE",{});var Xd=s(Dl);Ch=i(Xd,"ai-vs-ai-challenge"),Xd.forEach(t),xh=i(Pi," to exchange advice and ask questions."),Pi.forEach(t),Lh=f(Ui),Eo=a(Ui,"LI",{});var Sp=s(Eo);Wh=i(Sp,"If you didn\u2019t join the discord server yet, you can "),rt=a(Sp,"A",{href:!0,rel:!0});var qd=s(rt);Gh=i(qd,"join here"),qd.forEach(t),Sp.forEach(t),Ui.forEach(t),xs=f(e),W=a(e,"H2",{class:!0});var Ni=s(W);ce=a(Ni,"A",{id:!0,class:!0,href:!0});var Qd=s(ce);Fl=a(Qd,"SPAN",{});var Kd=s(Fl);d(it.$$.fragment,Kd),Kd.forEach(t),Qd.forEach(t),Bh=f(Ni),Vl=a(Ni,"SPAN",{});var em=s(Vl);Oh=i(em,"Step 0: Install MLAgents and download the correct executable"),em.forEach(t),Ni.forEach(t),Ls=f(e),pe=a(e,"P",{});var Ci=s(pe);Hh=i(Ci,"We advise you to use "),nt=a(Ci,"A",{href:!0,rel:!0});var tm=s(nt);Rh=i(tm,"conda"),tm.forEach(t),Zh=i(Ci," as a package manager and create a new environment."),Ci.forEach(t),Ws=f(e),de=a(e,"P",{});var xi=s(de);zh=i(xi,"With conda, we create a new environment called rl with "),Xl=a(xi,"STRONG",{});var om=s(Xl);Yh=i(om,"Python 3.9"),om.forEach(t),Dh=i(xi,":"),xi.forEach(t),Gs=f(e),d(ht.$$.fragment,e),Bs=f(e),_o=a(e,"P",{});var lm=s(_o);Fh=i(lm,"To be able to train our agents correctly and push to the Hub, we need to install ML-Agents"),lm.forEach(t),Os=f(e),d(ut.$$.fragment,e),Hs=f(e),To=a(e,"P",{});var am=s(To);Vh=i(am,"When the cloning is done (it takes 2.63 GB), we go inside the repository and install the package"),am.forEach(t),Rs=f(e),d(ft.$$.fragment,e),Zs=f(e),Mo=a(e,"P",{});var sm=s(Mo);Xh=i(sm,"We also need to install pytorch with:"),sm.forEach(t),zs=f(e),d(ct.$$.fragment,e),Ys=f(e),pt=a(e,"P",{});var $p=s(pt);qh=i($p,"Finally, you need to install git-lfs: "),dt=a($p,"A",{href:!0,rel:!0});var rm=s(dt);Qh=i(rm,"https://git-lfs.com/"),rm.forEach(t),$p.forEach(t),Ds=f(e),G=a(e,"P",{});var ns=s(G);Kh=i(ns,"Now that it\u2019s installed, we need to add the environment training executable. Based on your operating system you need to download one of them, unzip it and place it in a new folder inside "),ql=a(ns,"CODE",{});var im=s(ql);eu=i(im,"ml-agents"),im.forEach(t),tu=i(ns," that you call "),Ql=a(ns,"CODE",{});var nm=s(Ql);ou=i(nm,"training-envs-executables"),nm.forEach(t),ns.forEach(t),Fs=f(e),mt=a(e,"P",{});var kp=s(mt);lu=i(kp,"At the end your executable should be in "),Kl=a(kp,"CODE",{});var hm=s(Kl);au=i(hm,"ml-agents/training-envs-executables/SoccerTwos"),hm.forEach(t),kp.forEach(t),Vs=f(e),vt=a(e,"P",{});var jp=s(vt);su=i(jp,"Windows: Download "),wt=a(jp,"A",{href:!0,rel:!0});var um=s(wt);ru=i(um,"this executable"),um.forEach(t),jp.forEach(t),Xs=f(e),gt=a(e,"P",{});var Jp=s(gt);iu=i(Jp,"Linux (Ubuntu): Download "),yt=a(Jp,"A",{href:!0,rel:!0});var fm=s(yt);nu=i(fm,"this executable"),fm.forEach(t),Jp.forEach(t),qs=f(e),k=a(e,"P",{});var ml=s(k);hu=i(ml,"Mac: Download "),bt=a(ml,"A",{href:!0,rel:!0});var cm=s(bt);uu=i(cm,"this executable"),cm.forEach(t),fu=i(ml,`
\u26A0 For Mac you need also to call this `),ea=a(ml,"CODE",{});var pm=s(ea);cu=i(pm,"xattr -cr training-envs-executables/SoccerTwos/SoccerTwos.app"),pm.forEach(t),pu=i(ml," to be able to run SoccerTwos"),ml.forEach(t),Qs=f(e),B=a(e,"H2",{class:!0});var Li=s(B);me=a(Li,"A",{id:!0,class:!0,href:!0});var dm=s(me);ta=a(dm,"SPAN",{});var mm=s(ta);d(Et.$$.fragment,mm),mm.forEach(t),dm.forEach(t),du=f(Li),oa=a(Li,"SPAN",{});var vm=s(oa);mu=i(vm,"Step 1: Understand the environment"),vm.forEach(t),Li.forEach(t),Ks=f(e),O=a(e,"P",{});var hs=s(O);vu=i(hs,"The environment is called "),la=a(hs,"CODE",{});var wm=s(la);wu=i(wm,"SoccerTwos"),wm.forEach(t),gu=i(hs,". The Unity MLAgents Team made it. You can find its documentation "),_t=a(hs,"A",{href:!0,rel:!0});var gm=s(_t);yu=i(gm,"here"),gm.forEach(t),hs.forEach(t),er=f(e),Tt=a(e,"P",{});var Up=s(Tt);bu=i(Up,"The goal in this environment "),aa=a(Up,"STRONG",{});var ym=s(aa);Eu=i(ym,"is to get the ball into the opponent\u2019s goal while preventing the ball from entering your own goal."),ym.forEach(t),Up.forEach(t),tr=f(e),ve=a(e,"FIGURE",{});var Wi=s(ve);Ao=a(Wi,"IMG",{src:!0,alt:!0}),_u=f(Wi),Io=a(Wi,"FIGCAPTION",{});var Pp=s(Io);Tu=i(Pp,"This environment was made by the "),So=a(Pp,"A",{href:!0});var bm=s(So);Mu=i(bm,"Unity MLAgents Team"),bm.forEach(t),Pp.forEach(t),Wi.forEach(t),or=f(e),H=a(e,"H3",{class:!0});var Gi=s(H);we=a(Gi,"A",{id:!0,class:!0,href:!0});var Em=s(we);sa=a(Em,"SPAN",{});var _m=s(sa);d(Mt.$$.fragment,_m),_m.forEach(t),Em.forEach(t),Au=f(Gi),ra=a(Gi,"SPAN",{});var Tm=s(ra);Iu=i(Tm,"The reward function"),Tm.forEach(t),Gi.forEach(t),lr=f(e),$o=a(e,"P",{});var Mm=s($o);Su=i(Mm,"The reward function is:"),Mm.forEach(t),ar=f(e),At=a(e,"IMG",{src:!0,alt:!0}),sr=f(e),R=a(e,"H3",{class:!0});var Bi=s(R);ge=a(Bi,"A",{id:!0,class:!0,href:!0});var Am=s(ge);ia=a(Am,"SPAN",{});var Im=s(ia);d(It.$$.fragment,Im),Im.forEach(t),Am.forEach(t),$u=f(Bi),na=a(Bi,"SPAN",{});var Sm=s(na);ku=i(Sm,"The observation space"),Sm.forEach(t),Bi.forEach(t),rr=f(e),ko=a(e,"P",{});var $m=s(ko);ju=i($m,"The observation space is composed of vectors of size 336:"),$m.forEach(t),ir=f(e),j=a(e,"UL",{});var vl=s(j);ha=a(vl,"LI",{});var km=s(ha);Ju=i(km,"11 ray-casts forward distributed over 120 degrees (264 state dimensions)"),km.forEach(t),Uu=f(vl),ua=a(vl,"LI",{});var jm=s(ua);Pu=i(jm,"3 ray-casts backward distributed over 90 degrees (72 state dimensions)"),jm.forEach(t),Nu=f(vl),jo=a(vl,"LI",{});var Np=s(jo);Cu=i(Np,"Both of these ray-casts can detect 6 objects:"),y=a(Np,"UL",{});var A=s(y);fa=a(A,"LI",{});var Jm=s(fa);xu=i(Jm,"Ball"),Jm.forEach(t),Lu=f(A),ca=a(A,"LI",{});var Um=s(ca);Wu=i(Um,"Blue Goal"),Um.forEach(t),Gu=f(A),pa=a(A,"LI",{});var Pm=s(pa);Bu=i(Pm,"Purple Goal"),Pm.forEach(t),Ou=f(A),da=a(A,"LI",{});var Nm=s(da);Hu=i(Nm,"Wall"),Nm.forEach(t),Ru=f(A),ma=a(A,"LI",{});var Cm=s(ma);Zu=i(Cm,"Blue Agent"),Cm.forEach(t),zu=f(A),va=a(A,"LI",{});var xm=s(va);Yu=i(xm,"Purple Agent"),xm.forEach(t),A.forEach(t),Np.forEach(t),vl.forEach(t),nr=f(e),Z=a(e,"H3",{class:!0});var Oi=s(Z);ye=a(Oi,"A",{id:!0,class:!0,href:!0});var Lm=s(ye);wa=a(Lm,"SPAN",{});var Wm=s(wa);d(St.$$.fragment,Wm),Wm.forEach(t),Lm.forEach(t),Du=f(Oi),ga=a(Oi,"SPAN",{});var Gm=s(ga);Fu=i(Gm,"The action space"),Gm.forEach(t),Oi.forEach(t),hr=f(e),Jo=a(e,"P",{});var Bm=s(Jo);Vu=i(Bm,"The action space is three discrete branches:"),Bm.forEach(t),ur=f(e),$t=a(e,"IMG",{src:!0,alt:!0}),fr=f(e),z=a(e,"H2",{class:!0});var Hi=s(z);be=a(Hi,"A",{id:!0,class:!0,href:!0});var Om=s(be);ya=a(Om,"SPAN",{});var Hm=s(ya);d(kt.$$.fragment,Hm),Hm.forEach(t),Om.forEach(t),Xu=f(Hi),ba=a(Hi,"SPAN",{});var Rm=s(ba);qu=i(Rm,"Step 2: Understand MA-POCA"),Rm.forEach(t),Hi.forEach(t),cr=f(e),Ee=a(e,"P",{});var Ri=s(Ee);Qu=i(Ri,"We know how to train agents to play against others: "),Ea=a(Ri,"STRONG",{});var Zm=s(Ea);Ku=i(Zm,"we can use self-play."),Zm.forEach(t),ef=i(Ri," This is a perfect technique for a 1vs1."),Ri.forEach(t),pr=f(e),jt=a(e,"P",{});var Cp=s(jt);tf=i(Cp,"But in our case we\u2019re 2vs2, and each team has 2 agents. How then can we "),_a=a(Cp,"STRONG",{});var zm=s(_a);of=i(zm,"train cooperative behavior for groups of agents?"),zm.forEach(t),Cp.forEach(t),dr=f(e),J=a(e,"P",{});var wl=s(J);lf=i(wl,"As explained in the "),Jt=a(wl,"A",{href:!0,rel:!0});var Ym=s(Jt);af=i(Ym,"Unity Blog"),Ym.forEach(t),sf=i(wl,", agents typically receive a reward as a group (+1 - penalty) when the team scores a goal. This implies that "),Ta=a(wl,"STRONG",{});var Dm=s(Ta);rf=i(Dm,"every agent on the team is rewarded even if each agent didn\u2019t contribute the same to the win"),Dm.forEach(t),nf=i(wl,", which makes it difficult to learn what to do independently."),wl.forEach(t),mr=f(e),_e=a(e,"P",{});var Zi=s(_e);hf=i(Zi,"The Unity MLAgents team developed the solution in a new multi-agent trainer called "),Ma=a(Zi,"EM",{});var Fm=s(Ma);uf=i(Fm,"MA-POCA (Multi-Agent POsthumous Credit Assignment)"),Fm.forEach(t),ff=i(Zi,"."),Zi.forEach(t),vr=f(e),Te=a(e,"P",{});var zi=s(Te);cf=i(zi,"The idea is simple but powerful: a centralized critic "),Aa=a(zi,"STRONG",{});var Vm=s(Aa);pf=i(Vm,"processes the states of all agents in the team to estimate how well each agent is doing"),Vm.forEach(t),df=i(zi,". Think of this critic as a coach."),zi.forEach(t),wr=f(e),U=a(e,"P",{});var gl=s(U);mf=i(gl,"This allows each agent to "),Ia=a(gl,"STRONG",{});var Xm=s(Ia);vf=i(Xm,"make decisions based only on what it perceives locally"),Xm.forEach(t),wf=i(gl,", and "),Sa=a(gl,"STRONG",{});var qm=s(Sa);gf=i(qm,"simultaneously evaluate how good its behavior is in the context of the whole group"),qm.forEach(t),yf=i(gl,"."),gl.forEach(t),gr=f(e),Me=a(e,"FIGURE",{});var Yi=s(Me);Uo=a(Yi,"IMG",{src:!0,alt:!0}),bf=f(Yi),Po=a(Yi,"FIGCAPTION",{});var xp=s(Po);Ef=i(xp,"This illustrates MA-POCA\u2019s centralized learning and decentralized execution. Source: "),No=a(xp,"A",{href:!0});var Qm=s(No);_f=i(Qm,"MLAgents Plays Dodgeball"),Qm.forEach(t),xp.forEach(t),Yi.forEach(t),yr=f(e),Co=a(e,"P",{});var Km=s(Co);Tf=i(Km,"The solution then is to use Self-Play with an MA-POCA trainer (called poca). The poca trainer will help us to train cooperative behavior and self-play to win against an opponent team."),Km.forEach(t),br=f(e),Ae=a(e,"P",{});var Di=s(Ae);Mf=i(Di,"If you want to dive deeper into this MA-POCA algorithm, you need to read the paper they published "),Ut=a(Di,"A",{href:!0,rel:!0});var ev=s(Ut);Af=i(ev,"here"),ev.forEach(t),If=i(Di," and the sources we put on the additional readings section."),Di.forEach(t),Er=f(e),Y=a(e,"H2",{class:!0});var Fi=s(Y);Ie=a(Fi,"A",{id:!0,class:!0,href:!0});var tv=s(Ie);$a=a(tv,"SPAN",{});var ov=s($a);d(Pt.$$.fragment,ov),ov.forEach(t),tv.forEach(t),Sf=f(Fi),ka=a(Fi,"SPAN",{});var lv=s(ka);$f=i(lv,"Step 3: Define the config file"),lv.forEach(t),Fi.forEach(t),_r=f(e),D=a(e,"P",{});var us=s(D);kf=i(us,"We already learned in "),Nt=a(us,"A",{href:!0,rel:!0});var av=s(Nt);jf=i(av,"Unit 5"),av.forEach(t),Jf=i(us," that in ML-Agents, you define "),Ct=a(us,"STRONG",{});var Vi=s(Ct);Uf=i(Vi,"the training hyperparameters in "),ja=a(Vi,"CODE",{});var sv=s(ja);Pf=i(sv,"config.yaml"),sv.forEach(t),Nf=i(Vi," files."),Vi.forEach(t),us.forEach(t),Tr=f(e),xt=a(e,"P",{});var Lp=s(xt);Cf=i(Lp,"There are multiple hyperparameters. To understand them better, you should read the explanations for each of them in\xA0"),Ja=a(Lp,"STRONG",{});var rv=s(Ja);Lt=a(rv,"A",{href:!0,rel:!0});var iv=s(Lt);xf=i(iv,"the documentation"),iv.forEach(t),rv.forEach(t),Lp.forEach(t),Mr=f(e),Se=a(e,"P",{});var Xi=s(Se);Lf=i(Xi,"The config file we\u2019re going to use here is in  "),Ua=a(Xi,"CODE",{});var nv=s(Ua);Wf=i(nv,"./config/poca/SoccerTwos.yaml"),nv.forEach(t),Gf=i(Xi,". It looks like this:"),Xi.forEach(t),Ar=f(e),d(Wt.$$.fragment,e),Ir=f(e),xo=a(e,"P",{});var hv=s(xo);Bf=i(hv,"Compared to Pyramids or SnowballTarget, we have new hyperparameters with a self-play part. How you modify them can be critical in getting good results."),hv.forEach(t),Sr=f(e),Gt=a(e,"P",{});var Wp=s(Gt);Of=i(Wp,"The advice I can give you here is to check the explanation and recommended value for each parameters (especially self-play ones) against\xA0"),Lo=a(Wp,"STRONG",{});var Gp=s(Lo);Bt=a(Gp,"A",{href:!0,rel:!0});var uv=s(Bt);Hf=i(uv,"the documentation"),uv.forEach(t),Rf=i(Gp,"."),Gp.forEach(t),Wp.forEach(t),$r=f(e),Wo=a(e,"P",{});var fv=s(Wo);Zf=i(fv,"Now that you\u2019ve modified our config file, you\u2019re ready to train your agents."),fv.forEach(t),kr=f(e),F=a(e,"H2",{class:!0});var qi=s(F);$e=a(qi,"A",{id:!0,class:!0,href:!0});var cv=s($e);Pa=a(cv,"SPAN",{});var pv=s(Pa);d(Ot.$$.fragment,pv),pv.forEach(t),cv.forEach(t),zf=f(qi),Na=a(qi,"SPAN",{});var dv=s(Na);Yf=i(dv,"Step 4: Start the training"),dv.forEach(t),qi.forEach(t),jr=f(e),Ht=a(e,"P",{});var Bp=s(Ht);Df=i(Bp,"To train the agents, we need to\xA0"),Ca=a(Bp,"STRONG",{});var mv=s(Ca);Ff=i(mv,"launch mlagents-learn and select the executable containing the environment."),mv.forEach(t),Bp.forEach(t),Jr=f(e),Go=a(e,"P",{});var vv=s(Go);Vf=i(vv,"We define four parameters:"),vv.forEach(t),Ur=f(e),T=a(e,"OL",{});var Le=s(T);Bo=a(Le,"LI",{});var Op=s(Bo);xa=a(Op,"CODE",{});var wv=s(xa);Xf=i(wv,"mlagents-learn <config>"),wv.forEach(t),qf=i(Op,": the path where the hyperparameter config file is."),Op.forEach(t),Qf=f(Le),Oo=a(Le,"LI",{});var Hp=s(Oo);La=a(Hp,"CODE",{});var gv=s(La);Kf=i(gv,"-env"),gv.forEach(t),ec=i(Hp,": where the environment executable is."),Hp.forEach(t),tc=f(Le),Ho=a(Le,"LI",{});var Rp=s(Ho);Wa=a(Rp,"CODE",{});var yv=s(Wa);oc=i(yv,"-run_id"),yv.forEach(t),lc=i(Rp,": the name you want to give to your training run id."),Rp.forEach(t),ac=f(Le),Ro=a(Le,"LI",{});var Zp=s(Ro);Ga=a(Zp,"CODE",{});var bv=s(Ga);sc=i(bv,"-no-graphics"),bv.forEach(t),rc=i(Zp,": to not launch the visualization during the training."),Zp.forEach(t),Le.forEach(t),Pr=f(e),Zo=a(e,"P",{});var Ev=s(Zo);ic=i(Ev,"Depending on your hardware, 5M timesteps (the recommended value, but you can also try 10M) will take 5 to 8 hours of training. You can continue using your computer in the meantime, but I advise deactivating the computer standby mode to prevent the training from being stopped."),Ev.forEach(t),Nr=f(e),zo=a(e,"P",{});var _v=s(zo);nc=i(_v,"Depending on the executable you use (windows, ubuntu, mac) the training command will look like this (your executable path can be different so don\u2019t hesitate to check before running)."),_v.forEach(t),Cr=f(e),d(Rt.$$.fragment,e),xr=f(e),Yo=a(e,"P",{});var Tv=s(Yo);hc=i(Tv,"The executable contains 8 copies of SoccerTwos."),Tv.forEach(t),Lr=f(e),Do=a(e,"P",{});var Mv=s(Do);uc=i(Mv,"\u26A0\uFE0F It\u2019s normal if you don\u2019t see a big increase of ELO score (and even a decrease below 1200) before 2M timesteps, since your agents will spend most of their time moving randomly on the field before being able to goal."),Mv.forEach(t),Wr=f(e),Fo=a(e,"P",{});var Av=s(Fo);fc=i(Av,"\u26A0\uFE0F You can stop the training with Ctrl + C but beware of typing this command only once to stop the training since MLAgents needs to generate a final .onnx file before closing the run."),Av.forEach(t),Gr=f(e),V=a(e,"H2",{class:!0});var Qi=s(V);ke=a(Qi,"A",{id:!0,class:!0,href:!0});var Iv=s(ke);Ba=a(Iv,"SPAN",{});var Sv=s(Ba);d(Zt.$$.fragment,Sv),Sv.forEach(t),Iv.forEach(t),cc=f(Qi),Oa=a(Qi,"SPAN",{});var $v=s(Oa);pc=i($v,"Step 5: **Push the agent to the Hugging Face Hub**"),$v.forEach(t),Qi.forEach(t),Br=f(e),zt=a(e,"P",{});var zp=s(zt);dc=i(zp,"Now that we trained our agents, we\u2019re\xA0"),Ha=a(zp,"STRONG",{});var kv=s(Ha);mc=i(kv,"ready to push them to the Hub to be able to participate in the AI vs. AI challenge and visualize them playing on your browser\u{1F525}."),kv.forEach(t),zp.forEach(t),Or=f(e),Vo=a(e,"P",{});var jv=s(Vo);vc=i(jv,"To be able to share your model with the community, there are three more steps to follow:"),jv.forEach(t),Hr=f(e),Yt=a(e,"P",{});var Yp=s(Yt);wc=i(Yp,"1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1\xA0"),Dt=a(Yp,"A",{href:!0,rel:!0});var Jv=s(Dt);gc=i(Jv,"https://huggingface.co/join"),Jv.forEach(t),Yp.forEach(t),Rr=f(e),Xo=a(e,"P",{});var Uv=s(Xo);yc=i(Uv,"2\uFE0F\u20E3 Sign in and store your authentication token from the Hugging Face website."),Uv.forEach(t),Zr=f(e),X=a(e,"P",{});var fs=s(X);bc=i(fs,"Create a new token ("),Ft=a(fs,"A",{href:!0,rel:!0});var Pv=s(Ft);Ec=i(Pv,"https://huggingface.co/settings/tokens"),Pv.forEach(t),_c=i(fs,")\xA0"),Ra=a(fs,"STRONG",{});var Nv=s(Ra);Tc=i(Nv,"with write role"),Nv.forEach(t),fs.forEach(t),zr=f(e),Vt=a(e,"IMG",{src:!0,alt:!0}),Yr=f(e),qo=a(e,"P",{});var Cv=s(qo);Mc=i(Cv,"Copy the token, run this, and paste the token"),Cv.forEach(t),Dr=f(e),d(Xt.$$.fragment,e),Fr=f(e),je=a(e,"P",{});var Ki=s(je);Ac=i(Ki,"Then, we need to run "),Za=a(Ki,"CODE",{});var xv=s(Za);Ic=i(xv,"mlagents-push-to-hf"),xv.forEach(t),Sc=i(Ki,"."),Ki.forEach(t),Vr=f(e),Qo=a(e,"P",{});var Lv=s(Qo);$c=i(Lv,"And we define four parameters:"),Lv.forEach(t),Xr=f(e),M=a(e,"OL",{});var We=s(M);Ko=a(We,"LI",{});var Dp=s(Ko);za=a(Dp,"CODE",{});var Wv=s(za);kc=i(Wv,"-run-id"),Wv.forEach(t),jc=i(Dp,": the name of the training run id."),Dp.forEach(t),Jc=f(We),el=a(We,"LI",{});var Fp=s(el);Ya=a(Fp,"CODE",{});var Gv=s(Ya);Uc=i(Gv,"-local-dir"),Gv.forEach(t),Pc=i(Fp,": where the agent was saved, it\u2019s results/<run_id name>, so in my case results/First Training."),Fp.forEach(t),Nc=f(We),qt=a(We,"LI",{});var en=s(qt);Da=a(en,"CODE",{});var Bv=s(Da);Cc=i(Bv,"-repo-id"),Bv.forEach(t),xc=i(en,`: the name of the Hugging Face repo you want to create or update. It\u2019s always <your huggingface username>/<the repo name>
If the repo does not exist `),Fa=a(en,"STRONG",{});var Ov=s(Fa);Lc=i(Ov,"it will be created automatically"),Ov.forEach(t),en.forEach(t),Wc=f(We),tl=a(We,"LI",{});var Vp=s(tl);Va=a(Vp,"CODE",{});var Hv=s(Va);Gc=i(Hv,"--commit-message"),Hv.forEach(t),Bc=i(Vp,": since HF repos are git repositories you need to give a commit message."),Vp.forEach(t),We.forEach(t),qr=f(e),ol=a(e,"P",{});var Rv=s(ol);Oc=i(Rv,"In my case"),Rv.forEach(t),Qr=f(e),d(Qt.$$.fragment,e),Kr=f(e),d(Kt.$$.fragment,e),ei=f(e),ll=a(e,"P",{});var Zv=s(ll);Hc=i(Zv,"If everything worked you should see this at the end of the process (but with a different url \u{1F606}) :"),Zv.forEach(t),ti=f(e),eo=a(e,"P",{});var Xp=s(eo);Rc=i(Xp,"Your model is pushed to the Hub. You can view your model here: "),to=a(Xp,"A",{href:!0,rel:!0});var zv=s(to);Zc=i(zv,"https://huggingface.co/ThomasSimonini/poca-SoccerTwos"),zv.forEach(t),Xp.forEach(t),oi=f(e),oo=a(e,"P",{});var qp=s(oo);zc=i(qp,"It\u2019s the link to your model. It contains a model card that explains how to use it, your Tensorboard, and your config file. "),Xa=a(qp,"STRONG",{});var Yv=s(Xa);Yc=i(Yv,"What\u2019s awesome is that it\u2019s a git repository, which means you can have different commits, update your repository with a new push, etc."),Yv.forEach(t),qp.forEach(t),li=f(e),q=a(e,"H2",{class:!0});var tn=s(q);Je=a(tn,"A",{id:!0,class:!0,href:!0});var Dv=s(Je);qa=a(Dv,"SPAN",{});var Fv=s(qa);d(lo.$$.fragment,Fv),Fv.forEach(t),Dv.forEach(t),Dc=f(tn),Qa=a(tn,"SPAN",{});var Vv=s(Qa);Fc=i(Vv,"Step 6: Verify that your model is ready for AI vs AI Challenge"),Vv.forEach(t),tn.forEach(t),ai=f(e),Ue=a(e,"P",{});var on=s(Ue);Vc=i(on,"Now that your model is pushed to the Hub, "),Ka=a(on,"STRONG",{});var Xv=s(Ka);Xc=i(Xv,"it\u2019s going to be added automatically to the AI vs AI Challenge model pool."),Xv.forEach(t),qc=i(on," It can take a little bit of time before your model is added to the leaderboard given we do a run of matches every 4h."),on.forEach(t),si=f(e),al=a(e,"P",{});var qv=s(al);Qc=i(qv,"But to ensure that everything works perfectly you need to check:"),qv.forEach(t),ri=f(e),sl=a(e,"OL",{});var Qv=s(sl);es=a(Qv,"LI",{});var Kv=s(es);Kc=i(Kv,"That you have this tag in your model: ML-Agents-SoccerTwos. This is the tag we use to select models to be added to the challenge pool. To do that go to your model and check the tags"),Kv.forEach(t),Qv.forEach(t),ii=f(e),ao=a(e,"IMG",{src:!0,alt:!0}),ni=f(e),rl=a(e,"P",{});var ew=s(rl);ep=i(ew,"If it\u2019s not the case you just need to modify the readme and add it"),ew.forEach(t),hi=f(e),so=a(e,"IMG",{src:!0,alt:!0}),ui=f(e),ro=a(e,"OL",{start:!0});var tw=s(ro);io=a(tw,"LI",{});var ln=s(io);tp=i(ln,"That you have a "),ts=a(ln,"CODE",{});var ow=s(ts);op=i(ow,"SoccerTwos.onnx"),ow.forEach(t),lp=i(ln," file"),ln.forEach(t),tw.forEach(t),fi=f(e),no=a(e,"IMG",{src:!0,alt:!0}),ci=f(e),il=a(e,"P",{});var lw=s(il);ap=i(lw,"We strongly suggest that you create a new model when you push to the Hub if you want to train it again or train a new version."),lw.forEach(t),pi=f(e),Q=a(e,"H2",{class:!0});var an=s(Q);Pe=a(an,"A",{id:!0,class:!0,href:!0});var aw=s(Pe);os=a(aw,"SPAN",{});var sw=s(os);d(ho.$$.fragment,sw),sw.forEach(t),aw.forEach(t),sp=f(an),ls=a(an,"SPAN",{});var rw=s(ls);rp=i(rw,"Step 7: Visualize some match in our demo"),rw.forEach(t),an.forEach(t),di=f(e),K=a(e,"P",{});var cs=s(K);ip=i(cs,"Now that your model is part of AI vs AI Challenge, "),as=a(cs,"STRONG",{});var iw=s(as);np=i(iw,"you can visualize how good it is compared to others"),iw.forEach(t),hp=i(cs,": "),uo=a(cs,"A",{href:!0,rel:!0});var nw=s(uo);up=i(nw,"https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos"),nw.forEach(t),cs.forEach(t),mi=f(e),nl=a(e,"P",{});var hw=s(nl);fp=i(hw,"In order to do that, you just need to go to this demo:"),hw.forEach(t),vi=f(e),hl=a(e,"UL",{});var uw=s(hl);ul=a(uw,"LI",{});var Qp=s(ul);cp=i(Qp,"Select your model as team blue (or team purple if you prefer) and another model to compete against. The best opponents to compare your model to are either whoever is on top of the leaderboard or the "),fo=a(Qp,"A",{href:!0,rel:!0});var fw=s(fo);pp=i(fw,"baseline model"),fw.forEach(t),Qp.forEach(t),uw.forEach(t),wi=f(e),Ne=a(e,"P",{});var sn=s(Ne);dp=i(sn,"The matches you see live are not used in the calculation of your result "),ss=a(sn,"STRONG",{});var cw=s(ss);mp=i(cw,"but they are a good way to visualize how good your agent is"),cw.forEach(t),vp=i(sn,"."),sn.forEach(t),gi=f(e),fl=a(e,"P",{});var pw=s(fl);wp=i(pw,"And don\u2019t hesitate to share the best score your agent gets on discord in the #rl-i-made-this channel \u{1F525}"),pw.forEach(t),this.h()},h(){c(P,"name","hf:doc:metadata"),c(P,"content",JSON.stringify(Ew)),c(te,"id","handson"),c(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(te,"href","#handson"),c(N,"class","relative group"),c(Ze,"href","https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),c(Ze,"rel","nofollow"),c(ae,"id","what-is-ai-vs-ai"),c(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ae,"href","#what-is-ai-vs-ai"),c(C,"class","relative group"),c(Fe,"href","https://huggingface.co/spaces/huggingface-projects/AIvsAI-SoccerTwos"),c(Fe,"rel","nofollow"),c(Ve,"href","https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos"),c(Ve,"rel","nofollow"),c(qe,"href","https://huggingface.co/spaces/cyllum/soccertwos-analytics"),c(qe,"rel","nofollow"),c(Qe,"href","https://huggingface.co/blog/aivsai"),c(Qe,"rel","nofollow"),c(ne,"id","competition-rules"),c(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ne,"href","#competition-rules"),c(x,"class","relative group"),c(lt,"href","https://github.com/huggingface/deep-rl-class/issues"),c(lt,"rel","nofollow"),c(ue,"id","chat-with-your-classmates-share-advice-and-ask-questions-on-discord"),c(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ue,"href","#chat-with-your-classmates-share-advice-and-ask-questions-on-discord"),c(L,"class","relative group"),c(rt,"href","https://discord.gg/ydHrjt3WP5"),c(rt,"rel","nofollow"),c(ce,"id","step-0-install-mlagents-and-download-the-correct-executable"),c(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ce,"href","#step-0-install-mlagents-and-download-the-correct-executable"),c(W,"class","relative group"),c(nt,"href","https://docs.conda.io/en/latest/"),c(nt,"rel","nofollow"),c(dt,"href","https://git-lfs.com/"),c(dt,"rel","nofollow"),c(wt,"href","https://drive.google.com/file/d/1sqFxbEdTMubjVktnV4C6ICjp89wLhUcP/view?usp=sharing"),c(wt,"rel","nofollow"),c(yt,"href","https://drive.google.com/file/d/1KuqBKYiXiIcU4kNMqEzhgypuFP5_45CL/view?usp=sharing"),c(yt,"rel","nofollow"),c(bt,"href","https://drive.google.com/drive/folders/1h7YB0qwjoxxghApQdEUQmk95ZwIDxrPG?usp=share_link"),c(bt,"rel","nofollow"),c(me,"id","step-1-understand-the-environment"),c(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(me,"href","#step-1-understand-the-environment"),c(B,"class","relative group"),c(_t,"href","https://github.com/Unity-Technologies/ml-agents/blob/develop/docs/Learning-Environment-Examples.md#soccer-twos"),c(_t,"rel","nofollow"),Ge(Ao.src,ed="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/soccertwos.gif")||c(Ao,"src",ed),c(Ao,"alt","SoccerTwos"),c(So,"href","https://github.com/Unity-Technologies/ml-agents"),c(we,"id","the-reward-function"),c(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(we,"href","#the-reward-function"),c(H,"class","relative group"),Ge(At.src,td="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/soccerreward.png")||c(At,"src",td),c(At,"alt","SoccerTwos Reward"),c(ge,"id","the-observation-space"),c(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ge,"href","#the-observation-space"),c(R,"class","relative group"),c(ye,"id","the-action-space"),c(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ye,"href","#the-action-space"),c(Z,"class","relative group"),Ge($t.src,od="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/socceraction.png")||c($t,"src",od),c($t,"alt","SoccerTwos Action"),c(be,"id","step-2-understand-mapoca"),c(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(be,"href","#step-2-understand-mapoca"),c(z,"class","relative group"),c(Jt,"href","https://blog.unity.com/technology/ml-agents-v20-release-now-supports-training-complex-cooperative-behaviors"),c(Jt,"rel","nofollow"),Ge(Uo.src,ld="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/mapoca.png")||c(Uo,"src",ld),c(Uo,"alt","MA POCA"),c(No,"href","https://blog.unity.com/technology/ml-agents-plays-dodgeball"),c(Ut,"href","https://arxiv.org/pdf/2111.05992.pdf"),c(Ut,"rel","nofollow"),c(Ie,"id","step-3-define-the-config-file"),c(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ie,"href","#step-3-define-the-config-file"),c(Y,"class","relative group"),c(Nt,"href","https://huggingface.co/deep-rl-course/unit5/introduction"),c(Nt,"rel","nofollow"),c(Lt,"href","https://github.com/Unity-Technologies/ml-agents/blob/release_20_docs/docs/Training-Configuration-File.md"),c(Lt,"rel","nofollow"),c(Bt,"href","https://github.com/Unity-Technologies/ml-agents/blob/release_20_docs/docs/Training-Configuration-File.md"),c(Bt,"rel","nofollow"),c($e,"id","step-4-start-the-training"),c($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c($e,"href","#step-4-start-the-training"),c(F,"class","relative group"),c(ke,"id","step-5-push-the-agent-to-the-hugging-face-hub"),c(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ke,"href","#step-5-push-the-agent-to-the-hugging-face-hub"),c(V,"class","relative group"),c(Dt,"href","https://huggingface.co/join"),c(Dt,"rel","nofollow"),c(Ft,"href","https://huggingface.co/settings/tokens"),c(Ft,"rel","nofollow"),Ge(Vt.src,ad="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/create-token.jpg")||c(Vt,"src",ad),c(Vt,"alt","Create HF Token"),c(to,"href","https://huggingface.co/ThomasSimonini/poca-SoccerTwos"),c(to,"rel","nofollow"),c(Je,"id","step-6-verify-that-your-model-is-ready-for-ai-vs-ai-challenge"),c(Je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Je,"href","#step-6-verify-that-your-model-is-ready-for-ai-vs-ai-challenge"),c(q,"class","relative group"),Ge(ao.src,sd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/verify1.png")||c(ao,"src",sd),c(ao,"alt","Verify"),Ge(so.src,rd="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/verify2.png")||c(so,"src",rd),c(so,"alt","Verify"),c(ro,"start","2"),Ge(no.src,id="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit10/verify3.png")||c(no,"src",id),c(no,"alt","Verify"),c(Pe,"id","step-7-visualize-some-match-in-our-demo"),c(Pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Pe,"href","#step-7-visualize-some-match-in-our-demo"),c(Q,"class","relative group"),c(uo,"href","https://huggingface.co/spaces/unity/ML-Agents-SoccerTwos"),c(uo,"rel","nofollow"),c(fo,"href","https://huggingface.co/unity/MLAgents-SoccerTwos"),c(fo,"rel","nofollow")},m(e,n){o(document.head,P),h(e,ps,n),h(e,N,n),o(N,te),o(te,yl),m(Be,yl,null),o(N,rn),o(N,bl),o(bl,nn),h(e,ds,n),h(e,oe,n),o(oe,hn),o(oe,El),o(El,un),o(oe,fn),h(e,ms,n),h(e,Oe,n),o(Oe,cn),o(Oe,_l),o(_l,pn),h(e,vs,n),h(e,He,n),o(He,dn),o(He,Tl),o(Tl,mn),h(e,ws,n),h(e,Re,n),o(Re,vn),o(Re,Ze),o(Ze,wn),h(e,gs,n),h(e,le,n),o(le,gn),o(le,Ml),o(Ml,yn),o(le,bn),h(e,ys,n),h(e,co,n),o(co,En),h(e,bs,n),h(e,C,n),o(C,ae),o(ae,Al),m(ze,Al,null),o(C,_n),o(C,Il),o(Il,Tn),h(e,Es,n),h(e,po,n),o(po,Mn),h(e,_s,n),h(e,Ye,n),o(Ye,An),o(Ye,Sl),o(Sl,In),h(e,Ts,n),h(e,mo,n),o(mo,Sn),h(e,Ms,n),h(e,I,n),o(I,De),o(De,$n),o(De,$l),o($l,kn),o(De,jn),o(I,Jn),o(I,se),o(se,Un),o(se,kl),o(kl,Pn),o(se,Nn),o(se,Fe),o(Fe,Cn),o(I,xn),o(I,re),o(re,Ln),o(re,jl),o(jl,Wn),o(re,Gn),o(re,Ve),o(Ve,Bn),h(e,As,n),h(e,Xe,n),o(Xe,On),o(Xe,qe),o(qe,Hn),h(e,Is,n),h(e,ie,n),o(ie,Rn),o(ie,Qe),o(Qe,Zn),o(ie,zn),h(e,Ss,n),h(e,E,n),o(E,vo),o(vo,Yn),o(vo,Jl),o(Jl,Dn),o(E,Fn),o(E,wo),o(wo,Vn),o(wo,Ul),o(Ul,Xn),o(E,qn),o(E,Ke),o(Ke,Qn),o(Ke,Pl),o(Pl,Kn),o(Ke,eh),o(E,th),o(E,go),o(go,oh),o(go,Nl),o(Nl,lh),h(e,$s,n),h(e,x,n),o(x,ne),o(ne,Cl),m(et,Cl,null),o(x,ah),o(x,xl),o(xl,sh),h(e,ks,n),h(e,_,n),o(_,rh),o(_,Ll),o(Ll,ih),o(_,nh),o(_,Wl),o(Wl,hh),o(_,uh),o(_,Gl),o(Gl,fh),o(_,ch),h(e,js,n),h(e,yo,n),o(yo,ph),h(e,Js,n),h(e,S,n),o(S,bo),o(bo,Bl),o(Bl,dh),o(bo,mh),o(S,vh),o(S,tt),o(tt,wh),o(tt,Ol),o(Ol,gh),o(tt,yh),o(S,bh),o(S,ot),o(ot,Eh),o(ot,Hl),o(Hl,_h),o(ot,Th),h(e,Us,n),h(e,he,n),o(he,Mh),o(he,Rl),o(Rl,Ah),o(he,Ih),h(e,Ps,n),h(e,$,n),o($,Sh),o($,Zl),o(Zl,$h),o($,kh),o($,lt),o(lt,jh),o($,Jh),h(e,Ns,n),h(e,L,n),o(L,ue),o(ue,zl),m(at,zl,null),o(L,Uh),o(L,Yl),o(Yl,Ph),h(e,Cs,n),h(e,fe,n),o(fe,st),o(st,Nh),o(st,Dl),o(Dl,Ch),o(st,xh),o(fe,Lh),o(fe,Eo),o(Eo,Wh),o(Eo,rt),o(rt,Gh),h(e,xs,n),h(e,W,n),o(W,ce),o(ce,Fl),m(it,Fl,null),o(W,Bh),o(W,Vl),o(Vl,Oh),h(e,Ls,n),h(e,pe,n),o(pe,Hh),o(pe,nt),o(nt,Rh),o(pe,Zh),h(e,Ws,n),h(e,de,n),o(de,zh),o(de,Xl),o(Xl,Yh),o(de,Dh),h(e,Gs,n),m(ht,e,n),h(e,Bs,n),h(e,_o,n),o(_o,Fh),h(e,Os,n),m(ut,e,n),h(e,Hs,n),h(e,To,n),o(To,Vh),h(e,Rs,n),m(ft,e,n),h(e,Zs,n),h(e,Mo,n),o(Mo,Xh),h(e,zs,n),m(ct,e,n),h(e,Ys,n),h(e,pt,n),o(pt,qh),o(pt,dt),o(dt,Qh),h(e,Ds,n),h(e,G,n),o(G,Kh),o(G,ql),o(ql,eu),o(G,tu),o(G,Ql),o(Ql,ou),h(e,Fs,n),h(e,mt,n),o(mt,lu),o(mt,Kl),o(Kl,au),h(e,Vs,n),h(e,vt,n),o(vt,su),o(vt,wt),o(wt,ru),h(e,Xs,n),h(e,gt,n),o(gt,iu),o(gt,yt),o(yt,nu),h(e,qs,n),h(e,k,n),o(k,hu),o(k,bt),o(bt,uu),o(k,fu),o(k,ea),o(ea,cu),o(k,pu),h(e,Qs,n),h(e,B,n),o(B,me),o(me,ta),m(Et,ta,null),o(B,du),o(B,oa),o(oa,mu),h(e,Ks,n),h(e,O,n),o(O,vu),o(O,la),o(la,wu),o(O,gu),o(O,_t),o(_t,yu),h(e,er,n),h(e,Tt,n),o(Tt,bu),o(Tt,aa),o(aa,Eu),h(e,tr,n),h(e,ve,n),o(ve,Ao),o(ve,_u),o(ve,Io),o(Io,Tu),o(Io,So),o(So,Mu),h(e,or,n),h(e,H,n),o(H,we),o(we,sa),m(Mt,sa,null),o(H,Au),o(H,ra),o(ra,Iu),h(e,lr,n),h(e,$o,n),o($o,Su),h(e,ar,n),h(e,At,n),h(e,sr,n),h(e,R,n),o(R,ge),o(ge,ia),m(It,ia,null),o(R,$u),o(R,na),o(na,ku),h(e,rr,n),h(e,ko,n),o(ko,ju),h(e,ir,n),h(e,j,n),o(j,ha),o(ha,Ju),o(j,Uu),o(j,ua),o(ua,Pu),o(j,Nu),o(j,jo),o(jo,Cu),o(jo,y),o(y,fa),o(fa,xu),o(y,Lu),o(y,ca),o(ca,Wu),o(y,Gu),o(y,pa),o(pa,Bu),o(y,Ou),o(y,da),o(da,Hu),o(y,Ru),o(y,ma),o(ma,Zu),o(y,zu),o(y,va),o(va,Yu),h(e,nr,n),h(e,Z,n),o(Z,ye),o(ye,wa),m(St,wa,null),o(Z,Du),o(Z,ga),o(ga,Fu),h(e,hr,n),h(e,Jo,n),o(Jo,Vu),h(e,ur,n),h(e,$t,n),h(e,fr,n),h(e,z,n),o(z,be),o(be,ya),m(kt,ya,null),o(z,Xu),o(z,ba),o(ba,qu),h(e,cr,n),h(e,Ee,n),o(Ee,Qu),o(Ee,Ea),o(Ea,Ku),o(Ee,ef),h(e,pr,n),h(e,jt,n),o(jt,tf),o(jt,_a),o(_a,of),h(e,dr,n),h(e,J,n),o(J,lf),o(J,Jt),o(Jt,af),o(J,sf),o(J,Ta),o(Ta,rf),o(J,nf),h(e,mr,n),h(e,_e,n),o(_e,hf),o(_e,Ma),o(Ma,uf),o(_e,ff),h(e,vr,n),h(e,Te,n),o(Te,cf),o(Te,Aa),o(Aa,pf),o(Te,df),h(e,wr,n),h(e,U,n),o(U,mf),o(U,Ia),o(Ia,vf),o(U,wf),o(U,Sa),o(Sa,gf),o(U,yf),h(e,gr,n),h(e,Me,n),o(Me,Uo),o(Me,bf),o(Me,Po),o(Po,Ef),o(Po,No),o(No,_f),h(e,yr,n),h(e,Co,n),o(Co,Tf),h(e,br,n),h(e,Ae,n),o(Ae,Mf),o(Ae,Ut),o(Ut,Af),o(Ae,If),h(e,Er,n),h(e,Y,n),o(Y,Ie),o(Ie,$a),m(Pt,$a,null),o(Y,Sf),o(Y,ka),o(ka,$f),h(e,_r,n),h(e,D,n),o(D,kf),o(D,Nt),o(Nt,jf),o(D,Jf),o(D,Ct),o(Ct,Uf),o(Ct,ja),o(ja,Pf),o(Ct,Nf),h(e,Tr,n),h(e,xt,n),o(xt,Cf),o(xt,Ja),o(Ja,Lt),o(Lt,xf),h(e,Mr,n),h(e,Se,n),o(Se,Lf),o(Se,Ua),o(Ua,Wf),o(Se,Gf),h(e,Ar,n),m(Wt,e,n),h(e,Ir,n),h(e,xo,n),o(xo,Bf),h(e,Sr,n),h(e,Gt,n),o(Gt,Of),o(Gt,Lo),o(Lo,Bt),o(Bt,Hf),o(Lo,Rf),h(e,$r,n),h(e,Wo,n),o(Wo,Zf),h(e,kr,n),h(e,F,n),o(F,$e),o($e,Pa),m(Ot,Pa,null),o(F,zf),o(F,Na),o(Na,Yf),h(e,jr,n),h(e,Ht,n),o(Ht,Df),o(Ht,Ca),o(Ca,Ff),h(e,Jr,n),h(e,Go,n),o(Go,Vf),h(e,Ur,n),h(e,T,n),o(T,Bo),o(Bo,xa),o(xa,Xf),o(Bo,qf),o(T,Qf),o(T,Oo),o(Oo,La),o(La,Kf),o(Oo,ec),o(T,tc),o(T,Ho),o(Ho,Wa),o(Wa,oc),o(Ho,lc),o(T,ac),o(T,Ro),o(Ro,Ga),o(Ga,sc),o(Ro,rc),h(e,Pr,n),h(e,Zo,n),o(Zo,ic),h(e,Nr,n),h(e,zo,n),o(zo,nc),h(e,Cr,n),m(Rt,e,n),h(e,xr,n),h(e,Yo,n),o(Yo,hc),h(e,Lr,n),h(e,Do,n),o(Do,uc),h(e,Wr,n),h(e,Fo,n),o(Fo,fc),h(e,Gr,n),h(e,V,n),o(V,ke),o(ke,Ba),m(Zt,Ba,null),o(V,cc),o(V,Oa),o(Oa,pc),h(e,Br,n),h(e,zt,n),o(zt,dc),o(zt,Ha),o(Ha,mc),h(e,Or,n),h(e,Vo,n),o(Vo,vc),h(e,Hr,n),h(e,Yt,n),o(Yt,wc),o(Yt,Dt),o(Dt,gc),h(e,Rr,n),h(e,Xo,n),o(Xo,yc),h(e,Zr,n),h(e,X,n),o(X,bc),o(X,Ft),o(Ft,Ec),o(X,_c),o(X,Ra),o(Ra,Tc),h(e,zr,n),h(e,Vt,n),h(e,Yr,n),h(e,qo,n),o(qo,Mc),h(e,Dr,n),m(Xt,e,n),h(e,Fr,n),h(e,je,n),o(je,Ac),o(je,Za),o(Za,Ic),o(je,Sc),h(e,Vr,n),h(e,Qo,n),o(Qo,$c),h(e,Xr,n),h(e,M,n),o(M,Ko),o(Ko,za),o(za,kc),o(Ko,jc),o(M,Jc),o(M,el),o(el,Ya),o(Ya,Uc),o(el,Pc),o(M,Nc),o(M,qt),o(qt,Da),o(Da,Cc),o(qt,xc),o(qt,Fa),o(Fa,Lc),o(M,Wc),o(M,tl),o(tl,Va),o(Va,Gc),o(tl,Bc),h(e,qr,n),h(e,ol,n),o(ol,Oc),h(e,Qr,n),m(Qt,e,n),h(e,Kr,n),m(Kt,e,n),h(e,ei,n),h(e,ll,n),o(ll,Hc),h(e,ti,n),h(e,eo,n),o(eo,Rc),o(eo,to),o(to,Zc),h(e,oi,n),h(e,oo,n),o(oo,zc),o(oo,Xa),o(Xa,Yc),h(e,li,n),h(e,q,n),o(q,Je),o(Je,qa),m(lo,qa,null),o(q,Dc),o(q,Qa),o(Qa,Fc),h(e,ai,n),h(e,Ue,n),o(Ue,Vc),o(Ue,Ka),o(Ka,Xc),o(Ue,qc),h(e,si,n),h(e,al,n),o(al,Qc),h(e,ri,n),h(e,sl,n),o(sl,es),o(es,Kc),h(e,ii,n),h(e,ao,n),h(e,ni,n),h(e,rl,n),o(rl,ep),h(e,hi,n),h(e,so,n),h(e,ui,n),h(e,ro,n),o(ro,io),o(io,tp),o(io,ts),o(ts,op),o(io,lp),h(e,fi,n),h(e,no,n),h(e,ci,n),h(e,il,n),o(il,ap),h(e,pi,n),h(e,Q,n),o(Q,Pe),o(Pe,os),m(ho,os,null),o(Q,sp),o(Q,ls),o(ls,rp),h(e,di,n),h(e,K,n),o(K,ip),o(K,as),o(as,np),o(K,hp),o(K,uo),o(uo,up),h(e,mi,n),h(e,nl,n),o(nl,fp),h(e,vi,n),h(e,hl,n),o(hl,ul),o(ul,cp),o(ul,fo),o(fo,pp),h(e,wi,n),h(e,Ne,n),o(Ne,dp),o(Ne,ss),o(ss,mp),o(Ne,vp),h(e,gi,n),h(e,fl,n),o(fl,wp),yi=!0},p:gw,i(e){yi||(v(Be.$$.fragment,e),v(ze.$$.fragment,e),v(et.$$.fragment,e),v(at.$$.fragment,e),v(it.$$.fragment,e),v(ht.$$.fragment,e),v(ut.$$.fragment,e),v(ft.$$.fragment,e),v(ct.$$.fragment,e),v(Et.$$.fragment,e),v(Mt.$$.fragment,e),v(It.$$.fragment,e),v(St.$$.fragment,e),v(kt.$$.fragment,e),v(Pt.$$.fragment,e),v(Wt.$$.fragment,e),v(Ot.$$.fragment,e),v(Rt.$$.fragment,e),v(Zt.$$.fragment,e),v(Xt.$$.fragment,e),v(Qt.$$.fragment,e),v(Kt.$$.fragment,e),v(lo.$$.fragment,e),v(ho.$$.fragment,e),yi=!0)},o(e){w(Be.$$.fragment,e),w(ze.$$.fragment,e),w(et.$$.fragment,e),w(at.$$.fragment,e),w(it.$$.fragment,e),w(ht.$$.fragment,e),w(ut.$$.fragment,e),w(ft.$$.fragment,e),w(ct.$$.fragment,e),w(Et.$$.fragment,e),w(Mt.$$.fragment,e),w(It.$$.fragment,e),w(St.$$.fragment,e),w(kt.$$.fragment,e),w(Pt.$$.fragment,e),w(Wt.$$.fragment,e),w(Ot.$$.fragment,e),w(Rt.$$.fragment,e),w(Zt.$$.fragment,e),w(Xt.$$.fragment,e),w(Qt.$$.fragment,e),w(Kt.$$.fragment,e),w(lo.$$.fragment,e),w(ho.$$.fragment,e),yi=!1},d(e){t(P),e&&t(ps),e&&t(N),g(Be),e&&t(ds),e&&t(oe),e&&t(ms),e&&t(Oe),e&&t(vs),e&&t(He),e&&t(ws),e&&t(Re),e&&t(gs),e&&t(le),e&&t(ys),e&&t(co),e&&t(bs),e&&t(C),g(ze),e&&t(Es),e&&t(po),e&&t(_s),e&&t(Ye),e&&t(Ts),e&&t(mo),e&&t(Ms),e&&t(I),e&&t(As),e&&t(Xe),e&&t(Is),e&&t(ie),e&&t(Ss),e&&t(E),e&&t($s),e&&t(x),g(et),e&&t(ks),e&&t(_),e&&t(js),e&&t(yo),e&&t(Js),e&&t(S),e&&t(Us),e&&t(he),e&&t(Ps),e&&t($),e&&t(Ns),e&&t(L),g(at),e&&t(Cs),e&&t(fe),e&&t(xs),e&&t(W),g(it),e&&t(Ls),e&&t(pe),e&&t(Ws),e&&t(de),e&&t(Gs),g(ht,e),e&&t(Bs),e&&t(_o),e&&t(Os),g(ut,e),e&&t(Hs),e&&t(To),e&&t(Rs),g(ft,e),e&&t(Zs),e&&t(Mo),e&&t(zs),g(ct,e),e&&t(Ys),e&&t(pt),e&&t(Ds),e&&t(G),e&&t(Fs),e&&t(mt),e&&t(Vs),e&&t(vt),e&&t(Xs),e&&t(gt),e&&t(qs),e&&t(k),e&&t(Qs),e&&t(B),g(Et),e&&t(Ks),e&&t(O),e&&t(er),e&&t(Tt),e&&t(tr),e&&t(ve),e&&t(or),e&&t(H),g(Mt),e&&t(lr),e&&t($o),e&&t(ar),e&&t(At),e&&t(sr),e&&t(R),g(It),e&&t(rr),e&&t(ko),e&&t(ir),e&&t(j),e&&t(nr),e&&t(Z),g(St),e&&t(hr),e&&t(Jo),e&&t(ur),e&&t($t),e&&t(fr),e&&t(z),g(kt),e&&t(cr),e&&t(Ee),e&&t(pr),e&&t(jt),e&&t(dr),e&&t(J),e&&t(mr),e&&t(_e),e&&t(vr),e&&t(Te),e&&t(wr),e&&t(U),e&&t(gr),e&&t(Me),e&&t(yr),e&&t(Co),e&&t(br),e&&t(Ae),e&&t(Er),e&&t(Y),g(Pt),e&&t(_r),e&&t(D),e&&t(Tr),e&&t(xt),e&&t(Mr),e&&t(Se),e&&t(Ar),g(Wt,e),e&&t(Ir),e&&t(xo),e&&t(Sr),e&&t(Gt),e&&t($r),e&&t(Wo),e&&t(kr),e&&t(F),g(Ot),e&&t(jr),e&&t(Ht),e&&t(Jr),e&&t(Go),e&&t(Ur),e&&t(T),e&&t(Pr),e&&t(Zo),e&&t(Nr),e&&t(zo),e&&t(Cr),g(Rt,e),e&&t(xr),e&&t(Yo),e&&t(Lr),e&&t(Do),e&&t(Wr),e&&t(Fo),e&&t(Gr),e&&t(V),g(Zt),e&&t(Br),e&&t(zt),e&&t(Or),e&&t(Vo),e&&t(Hr),e&&t(Yt),e&&t(Rr),e&&t(Xo),e&&t(Zr),e&&t(X),e&&t(zr),e&&t(Vt),e&&t(Yr),e&&t(qo),e&&t(Dr),g(Xt,e),e&&t(Fr),e&&t(je),e&&t(Vr),e&&t(Qo),e&&t(Xr),e&&t(M),e&&t(qr),e&&t(ol),e&&t(Qr),g(Qt,e),e&&t(Kr),g(Kt,e),e&&t(ei),e&&t(ll),e&&t(ti),e&&t(eo),e&&t(oi),e&&t(oo),e&&t(li),e&&t(q),g(lo),e&&t(ai),e&&t(Ue),e&&t(si),e&&t(al),e&&t(ri),e&&t(sl),e&&t(ii),e&&t(ao),e&&t(ni),e&&t(rl),e&&t(hi),e&&t(so),e&&t(ui),e&&t(ro),e&&t(fi),e&&t(no),e&&t(ci),e&&t(il),e&&t(pi),e&&t(Q),g(ho),e&&t(di),e&&t(K),e&&t(mi),e&&t(nl),e&&t(vi),e&&t(hl),e&&t(wi),e&&t(Ne),e&&t(gi),e&&t(fl)}}}const Ew={local:"handson",sections:[{local:"what-is-ai-vs-ai",sections:[{local:"competition-rules",title:"Competition Rules"},{local:"chat-with-your-classmates-share-advice-and-ask-questions-on-discord",title:"Chat with your classmates, share advice and ask questions on Discord"}],title:"What is AI vs. AI?"},{local:"step-0-install-mlagents-and-download-the-correct-executable",title:"Step 0: Install MLAgents and download the correct executable"},{local:"step-1-understand-the-environment",sections:[{local:"the-reward-function",title:"The reward function"},{local:"the-observation-space",title:"The observation space"},{local:"the-action-space",title:"The action space"}],title:"Step 1: Understand the environment"},{local:"step-2-understand-mapoca",title:"Step 2: Understand MA-POCA"},{local:"step-3-define-the-config-file",title:"Step 3: Define the config file"},{local:"step-4-start-the-training",title:"Step 4: Start the training"},{local:"step-5-push-the-agent-to-the-hugging-face-hub",title:"Step 5: **Push the agent to the Hugging Face Hub**"},{local:"step-6-verify-that-your-model-is-ready-for-ai-vs-ai-challenge",title:"Step 6: Verify that your model is ready for AI vs AI Challenge"},{local:"step-7-visualize-some-match-in-our-demo",title:"Step 7: Visualize some match in our demo"}],title:"Hands-on"};function _w(Kp){return yw(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Iw extends dw{constructor(P){super();mw(this,P,_w,bw,vw,{})}}export{Iw as default,Ew as metadata};
