import{S as Eo,i as To,s as wo,e as a,k as h,w as yo,t as s,M as bo,c as o,d as t,m as c,a as r,x as No,h as n,b as E,G as e,g as u,y as Qo,L as So,q as Ro,o as Go,B as xo,v as Lo}from"../../chunks/vendor-hf-doc-builder.js";import{I as ko}from"../../chunks/IconCopyLink-hf-doc-builder.js";function Oo(Na){let T,$e,w,b,H,L,Xe,J,Ye,qe,M,Ze,Me,d,j,N,K,et,tt,W,at,ot,rt,X,Q,Y,st,nt,Z,lt,it,ht,ee,S,te,ct,pt,ae,ft,dt,gt,k,oe,re,ut,mt,O,R,se,vt,_t,ne,Et,Tt,wt,C,le,yt,bt,Nt,I,ie,he,Qt,St,m,ce,U,pe,Rt,Gt,xt,fe,B,de,Lt,kt,Ot,ge,i,ue,It,Pt,me,Dt,At,ve,Vt,$t,_e,qt,Mt,Ee,Ct,Ut,Te,Bt,zt,we,Ft,Ht,ye,Jt,jt,Kt,P,g,be,Wt,Xt,Ne,Yt,Zt,Qe,ea,ta,Se,aa,oa,ra,D,G,Re,sa,na,Ge,la,ia,ha,v,xe,ca,pa,Le,fa,da,ke,ga,ua,Ce,A,ma,V,va,Ue,z,_a,Be,F,Oe,$,Ea,ze;return L=new ko({}),{c(){T=a("meta"),$e=h(),w=a("h1"),b=a("a"),H=a("span"),yo(L.$$.fragment),Xe=h(),J=a("span"),Ye=s("Glossary"),qe=h(),M=a("p"),Ze=s("This is a community-created glossary. Contributions are welcomed!"),Me=h(),d=a("ul"),j=a("li"),N=a("p"),K=a("strong"),et=s("Tabular Method:"),tt=s(` Type of problem in which the state and action spaces are small enough to approximate value functions to be represented as arrays and tables.
`),W=a("strong"),at=s("Q-learning"),ot=s(" is an example of tabular method since a table is used to represent the value for different state-action pairs."),rt=h(),X=a("li"),Q=a("p"),Y=a("strong"),st=s("Deep Q-Learning:"),nt=s(" Method that trains a neural network to approximate, given a state, the different "),Z=a("strong"),lt=s("Q-values"),it=s(` for each possible action at that state.
It is used to solve problems when observational space is too big to apply a tabular Q-Learning approach.`),ht=h(),ee=a("li"),S=a("p"),te=a("strong"),ct=s("Temporal Limitation"),pt=s(` is a difficulty presented when the environment state is represented by frames. A frame by itself does not provide temporal information.
In order to obtain temporal information, we need to `),ae=a("strong"),ft=s("stack"),dt=s(" a number of frames together."),gt=h(),k=a("li"),oe=a("p"),re=a("strong"),ut=s("Phases of Deep Q-Learning:"),mt=h(),O=a("ul"),R=a("li"),se=a("strong"),vt=s("Sampling:"),_t=s(" Actions are performed, and observed experience tuples are stored in a "),ne=a("strong"),Et=s("replay memory"),Tt=s("."),wt=h(),C=a("li"),le=a("strong"),yt=s("Training:"),bt=s(" Batches of tuples are selected randomly and the neural network updates its weights using gradient descent."),Nt=h(),I=a("li"),ie=a("p"),he=a("strong"),Qt=s("Solutions to stabilize Deep Q-Learning:"),St=h(),m=a("ul"),ce=a("li"),U=a("p"),pe=a("strong"),Rt=s("Experience Replay:"),Gt=s(` A replay memory is created to save experiences samples that can be reused during training.
This allows the agent to learn from the same experiences multiple times. Also, it helps the agent avoid forgetting previous experiences as it gets new ones.`),xt=h(),fe=a("li"),B=a("p"),de=a("strong"),Lt=s("Random sampling"),kt=s(` from replay buffer allows to remove correlation in the observation sequences and prevents action values from oscillating or diverging
catastrophically.`),Ot=h(),ge=a("li"),i=a("p"),ue=a("strong"),It=s("Fixed Q-Target:"),Pt=s(" In order to calculate the "),me=a("strong"),Dt=s("Q-Target"),At=s(" we need to estimate the discounted optimal "),ve=a("strong"),Vt=s("Q-value"),$t=s(` of the next state by using Bellman equation. The problem
is that the same network weights are used to calculate the `),_e=a("strong"),qt=s("Q-Target"),Mt=s(" and the "),Ee=a("strong"),Ct=s("Q-value"),Ut=s(". This means that everytime we are modifying the "),Te=a("strong"),Bt=s("Q-value"),zt=s(", the "),we=a("strong"),Ft=s("Q-Target"),Ht=s(` also moves with it.
To avoid this issue, a separate network with fixed parameters is used for estimating the Temporal Difference Target. The target network is updated by copying parameters from
our Deep Q-Network after certain `),ye=a("strong"),Jt=s("C steps"),jt=s("."),Kt=h(),P=a("li"),g=a("p"),be=a("strong"),Wt=s("Double DQN:"),Xt=s(" Method to handle "),Ne=a("strong"),Yt=s("overestimation"),Zt=s(" of "),Qe=a("strong"),ea=s("Q-Values"),ta=s(". This solution uses two networks to decouple the action selection from the target "),Se=a("strong"),aa=s("Value generation"),oa=s(":"),ra=h(),D=a("ul"),G=a("li"),Re=a("strong"),sa=s("DQN Network"),na=s(" to select the best action to take for the next state (the action with the highest "),Ge=a("strong"),la=s("Q-Value"),ia=s(")"),ha=h(),v=a("li"),xe=a("strong"),ca=s("Target Network"),pa=s(" to calculate the target "),Le=a("strong"),fa=s("Q-Value"),da=s(` of taking that action at the next state.
This approach reduces the `),ke=a("strong"),ga=s("Q-Values"),ua=s(" overestimation, it helps to train faster and have more stable learning."),Ce=h(),A=a("p"),ma=s("If you want to improve the course, you can "),V=a("a"),va=s("open a Pull Request."),Ue=h(),z=a("p"),_a=s("This glossary was made possible thanks to:"),Be=h(),F=a("ul"),Oe=a("li"),$=a("a"),Ea=s("Dario Paez"),this.h()},l(l){const f=bo('[data-svelte="svelte-1phssyn"]',document.head);T=o(f,"META",{name:!0,content:!0}),f.forEach(t),$e=c(l),w=o(l,"H1",{class:!0});var Fe=r(w);b=o(Fe,"A",{id:!0,class:!0,href:!0});var Qa=r(b);H=o(Qa,"SPAN",{});var Sa=r(H);No(L.$$.fragment,Sa),Sa.forEach(t),Qa.forEach(t),Xe=c(Fe),J=o(Fe,"SPAN",{});var Ra=r(J);Ye=n(Ra,"Glossary"),Ra.forEach(t),Fe.forEach(t),qe=c(l),M=o(l,"P",{});var Ga=r(M);Ze=n(Ga,"This is a community-created glossary. Contributions are welcomed!"),Ga.forEach(t),Me=c(l),d=o(l,"UL",{});var _=r(d);j=o(_,"LI",{});var xa=r(j);N=o(xa,"P",{});var Ie=r(N);K=o(Ie,"STRONG",{});var La=r(K);et=n(La,"Tabular Method:"),La.forEach(t),tt=n(Ie,` Type of problem in which the state and action spaces are small enough to approximate value functions to be represented as arrays and tables.
`),W=o(Ie,"STRONG",{});var ka=r(W);at=n(ka,"Q-learning"),ka.forEach(t),ot=n(Ie," is an example of tabular method since a table is used to represent the value for different state-action pairs."),Ie.forEach(t),xa.forEach(t),rt=c(_),X=o(_,"LI",{});var Oa=r(X);Q=o(Oa,"P",{});var Pe=r(Q);Y=o(Pe,"STRONG",{});var Ia=r(Y);st=n(Ia,"Deep Q-Learning:"),Ia.forEach(t),nt=n(Pe," Method that trains a neural network to approximate, given a state, the different "),Z=o(Pe,"STRONG",{});var Pa=r(Z);lt=n(Pa,"Q-values"),Pa.forEach(t),it=n(Pe,` for each possible action at that state.
It is used to solve problems when observational space is too big to apply a tabular Q-Learning approach.`),Pe.forEach(t),Oa.forEach(t),ht=c(_),ee=o(_,"LI",{});var Da=r(ee);S=o(Da,"P",{});var De=r(S);te=o(De,"STRONG",{});var Aa=r(te);ct=n(Aa,"Temporal Limitation"),Aa.forEach(t),pt=n(De,` is a difficulty presented when the environment state is represented by frames. A frame by itself does not provide temporal information.
In order to obtain temporal information, we need to `),ae=o(De,"STRONG",{});var Va=r(ae);ft=n(Va,"stack"),Va.forEach(t),dt=n(De," a number of frames together."),De.forEach(t),Da.forEach(t),gt=c(_),k=o(_,"LI",{});var He=r(k);oe=o(He,"P",{});var $a=r(oe);re=o($a,"STRONG",{});var qa=r(re);ut=n(qa,"Phases of Deep Q-Learning:"),qa.forEach(t),$a.forEach(t),mt=c(He),O=o(He,"UL",{});var Je=r(O);R=o(Je,"LI",{});var Ae=r(R);se=o(Ae,"STRONG",{});var Ma=r(se);vt=n(Ma,"Sampling:"),Ma.forEach(t),_t=n(Ae," Actions are performed, and observed experience tuples are stored in a "),ne=o(Ae,"STRONG",{});var Ca=r(ne);Et=n(Ca,"replay memory"),Ca.forEach(t),Tt=n(Ae,"."),Ae.forEach(t),wt=c(Je),C=o(Je,"LI",{});var Ta=r(C);le=o(Ta,"STRONG",{});var Ua=r(le);yt=n(Ua,"Training:"),Ua.forEach(t),bt=n(Ta," Batches of tuples are selected randomly and the neural network updates its weights using gradient descent."),Ta.forEach(t),Je.forEach(t),He.forEach(t),Nt=c(_),I=o(_,"LI",{});var je=r(I);ie=o(je,"P",{});var Ba=r(ie);he=o(Ba,"STRONG",{});var za=r(he);Qt=n(za,"Solutions to stabilize Deep Q-Learning:"),za.forEach(t),Ba.forEach(t),St=c(je),m=o(je,"UL",{});var x=r(m);ce=o(x,"LI",{});var Fa=r(ce);U=o(Fa,"P",{});var wa=r(U);pe=o(wa,"STRONG",{});var Ha=r(pe);Rt=n(Ha,"Experience Replay:"),Ha.forEach(t),Gt=n(wa,` A replay memory is created to save experiences samples that can be reused during training.
This allows the agent to learn from the same experiences multiple times. Also, it helps the agent avoid forgetting previous experiences as it gets new ones.`),wa.forEach(t),Fa.forEach(t),xt=c(x),fe=o(x,"LI",{});var Ja=r(fe);B=o(Ja,"P",{});var ya=r(B);de=o(ya,"STRONG",{});var ja=r(de);Lt=n(ja,"Random sampling"),ja.forEach(t),kt=n(ya,` from replay buffer allows to remove correlation in the observation sequences and prevents action values from oscillating or diverging
catastrophically.`),ya.forEach(t),Ja.forEach(t),Ot=c(x),ge=o(x,"LI",{});var Ka=r(ge);i=o(Ka,"P",{});var p=r(i);ue=o(p,"STRONG",{});var Wa=r(ue);It=n(Wa,"Fixed Q-Target:"),Wa.forEach(t),Pt=n(p," In order to calculate the "),me=o(p,"STRONG",{});var Xa=r(me);Dt=n(Xa,"Q-Target"),Xa.forEach(t),At=n(p," we need to estimate the discounted optimal "),ve=o(p,"STRONG",{});var Ya=r(ve);Vt=n(Ya,"Q-value"),Ya.forEach(t),$t=n(p,` of the next state by using Bellman equation. The problem
is that the same network weights are used to calculate the `),_e=o(p,"STRONG",{});var Za=r(_e);qt=n(Za,"Q-Target"),Za.forEach(t),Mt=n(p," and the "),Ee=o(p,"STRONG",{});var eo=r(Ee);Ct=n(eo,"Q-value"),eo.forEach(t),Ut=n(p,". This means that everytime we are modifying the "),Te=o(p,"STRONG",{});var to=r(Te);Bt=n(to,"Q-value"),to.forEach(t),zt=n(p,", the "),we=o(p,"STRONG",{});var ao=r(we);Ft=n(ao,"Q-Target"),ao.forEach(t),Ht=n(p,` also moves with it.
To avoid this issue, a separate network with fixed parameters is used for estimating the Temporal Difference Target. The target network is updated by copying parameters from
our Deep Q-Network after certain `),ye=o(p,"STRONG",{});var oo=r(ye);Jt=n(oo,"C steps"),oo.forEach(t),jt=n(p,"."),p.forEach(t),Ka.forEach(t),Kt=c(x),P=o(x,"LI",{});var Ke=r(P);g=o(Ke,"P",{});var y=r(g);be=o(y,"STRONG",{});var ro=r(be);Wt=n(ro,"Double DQN:"),ro.forEach(t),Xt=n(y," Method to handle "),Ne=o(y,"STRONG",{});var so=r(Ne);Yt=n(so,"overestimation"),so.forEach(t),Zt=n(y," of "),Qe=o(y,"STRONG",{});var no=r(Qe);ea=n(no,"Q-Values"),no.forEach(t),ta=n(y,". This solution uses two networks to decouple the action selection from the target "),Se=o(y,"STRONG",{});var lo=r(Se);aa=n(lo,"Value generation"),lo.forEach(t),oa=n(y,":"),y.forEach(t),ra=c(Ke),D=o(Ke,"UL",{});var We=r(D);G=o(We,"LI",{});var Ve=r(G);Re=o(Ve,"STRONG",{});var io=r(Re);sa=n(io,"DQN Network"),io.forEach(t),na=n(Ve," to select the best action to take for the next state (the action with the highest "),Ge=o(Ve,"STRONG",{});var ho=r(Ge);la=n(ho,"Q-Value"),ho.forEach(t),ia=n(Ve,")"),Ve.forEach(t),ha=c(We),v=o(We,"LI",{});var q=r(v);xe=o(q,"STRONG",{});var co=r(xe);ca=n(co,"Target Network"),co.forEach(t),pa=n(q," to calculate the target "),Le=o(q,"STRONG",{});var po=r(Le);fa=n(po,"Q-Value"),po.forEach(t),da=n(q,` of taking that action at the next state.
This approach reduces the `),ke=o(q,"STRONG",{});var fo=r(ke);ga=n(fo,"Q-Values"),fo.forEach(t),ua=n(q," overestimation, it helps to train faster and have more stable learning."),q.forEach(t),We.forEach(t),Ke.forEach(t),x.forEach(t),je.forEach(t),_.forEach(t),Ce=c(l),A=o(l,"P",{});var ba=r(A);ma=n(ba,"If you want to improve the course, you can "),V=o(ba,"A",{href:!0,rel:!0});var go=r(V);va=n(go,"open a Pull Request."),go.forEach(t),ba.forEach(t),Ue=c(l),z=o(l,"P",{});var uo=r(z);_a=n(uo,"This glossary was made possible thanks to:"),uo.forEach(t),Be=c(l),F=o(l,"UL",{});var mo=r(F);Oe=o(mo,"LI",{});var vo=r(Oe);$=o(vo,"A",{href:!0,rel:!0});var _o=r($);Ea=n(_o,"Dario Paez"),_o.forEach(t),vo.forEach(t),mo.forEach(t),this.h()},h(){E(T,"name","hf:doc:metadata"),E(T,"content",JSON.stringify(Io)),E(b,"id","glossary"),E(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(b,"href","#glossary"),E(w,"class","relative group"),E(V,"href","https://github.com/huggingface/deep-rl-class/pulls"),E(V,"rel","nofollow"),E($,"href","https://github.com/dario248"),E($,"rel","nofollow")},m(l,f){e(document.head,T),u(l,$e,f),u(l,w,f),e(w,b),e(b,H),Qo(L,H,null),e(w,Xe),e(w,J),e(J,Ye),u(l,qe,f),u(l,M,f),e(M,Ze),u(l,Me,f),u(l,d,f),e(d,j),e(j,N),e(N,K),e(K,et),e(N,tt),e(N,W),e(W,at),e(N,ot),e(d,rt),e(d,X),e(X,Q),e(Q,Y),e(Y,st),e(Q,nt),e(Q,Z),e(Z,lt),e(Q,it),e(d,ht),e(d,ee),e(ee,S),e(S,te),e(te,ct),e(S,pt),e(S,ae),e(ae,ft),e(S,dt),e(d,gt),e(d,k),e(k,oe),e(oe,re),e(re,ut),e(k,mt),e(k,O),e(O,R),e(R,se),e(se,vt),e(R,_t),e(R,ne),e(ne,Et),e(R,Tt),e(O,wt),e(O,C),e(C,le),e(le,yt),e(C,bt),e(d,Nt),e(d,I),e(I,ie),e(ie,he),e(he,Qt),e(I,St),e(I,m),e(m,ce),e(ce,U),e(U,pe),e(pe,Rt),e(U,Gt),e(m,xt),e(m,fe),e(fe,B),e(B,de),e(de,Lt),e(B,kt),e(m,Ot),e(m,ge),e(ge,i),e(i,ue),e(ue,It),e(i,Pt),e(i,me),e(me,Dt),e(i,At),e(i,ve),e(ve,Vt),e(i,$t),e(i,_e),e(_e,qt),e(i,Mt),e(i,Ee),e(Ee,Ct),e(i,Ut),e(i,Te),e(Te,Bt),e(i,zt),e(i,we),e(we,Ft),e(i,Ht),e(i,ye),e(ye,Jt),e(i,jt),e(m,Kt),e(m,P),e(P,g),e(g,be),e(be,Wt),e(g,Xt),e(g,Ne),e(Ne,Yt),e(g,Zt),e(g,Qe),e(Qe,ea),e(g,ta),e(g,Se),e(Se,aa),e(g,oa),e(P,ra),e(P,D),e(D,G),e(G,Re),e(Re,sa),e(G,na),e(G,Ge),e(Ge,la),e(G,ia),e(D,ha),e(D,v),e(v,xe),e(xe,ca),e(v,pa),e(v,Le),e(Le,fa),e(v,da),e(v,ke),e(ke,ga),e(v,ua),u(l,Ce,f),u(l,A,f),e(A,ma),e(A,V),e(V,va),u(l,Ue,f),u(l,z,f),e(z,_a),u(l,Be,f),u(l,F,f),e(F,Oe),e(Oe,$),e($,Ea),ze=!0},p:So,i(l){ze||(Ro(L.$$.fragment,l),ze=!0)},o(l){Go(L.$$.fragment,l),ze=!1},d(l){t(T),l&&t($e),l&&t(w),xo(L),l&&t(qe),l&&t(M),l&&t(Me),l&&t(d),l&&t(Ce),l&&t(A),l&&t(Ue),l&&t(z),l&&t(Be),l&&t(F)}}}const Io={local:"glossary",title:"Glossary "};function Po(Na){return Lo(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Vo extends Eo{constructor(T){super();To(this,T,Po,Oo,wo,{})}}export{Vo as default,Io as metadata};
