import{S as ao,i as ro,s as so,e as s,k as h,w as Kt,t as a,M as io,c as i,d as t,m as u,a as p,x as Vt,h as r,b as c,N as ne,G as o,g as l,y as Xt,L as no,q as Zt,o as eo,B as to,v as lo}from"../../chunks/vendor-hf-doc-builder.js";import{I as oo}from"../../chunks/IconCopyLink-hf-doc-builder.js";function po(kt){let d,le,_,N,j,Q,De,C,Le,pe,P,Nt,ce,f,$e,H,Re,Oe,U,We,Ie,he,m,Ae,z,qe,Be,F,Me,je,ue,y,E,Y,S,Ce,J,He,fe,g,Ue,K,ze,Fe,V,Ye,Je,me,w,Ke,X,Ve,Xe,Z,Ze,et,ge,T,tt,ee,ot,at,we,G,Et,ve,k,te,rt,st,oe,it,nt,de,x,Tt,_e,b,lt,ae,pt,ct,ye,D,bt,ke,v,ht,re,ut,ft,se,mt,gt,Ne,L,wt,$,vt,Ee,O,dt,Te,R,Qt,be,W,_t,Qe;return Q=new oo({}),S=new oo({}),{c(){d=s("meta"),le=h(),_=s("h1"),N=s("a"),j=s("span"),Kt(Q.$$.fragment),De=h(),C=s("span"),Le=a("The Deep Q-Network (DQN)"),pe=a(`

This is the architecture of our Deep Q-Learning network:
`),P=s("img"),ce=h(),f=s("p"),$e=a("As input, we take a "),H=s("strong"),Re=a("stack of 4 frames"),Oe=a(" passed through the network as a state and output a "),U=s("strong"),We=a("vector of Q-values for each possible action at that state"),Ie=a(". Then, like with Q-Learning, we just need to use our epsilon-greedy policy to select which action to take."),he=h(),m=s("p"),Ae=a("When the Neural Network is initialized, "),z=s("strong"),qe=a("the Q-value estimation is terrible"),Be=a(". But during training, our Deep Q-Network agent will associate a situation with the appropriate action and "),F=s("strong"),Me=a("learn to play the game well"),je=a("."),ue=h(),y=s("h2"),E=s("a"),Y=s("span"),Kt(S.$$.fragment),Ce=h(),J=s("span"),He=a("Preprocessing the input and temporal limitation"),fe=h(),g=s("p"),Ue=a("We need to "),K=s("strong"),ze=a("preprocess the input"),Fe=a(". It\u2019s an essential step since we want to "),V=s("strong"),Ye=a("reduce the complexity of our state to reduce the computation time needed for training"),Je=a("."),me=h(),w=s("p"),Ke=a("To achieve this, we "),X=s("strong"),Ve=a("reduce the state space to 84x84 and grayscale it"),Xe=a(`. We can do this since the colors in Atari environments don\u2019t add important information.
This is a big improvement since we `),Z=s("strong"),Ze=a("reduce our three color channels (RGB) to 1"),et=a("."),ge=h(),T=s("p"),tt=a("We can also "),ee=s("strong"),ot=a("crop a part of the screen in some games"),at=a(` if it does not contain important information.
Then we stack four frames together.`),we=h(),G=s("img"),ve=h(),k=s("p"),te=s("strong"),rt=a("Why do we stack four frames together?"),st=a(`
We stack frames together because it helps us `),oe=s("strong"),it=a("handle the problem of temporal limitation"),nt=a(". Let\u2019s take an example with the game of Pong. When you see this frame:"),de=h(),x=s("img"),_e=h(),b=s("p"),lt=a(`Can you tell me where the ball is going?
No, because one frame is not enough to have a sense of motion! But what if I add three more frames? `),ae=s("strong"),pt=a("Here you can see that the ball is going to the right"),ct=a("."),ye=h(),D=s("img"),ke=a(`
That\u2019s why, to capture temporal information, we stack four frames together.
`),v=s("p"),ht=a("Then the stacked frames are processed by three convolutional layers. These layers "),re=s("strong"),ut=a("allow us to capture and exploit spatial relationships in images"),ft=a(". But also, because the frames are stacked together, "),se=s("strong"),mt=a("we can exploit some temporal properties across those frames"),gt=a("."),Ne=h(),L=s("p"),wt=a("If you don\u2019t know what convolutional layers are, don\u2019t worry. You can check out "),$=s("a"),vt=a("Lesson 4 of this free Deep Learning Course by Udacity"),Ee=h(),O=s("p"),dt=a("Finally, we have a couple of fully connected layers that output a Q-value for each possible action at that state."),Te=h(),R=s("img"),be=h(),W=s("p"),_t=a("So, we see that Deep Q-Learning uses a neural network to approximate, given a state, the different Q-values for each possible action at that state. Now let\u2019s study the Deep Q-Learning algorithm."),this.h()},l(e){const n=io('[data-svelte="svelte-1phssyn"]',document.head);d=i(n,"META",{name:!0,content:!0}),n.forEach(t),le=u(e),_=i(e,"H1",{class:!0});var Pe=p(_);N=i(Pe,"A",{id:!0,class:!0,href:!0});var Pt=p(N);j=i(Pt,"SPAN",{});var St=p(j);Vt(Q.$$.fragment,St),St.forEach(t),Pt.forEach(t),De=u(Pe),C=i(Pe,"SPAN",{});var Gt=p(C);Le=r(Gt,"The Deep Q-Network (DQN)"),Gt.forEach(t),Pe.forEach(t),pe=r(e,`

This is the architecture of our Deep Q-Learning network:
`),P=i(e,"IMG",{src:!0,alt:!0}),ce=u(e),f=i(e,"P",{});var I=p(f);$e=r(I,"As input, we take a "),H=i(I,"STRONG",{});var xt=p(H);Re=r(xt,"stack of 4 frames"),xt.forEach(t),Oe=r(I," passed through the network as a state and output a "),U=i(I,"STRONG",{});var Dt=p(U);We=r(Dt,"vector of Q-values for each possible action at that state"),Dt.forEach(t),Ie=r(I,". Then, like with Q-Learning, we just need to use our epsilon-greedy policy to select which action to take."),I.forEach(t),he=u(e),m=i(e,"P",{});var A=p(m);Ae=r(A,"When the Neural Network is initialized, "),z=i(A,"STRONG",{});var Lt=p(z);qe=r(Lt,"the Q-value estimation is terrible"),Lt.forEach(t),Be=r(A,". But during training, our Deep Q-Network agent will associate a situation with the appropriate action and "),F=i(A,"STRONG",{});var $t=p(F);Me=r($t,"learn to play the game well"),$t.forEach(t),je=r(A,"."),A.forEach(t),ue=u(e),y=i(e,"H2",{class:!0});var Se=p(y);E=i(Se,"A",{id:!0,class:!0,href:!0});var Rt=p(E);Y=i(Rt,"SPAN",{});var Ot=p(Y);Vt(S.$$.fragment,Ot),Ot.forEach(t),Rt.forEach(t),Ce=u(Se),J=i(Se,"SPAN",{});var Wt=p(J);He=r(Wt,"Preprocessing the input and temporal limitation"),Wt.forEach(t),Se.forEach(t),fe=u(e),g=i(e,"P",{});var q=p(g);Ue=r(q,"We need to "),K=i(q,"STRONG",{});var It=p(K);ze=r(It,"preprocess the input"),It.forEach(t),Fe=r(q,". It\u2019s an essential step since we want to "),V=i(q,"STRONG",{});var At=p(V);Ye=r(At,"reduce the complexity of our state to reduce the computation time needed for training"),At.forEach(t),Je=r(q,"."),q.forEach(t),me=u(e),w=i(e,"P",{});var B=p(w);Ke=r(B,"To achieve this, we "),X=i(B,"STRONG",{});var qt=p(X);Ve=r(qt,"reduce the state space to 84x84 and grayscale it"),qt.forEach(t),Xe=r(B,`. We can do this since the colors in Atari environments don\u2019t add important information.
This is a big improvement since we `),Z=i(B,"STRONG",{});var Bt=p(Z);Ze=r(Bt,"reduce our three color channels (RGB) to 1"),Bt.forEach(t),et=r(B,"."),B.forEach(t),ge=u(e),T=i(e,"P",{});var Ge=p(T);tt=r(Ge,"We can also "),ee=i(Ge,"STRONG",{});var Mt=p(ee);ot=r(Mt,"crop a part of the screen in some games"),Mt.forEach(t),at=r(Ge,` if it does not contain important information.
Then we stack four frames together.`),Ge.forEach(t),we=u(e),G=i(e,"IMG",{src:!0,alt:!0}),ve=u(e),k=i(e,"P",{});var ie=p(k);te=i(ie,"STRONG",{});var jt=p(te);rt=r(jt,"Why do we stack four frames together?"),jt.forEach(t),st=r(ie,`
We stack frames together because it helps us `),oe=i(ie,"STRONG",{});var Ct=p(oe);it=r(Ct,"handle the problem of temporal limitation"),Ct.forEach(t),nt=r(ie,". Let\u2019s take an example with the game of Pong. When you see this frame:"),ie.forEach(t),de=u(e),x=i(e,"IMG",{src:!0,alt:!0}),_e=u(e),b=i(e,"P",{});var xe=p(b);lt=r(xe,`Can you tell me where the ball is going?
No, because one frame is not enough to have a sense of motion! But what if I add three more frames? `),ae=i(xe,"STRONG",{});var Ht=p(ae);pt=r(Ht,"Here you can see that the ball is going to the right"),Ht.forEach(t),ct=r(xe,"."),xe.forEach(t),ye=u(e),D=i(e,"IMG",{src:!0,alt:!0}),ke=r(e,`
That\u2019s why, to capture temporal information, we stack four frames together.
`),v=i(e,"P",{});var M=p(v);ht=r(M,"Then the stacked frames are processed by three convolutional layers. These layers "),re=i(M,"STRONG",{});var Ut=p(re);ut=r(Ut,"allow us to capture and exploit spatial relationships in images"),Ut.forEach(t),ft=r(M,". But also, because the frames are stacked together, "),se=i(M,"STRONG",{});var zt=p(se);mt=r(zt,"we can exploit some temporal properties across those frames"),zt.forEach(t),gt=r(M,"."),M.forEach(t),Ne=u(e),L=i(e,"P",{});var yt=p(L);wt=r(yt,"If you don\u2019t know what convolutional layers are, don\u2019t worry. You can check out "),$=i(yt,"A",{href:!0,rel:!0});var Ft=p($);vt=r(Ft,"Lesson 4 of this free Deep Learning Course by Udacity"),Ft.forEach(t),yt.forEach(t),Ee=u(e),O=i(e,"P",{});var Yt=p(O);dt=r(Yt,"Finally, we have a couple of fully connected layers that output a Q-value for each possible action at that state."),Yt.forEach(t),Te=u(e),R=i(e,"IMG",{src:!0,alt:!0}),be=u(e),W=i(e,"P",{});var Jt=p(W);_t=r(Jt,"So, we see that Deep Q-Learning uses a neural network to approximate, given a state, the different Q-values for each possible action at that state. Now let\u2019s study the Deep Q-Learning algorithm."),Jt.forEach(t),this.h()},h(){c(d,"name","hf:doc:metadata"),c(d,"content",JSON.stringify(co)),c(N,"id","deep-q-network"),c(N,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(N,"href","#deep-q-network"),c(_,"class","relative group"),ne(P.src,Nt="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/deep-q-network.jpg")||c(P,"src",Nt),c(P,"alt","Deep Q Network"),c(E,"id","preprocessing"),c(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(E,"href","#preprocessing"),c(y,"class","relative group"),ne(G.src,Et="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/preprocessing.jpg")||c(G,"src",Et),c(G,"alt","Preprocessing"),ne(x.src,Tt="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/temporal-limitation.jpg")||c(x,"src",Tt),c(x,"alt","Temporal Limitation"),ne(D.src,bt="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/temporal-limitation-2.jpg")||c(D,"src",bt),c(D,"alt","Temporal Limitation"),c($,"href","https://www.udacity.com/course/deep-learning-pytorch--ud188"),c($,"rel","nofollow"),ne(R.src,Qt="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit4/deep-q-network.jpg")||c(R,"src",Qt),c(R,"alt","Deep Q Network")},m(e,n){o(document.head,d),l(e,le,n),l(e,_,n),o(_,N),o(N,j),Xt(Q,j,null),o(_,De),o(_,C),o(C,Le),l(e,pe,n),l(e,P,n),l(e,ce,n),l(e,f,n),o(f,$e),o(f,H),o(H,Re),o(f,Oe),o(f,U),o(U,We),o(f,Ie),l(e,he,n),l(e,m,n),o(m,Ae),o(m,z),o(z,qe),o(m,Be),o(m,F),o(F,Me),o(m,je),l(e,ue,n),l(e,y,n),o(y,E),o(E,Y),Xt(S,Y,null),o(y,Ce),o(y,J),o(J,He),l(e,fe,n),l(e,g,n),o(g,Ue),o(g,K),o(K,ze),o(g,Fe),o(g,V),o(V,Ye),o(g,Je),l(e,me,n),l(e,w,n),o(w,Ke),o(w,X),o(X,Ve),o(w,Xe),o(w,Z),o(Z,Ze),o(w,et),l(e,ge,n),l(e,T,n),o(T,tt),o(T,ee),o(ee,ot),o(T,at),l(e,we,n),l(e,G,n),l(e,ve,n),l(e,k,n),o(k,te),o(te,rt),o(k,st),o(k,oe),o(oe,it),o(k,nt),l(e,de,n),l(e,x,n),l(e,_e,n),l(e,b,n),o(b,lt),o(b,ae),o(ae,pt),o(b,ct),l(e,ye,n),l(e,D,n),l(e,ke,n),l(e,v,n),o(v,ht),o(v,re),o(re,ut),o(v,ft),o(v,se),o(se,mt),o(v,gt),l(e,Ne,n),l(e,L,n),o(L,wt),o(L,$),o($,vt),l(e,Ee,n),l(e,O,n),o(O,dt),l(e,Te,n),l(e,R,n),l(e,be,n),l(e,W,n),o(W,_t),Qe=!0},p:no,i(e){Qe||(Zt(Q.$$.fragment,e),Zt(S.$$.fragment,e),Qe=!0)},o(e){eo(Q.$$.fragment,e),eo(S.$$.fragment,e),Qe=!1},d(e){t(d),e&&t(le),e&&t(_),to(Q),e&&t(pe),e&&t(P),e&&t(ce),e&&t(f),e&&t(he),e&&t(m),e&&t(ue),e&&t(y),to(S),e&&t(fe),e&&t(g),e&&t(me),e&&t(w),e&&t(ge),e&&t(T),e&&t(we),e&&t(G),e&&t(ve),e&&t(k),e&&t(de),e&&t(x),e&&t(_e),e&&t(b),e&&t(ye),e&&t(D),e&&t(ke),e&&t(v),e&&t(Ne),e&&t(L),e&&t(Ee),e&&t(O),e&&t(Te),e&&t(R),e&&t(be),e&&t(W)}}}const co={local:"deep-q-network",sections:[{local:"preprocessing",title:"Preprocessing the input and temporal limitation "}],title:"The Deep Q-Network (DQN)  "};function ho(kt){return lo(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class mo extends ao{constructor(d){super();ro(this,d,ho,po,so,{})}}export{mo as default,co as metadata};
