import{S as pt,i as ht,s as mt,e as i,k as f,w as vt,t as o,M as gt,c as n,d as a,m as p,a as l,x as _t,h as s,b as u,N as fe,G as t,g as c,y as dt,L as wt,q as Et,o as Qt,B as bt,v as yt}from"../../chunks/vendor-hf-doc-builder.js";import{I as Lt}from"../../chunks/IconCopyLink-hf-doc-builder.js";function Gt(ze){let m,V,v,d,P,S,pe,q,he,X,g,x,me,ve,I,ge,_e,Y,w,M,h,de,$,we,Ee,O,Qe,be,A,ye,Le,j,Ge,Re,B,T,Se,U,Te,Z,E,Ce,ee,Q,z,k,ke,C,Ne,Pe,W,b,qe,H,xe,Ie,J,Me,te,y,We,ae,L,$e,D,Oe,Ae,ie,G,He,ne,N,je,le,R,Je,re;return S=new Lt({}),{c(){m=i("meta"),V=f(),v=i("h1"),d=i("a"),P=i("span"),vt(S.$$.fragment),pe=f(),q=i("span"),he=o("Q-Learning Recap"),X=f(),g=i("p"),x=i("em"),me=o("Q-Learning"),ve=f(),I=i("strong"),ge=o("is the RL algorithm that"),_e=o(" :"),Y=f(),w=i("ul"),M=i("li"),h=i("p"),de=o("Trains a "),$=i("em"),we=o("Q-function"),Ee=o(", an "),O=i("strong"),Qe=o("action-value function"),be=o(" encoded, in internal memory, by a "),A=i("em"),ye=o("Q-table"),Le=f(),j=i("strong"),Ge=o("containing all the state-action pair values."),Re=f(),B=i("li"),T=i("p"),Se=o("Given a state and action, our Q-function "),U=i("strong"),Te=o("will search its Q-table for the corresponding value."),Z=f(),E=i("img"),ee=f(),Q=i("ul"),z=i("li"),k=i("p"),ke=o("When the training is done, "),C=i("strong"),Ne=o("we have an optimal Q-function, or, equivalently, an optimal Q-table."),Pe=f(),W=i("li"),b=i("p"),qe=o("And if we "),H=i("strong"),xe=o("have an optimal Q-function"),Ie=o(`, we
have an optimal policy, since we `),J=i("strong"),Me=o("know, for each state, the best action to take."),te=f(),y=i("img"),ae=f(),L=i("p"),$e=o("But, in the beginning,\xA0our "),D=i("strong"),Oe=o("Q-table is useless since it gives arbitrary values for each state-action pair\xA0(most of the time we initialize the Q-table to 0 values)"),Ae=o(". But, as we\xA0explore the environment and update our Q-table it will give us a better and better approximation."),ie=f(),G=i("img"),ne=f(),N=i("p"),je=o("This is the Q-Learning pseudocode:"),le=f(),R=i("img"),this.h()},l(e){const r=gt('[data-svelte="svelte-1phssyn"]',document.head);m=n(r,"META",{name:!0,content:!0}),r.forEach(a),V=p(e),v=n(e,"H1",{class:!0});var oe=l(v);d=n(oe,"A",{id:!0,class:!0,href:!0});var De=l(d);P=n(De,"SPAN",{});var Fe=l(P);_t(S.$$.fragment,Fe),Fe.forEach(a),De.forEach(a),pe=p(oe),q=n(oe,"SPAN",{});var Ke=l(q);he=s(Ke,"Q-Learning Recap"),Ke.forEach(a),oe.forEach(a),X=p(e),g=n(e,"P",{});var F=l(g);x=n(F,"EM",{});var Ve=l(x);me=s(Ve,"Q-Learning"),Ve.forEach(a),ve=p(F),I=n(F,"STRONG",{});var Xe=l(I);ge=s(Xe,"is the RL algorithm that"),Xe.forEach(a),_e=s(F," :"),F.forEach(a),Y=p(e),w=n(e,"UL",{});var se=l(w);M=n(se,"LI",{});var Ye=l(M);h=n(Ye,"P",{});var _=l(h);de=s(_,"Trains a "),$=n(_,"EM",{});var Ze=l($);we=s(Ze,"Q-function"),Ze.forEach(a),Ee=s(_,", an "),O=n(_,"STRONG",{});var et=l(O);Qe=s(et,"action-value function"),et.forEach(a),be=s(_," encoded, in internal memory, by a "),A=n(_,"EM",{});var tt=l(A);ye=s(tt,"Q-table"),tt.forEach(a),Le=p(_),j=n(_,"STRONG",{});var at=l(j);Ge=s(at,"containing all the state-action pair values."),at.forEach(a),_.forEach(a),Ye.forEach(a),Re=p(se),B=n(se,"LI",{});var it=l(B);T=n(it,"P",{});var Be=l(T);Se=s(Be,"Given a state and action, our Q-function "),U=n(Be,"STRONG",{});var nt=l(U);Te=s(nt,"will search its Q-table for the corresponding value."),nt.forEach(a),Be.forEach(a),it.forEach(a),se.forEach(a),Z=p(e),E=n(e,"IMG",{src:!0,alt:!0,width:!0}),ee=p(e),Q=n(e,"UL",{});var ce=l(Q);z=n(ce,"LI",{});var lt=l(z);k=n(lt,"P",{});var Ue=l(k);ke=s(Ue,"When the training is done, "),C=n(Ue,"STRONG",{});var rt=l(C);Ne=s(rt,"we have an optimal Q-function, or, equivalently, an optimal Q-table."),rt.forEach(a),Ue.forEach(a),lt.forEach(a),Pe=p(ce),W=n(ce,"LI",{});var ot=l(W);b=n(ot,"P",{});var K=l(b);qe=s(K,"And if we "),H=n(K,"STRONG",{});var st=l(H);xe=s(st,"have an optimal Q-function"),st.forEach(a),Ie=s(K,`, we
have an optimal policy, since we `),J=n(K,"STRONG",{});var ct=l(J);Me=s(ct,"know, for each state, the best action to take."),ct.forEach(a),K.forEach(a),ot.forEach(a),ce.forEach(a),te=p(e),y=n(e,"IMG",{src:!0,alt:!0,width:!0}),ae=p(e),L=n(e,"P",{});var ue=l(L);$e=s(ue,"But, in the beginning,\xA0our "),D=n(ue,"STRONG",{});var ut=l(D);Oe=s(ut,"Q-table is useless since it gives arbitrary values for each state-action pair\xA0(most of the time we initialize the Q-table to 0 values)"),ut.forEach(a),Ae=s(ue,". But, as we\xA0explore the environment and update our Q-table it will give us a better and better approximation."),ue.forEach(a),ie=p(e),G=n(e,"IMG",{src:!0,alt:!0,width:!0}),ne=p(e),N=n(e,"P",{});var ft=l(N);je=s(ft,"This is the Q-Learning pseudocode:"),ft.forEach(a),le=p(e),R=n(e,"IMG",{src:!0,alt:!0,width:!0}),this.h()},h(){u(m,"name","hf:doc:metadata"),u(m,"content",JSON.stringify(Rt)),u(d,"id","q-learning-recap"),u(d,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(d,"href","#q-learning-recap"),u(v,"class","relative group"),fe(E.src,Ce="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit3/Q-function-2.jpg")||u(E,"src",Ce),u(E,"alt","Q function"),u(E,"width","100%"),fe(y.src,We="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit3/link-value-policy.jpg")||u(y,"src",We),u(y,"alt","Link value policy"),u(y,"width","100%"),fe(G.src,He="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/unit2/q-learning.jpeg")||u(G,"src",He),u(G,"alt","q-learning.jpeg"),u(G,"width","100%"),fe(R.src,Je="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit3/Q-learning-2.jpg")||u(R,"src",Je),u(R,"alt","Q-Learning"),u(R,"width","100%")},m(e,r){t(document.head,m),c(e,V,r),c(e,v,r),t(v,d),t(d,P),dt(S,P,null),t(v,pe),t(v,q),t(q,he),c(e,X,r),c(e,g,r),t(g,x),t(x,me),t(g,ve),t(g,I),t(I,ge),t(g,_e),c(e,Y,r),c(e,w,r),t(w,M),t(M,h),t(h,de),t(h,$),t($,we),t(h,Ee),t(h,O),t(O,Qe),t(h,be),t(h,A),t(A,ye),t(h,Le),t(h,j),t(j,Ge),t(w,Re),t(w,B),t(B,T),t(T,Se),t(T,U),t(U,Te),c(e,Z,r),c(e,E,r),c(e,ee,r),c(e,Q,r),t(Q,z),t(z,k),t(k,ke),t(k,C),t(C,Ne),t(Q,Pe),t(Q,W),t(W,b),t(b,qe),t(b,H),t(H,xe),t(b,Ie),t(b,J),t(J,Me),c(e,te,r),c(e,y,r),c(e,ae,r),c(e,L,r),t(L,$e),t(L,D),t(D,Oe),t(L,Ae),c(e,ie,r),c(e,G,r),c(e,ne,r),c(e,N,r),t(N,je),c(e,le,r),c(e,R,r),re=!0},p:wt,i(e){re||(Et(S.$$.fragment,e),re=!0)},o(e){Qt(S.$$.fragment,e),re=!1},d(e){a(m),e&&a(V),e&&a(v),bt(S),e&&a(X),e&&a(g),e&&a(Y),e&&a(w),e&&a(Z),e&&a(E),e&&a(ee),e&&a(Q),e&&a(te),e&&a(y),e&&a(ae),e&&a(L),e&&a(ie),e&&a(G),e&&a(ne),e&&a(N),e&&a(le),e&&a(R)}}}const Rt={local:"q-learning-recap",title:"Q-Learning Recap "};function St(ze){return yt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Nt extends pt{constructor(m){super();ht(this,m,St,Gt,mt,{})}}export{Nt as default,Rt as metadata};
