import{S as Me,i as Oe,s as qe,e as i,k as f,w as Ae,t as r,M as Qe,c as o,d as t,m as p,a as l,x as je,h as u,b as c,N as ye,G as a,g as n,y as ze,L as De,q as Ue,o as We,B as Ye,v as Be}from"../../chunks/vendor-hf-doc-builder.js";import{I as Fe}from"../../chunks/IconCopyLink-hf-doc-builder.js";function He(_e){let h,q,d,v,N,y,V,R,X,A,_,we,Q,w,Z,$,ee,j,b,te,k,ae,z,E,be,D,g,se,C,ie,oe,U,P,ne,L,le,W,m,re,M,ue,ce,O,fe,pe,Y,S,me,B,I,x,Ee,F,T,he,H;return y=new Fe({}),{c(){h=i("meta"),q=f(),d=i("h1"),v=i("a"),N=i("span"),Ae(y.$$.fragment),V=f(),R=i("span"),X=r("Introduction"),A=f(),_=i("img"),Q=f(),w=i("p"),Z=r("In the last unit, we learned about Deep Q-Learning. In this value-based deep reinforcement learning algorithm, we\xA0"),$=i("strong"),ee=r("used a deep neural network to approximate the different Q-values for each possible action at a state."),j=f(),b=i("p"),te=r("Since the beginning of the course, we have only studied value-based methods,\xA0"),k=i("strong"),ae=r("where we estimate a value function as an intermediate step towards finding an optimal policy."),z=f(),E=i("img"),D=f(),g=i("p"),se=r("In value-based methods, the policy "),C=i("strong"),ie=r("(\u03C0) only exists because of the action value estimates since the policy is just a function"),oe=r(" (for instance, greedy-policy) that will select the action with the highest value given a state."),U=f(),P=i("p"),ne=r("With policy-based methods, we want to optimize the policy directly\xA0"),L=i("strong"),le=r("without having an intermediate step of learning a value function."),W=f(),m=i("p"),re=r("So today,\xA0"),M=i("strong"),ue=r("we\u2019ll learn about policy-based methods and study a subset of these methods called policy gradient"),ce=r(". Then we\u2019ll implement our first policy gradient algorithm called Monte Carlo "),O=i("strong"),fe=r("Reinforce"),pe=r(` from scratch using PyTorch.
Then, we\u2019ll test its robustness using the CartPole-v1 and PixelCopter environments.`),Y=f(),S=i("p"),me=r("You\u2019ll then be able to iterate and improve this implementation for more advanced environments."),B=f(),I=i("figure"),x=i("img"),F=f(),T=i("p"),he=r("Let\u2019s get started!"),this.h()},l(e){const s=Qe('[data-svelte="svelte-1phssyn"]',document.head);h=o(s,"META",{name:!0,content:!0}),s.forEach(t),q=p(e),d=o(e,"H1",{class:!0});var J=l(d);v=o(J,"A",{id:!0,class:!0,href:!0});var Pe=l(v);N=o(Pe,"SPAN",{});var Ie=l(N);je(y.$$.fragment,Ie),Ie.forEach(t),Pe.forEach(t),V=p(J),R=o(J,"SPAN",{});var Se=l(R);X=u(Se,"Introduction"),Se.forEach(t),J.forEach(t),A=p(e),_=o(e,"IMG",{src:!0,alt:!0}),Q=p(e),w=o(e,"P",{});var de=l(w);Z=u(de,"In the last unit, we learned about Deep Q-Learning. In this value-based deep reinforcement learning algorithm, we\xA0"),$=o(de,"STRONG",{});var xe=l($);ee=u(xe,"used a deep neural network to approximate the different Q-values for each possible action at a state."),xe.forEach(t),de.forEach(t),j=p(e),b=o(e,"P",{});var ve=l(b);te=u(ve,"Since the beginning of the course, we have only studied value-based methods,\xA0"),k=o(ve,"STRONG",{});var Te=l(k);ae=u(Te,"where we estimate a value function as an intermediate step towards finding an optimal policy."),Te.forEach(t),ve.forEach(t),z=p(e),E=o(e,"IMG",{src:!0,alt:!0}),D=p(e),g=o(e,"P",{});var K=l(g);se=u(K,"In value-based methods, the policy "),C=o(K,"STRONG",{});var Ge=l(C);ie=u(Ge,"(\u03C0) only exists because of the action value estimates since the policy is just a function"),Ge.forEach(t),oe=u(K," (for instance, greedy-policy) that will select the action with the highest value given a state."),K.forEach(t),U=p(e),P=o(e,"P",{});var ge=l(P);ne=u(ge,"With policy-based methods, we want to optimize the policy directly\xA0"),L=o(ge,"STRONG",{});var Ne=l(L);le=u(Ne,"without having an intermediate step of learning a value function."),Ne.forEach(t),ge.forEach(t),W=p(e),m=o(e,"P",{});var G=l(m);re=u(G,"So today,\xA0"),M=o(G,"STRONG",{});var Re=l(M);ue=u(Re,"we\u2019ll learn about policy-based methods and study a subset of these methods called policy gradient"),Re.forEach(t),ce=u(G,". Then we\u2019ll implement our first policy gradient algorithm called Monte Carlo "),O=o(G,"STRONG",{});var $e=l(O);fe=u($e,"Reinforce"),$e.forEach(t),pe=u(G,` from scratch using PyTorch.
Then, we\u2019ll test its robustness using the CartPole-v1 and PixelCopter environments.`),G.forEach(t),Y=p(e),S=o(e,"P",{});var ke=l(S);me=u(ke,"You\u2019ll then be able to iterate and improve this implementation for more advanced environments."),ke.forEach(t),B=p(e),I=o(e,"FIGURE",{class:!0});var Ce=l(I);x=o(Ce,"IMG",{src:!0,alt:!0}),Ce.forEach(t),F=p(e),T=o(e,"P",{});var Le=l(T);he=u(Le,"Let\u2019s get started!"),Le.forEach(t),this.h()},h(){c(h,"name","hf:doc:metadata"),c(h,"content",JSON.stringify(Je)),c(v,"id","introduction"),c(v,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(v,"href","#introduction"),c(d,"class","relative group"),ye(_.src,we="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit6/thumbnail.png")||c(_,"src",we),c(_,"alt","thumbnail"),ye(E.src,be="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit3/link-value-policy.jpg")||c(E,"src",be),c(E,"alt","Link value policy"),ye(x.src,Ee="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit6/envs.gif")||c(x,"src",Ee),c(x,"alt","Environments"),c(I,"class","image table text-center m-0 w-full")},m(e,s){a(document.head,h),n(e,q,s),n(e,d,s),a(d,v),a(v,N),ze(y,N,null),a(d,V),a(d,R),a(R,X),n(e,A,s),n(e,_,s),n(e,Q,s),n(e,w,s),a(w,Z),a(w,$),a($,ee),n(e,j,s),n(e,b,s),a(b,te),a(b,k),a(k,ae),n(e,z,s),n(e,E,s),n(e,D,s),n(e,g,s),a(g,se),a(g,C),a(C,ie),a(g,oe),n(e,U,s),n(e,P,s),a(P,ne),a(P,L),a(L,le),n(e,W,s),n(e,m,s),a(m,re),a(m,M),a(M,ue),a(m,ce),a(m,O),a(O,fe),a(m,pe),n(e,Y,s),n(e,S,s),a(S,me),n(e,B,s),n(e,I,s),a(I,x),n(e,F,s),n(e,T,s),a(T,he),H=!0},p:De,i(e){H||(Ue(y.$$.fragment,e),H=!0)},o(e){We(y.$$.fragment,e),H=!1},d(e){t(h),e&&t(q),e&&t(d),Ye(y),e&&t(A),e&&t(_),e&&t(Q),e&&t(w),e&&t(j),e&&t(b),e&&t(z),e&&t(E),e&&t(D),e&&t(g),e&&t(U),e&&t(P),e&&t(W),e&&t(m),e&&t(Y),e&&t(S),e&&t(B),e&&t(I),e&&t(F),e&&t(T)}}}const Je={local:"introduction",title:"Introduction "};function Ke(_e){return Be(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ze extends Me{constructor(h){super();Oe(this,h,Ke,He,qe,{})}}export{Ze as default,Je as metadata};
