import{S as Jj,i as Uj,s as jj,e as a,k as w,w as p,t as J,M as wj,c as e,d as M,m as o,a as t,x as T,h as U,b as j,N as Q,f as nj,G as s,g as n,y as r,L as oj,q as i,o as c,B as I,v as pj}from"../../chunks/vendor-hf-doc-builder.js";import{I as m}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as Tj}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function rj(Hn){let A,Ua,Z,V,DM,ol,at,$M,et,ja,pl,wa,Tl,tt,PM,yt,oa,TM,nt,pa,rM,Jt,Ta,F,B,Ut,rl,jt,wt,il,ot,pt,Tt,iM,rt,cl,it,ra,cM,ct,ia,IM,OM,Il,It,ca,dl,d,dM,zn,dt,Ia,hM,ht,da,hl,mt,ls,Ct,ha,h,qn,mM,CM,ut,ma,uM,bt,Ca,bM,At,ua,AM,ml,ZM,xn,ba,G,N,Ms,Cl,Zt,ss,Bt,Aa,ul,Ln,Za,H,Gt,as,ft,Wt,Ba,BM,vt,Ga,GM,es,bl,Xt,fa,u,gt,ts,_t,kt,Al,Et,Rt,Wa,f,z,ys,Zl,Yt,ns,St,va,fM,Qt,Xa,q,Bl,Vt,Js,Ft,Nt,Ht,Gl,zt,Us,qt,xt,ga,W,x,js,fl,Lt,ws,Kt,_a,WM,Dt,ka,L,$t,Wl,Pt,Ot,Ea,b,ly,vl,My,sy,os,ay,ey,Ra,Xl,ty,ps,yy,Ya,gl,ny,_l,Jy,Sa,v,K,Ts,kl,Uy,rs,jy,Qa,vM,D,wy,is,oy,py,cs,Ty,Va,El,Kn,Fa,XM,Is,ds,ry,Na,Rl,Dn,Ha,X,$,hs,Yl,iy,ms,cy,za,P,Iy,Cs,dy,hy,qa,gM,my,xa,Sl,La,Ql,Ka,g,O,us,Vl,Cy,bs,uy,Da,Fl,$a,_,ll,As,Nl,by,Zs,Ay,Pa,Hl,Zy,zl,By,Oa,ql,le,k,Ml,Bs,xl,Gy,Gs,fy,Me,_M,fs,Wy,se,Ll,ae,kM,Kl,vy,Ws,Xy,gy,ee,Dl,te,sl,vs,Xs,_y,ky,$l,gs,Ey,Ry,E,EM,_s,Yy,Sy,Qy,RM,ks,Vy,Fy,Ny,YM,Es,Hy,zy,ye,Pl,ne,SM,Rs,qy,Je,Ol,Ue,QM,Ys,xy,je,lM,we,VM,Ly,oe,MM,Ky,sM,Dy,pe,FM,$y,Te,NM,al,Py,aM,Oy,ln,Ss,Mn,re,eM,$n,ie,el,Qs,sn,an,Vs,en,ce,tM,Ie,yM,tn,Fs,yn,de,R,tl,Ns,nM,nn,Hs,Jn,he,JM,Un,zs,jn,me,yl,qs,xs,wn,on,Ls,HM,pn,Ks,Tn,Ce,UM,Pn,ue,jM,On,be,nl,Ds,Jl,rn,$s,cn,In,Ps,dn,hn,Os,la,mn,Ae,wM,Ze,Y,Ul,Ma,oM,Cn,sa,un,Be,jl,bn,aa,An,Zn,Ge,zM,Bn,fe,S,wl,ea,pM,Gn,ta,fn,We;return ol=new m({}),pl=new Tj({props:{classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/deep-rl-class/blob/main/notebooks/unit8/unit8_part1.ipynb"}],askForHelpUrl:"http://hf.co/join/discord"}}),Cl=new m({}),Zl=new m({}),fl=new m({}),kl=new m({}),Yl=new m({}),Sl=new C({props:{code:"YXB0JTIwaW5zdGFsbCUyMHB5dGhvbi1vcGVuZ2wlMEFhcHQlMjBpbnN0YWxsJTIwZmZtcGVnJTBBYXB0JTIwaW5zdGFsbCUyMHh2ZmIlMEFwaXAlMjBpbnN0YWxsJTIwcHlnbGV0JTNEJTNEMS41JTBBcGlwJTIwaW5zdGFsbCUyMHB5dmlydHVhbGRpc3BsYXk=",highlighted:`apt install python-opengl
apt install ffmpeg
apt install xvfb
pip install pyglet==<span class="hljs-number">1.5</span>
pip install pyvirtualdisplay`}}),Ql=new C({props:{code:"JTIzJTIwVmlydHVhbCUyMGRpc3BsYXklMEFmcm9tJTIwcHl2aXJ0dWFsZGlzcGxheSUyMGltcG9ydCUyMERpc3BsYXklMEElMEF2aXJ0dWFsX2Rpc3BsYXklMjAlM0QlMjBEaXNwbGF5KHZpc2libGUlM0QwJTJDJTIwc2l6ZSUzRCgxNDAwJTJDJTIwOTAwKSklMEF2aXJ0dWFsX2Rpc3BsYXkuc3RhcnQoKQ==",highlighted:`<span class="hljs-comment"># Virtual display</span>
<span class="hljs-keyword">from</span> pyvirtualdisplay <span class="hljs-keyword">import</span> Display

virtual_display = Display(visible=<span class="hljs-number">0</span>, size=(<span class="hljs-number">1400</span>, <span class="hljs-number">900</span>))
virtual_display.start()`}}),Vl=new m({}),Fl=new C({props:{code:"cGlwJTIwaW5zdGFsbCUyMGd5bSUzRCUzRDAuMjIlMEFwaXAlMjBpbnN0YWxsJTIwaW1hZ2Vpby1mZm1wZWclMEFwaXAlMjBpbnN0YWxsJTIwaHVnZ2luZ2ZhY2VfaHViJTBBcGlwJTIwaW5zdGFsbCUyMGd5bSU1QmJveDJkJTVEJTNEJTNEMC4yMg==",highlighted:`pip install gym==<span class="hljs-number">0.22</span>
pip install imageio-ffmpeg
pip install huggingface_hub
pip install gym[box2d]==<span class="hljs-number">0.22</span>`}}),Nl=new m({}),ql=new C({props:{code:"ZnJvbSUyMElQeXRob24uZGlzcGxheSUyMGltcG9ydCUyMEhUTUwlMEElMEFIVE1MKCUwQSUyMCUyMCUyMCUyMCclM0NpZnJhbWUlMjB3aWR0aCUzRCUyMjU2MCUyMiUyMGhlaWdodCUzRCUyMjMxNSUyMiUyMHNyYyUzRCUyMmh0dHBzJTNBJTJGJTJGd3d3LnlvdXR1YmUuY29tJTJGZW1iZWQlMkZNRXQ2cnJ4SDhXNCUyMiUyMHRpdGxlJTNEJTIyWW91VHViZSUyMHZpZGVvJTIwcGxheWVyJTIyJTIwZnJhbWVib3JkZXIlM0QlMjIwJTIyJTIwYWxsb3clM0QlMjJhY2NlbGVyb21ldGVyJTNCJTIwYXV0b3BsYXklM0IlMjBjbGlwYm9hcmQtd3JpdGUlM0IlMjBlbmNyeXB0ZWQtbWVkaWElM0IlMjBneXJvc2NvcGUlM0IlMjBwaWN0dXJlLWluLXBpY3R1cmUlMjIlMjBhbGxvd2Z1bGxzY3JlZW4lM0UlM0MlMkZpZnJhbWUlM0UnJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> IPython.display <span class="hljs-keyword">import</span> HTML

HTML(
    <span class="hljs-string">&#x27;&lt;iframe width=&quot;560&quot; height=&quot;315&quot; src=&quot;https://www.youtube.com/embed/MEt6rrxH8W4&quot; title=&quot;YouTube video player&quot; frameborder=&quot;0&quot; allow=&quot;accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture&quot; allowfullscreen&gt;&lt;/iframe&gt;&#x27;</span>
)`}}),xl=new m({}),Ll=new C({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmQXBpJTJDJTIwdXBsb2FkX2ZvbGRlciUwQWZyb20lMjBodWdnaW5nZmFjZV9odWIucmVwb2NhcmQlMjBpbXBvcnQlMjBtZXRhZGF0YV9ldmFsX3Jlc3VsdCUyQyUyMG1ldGFkYXRhX3NhdmUlMEElMEFmcm9tJTIwcGF0aGxpYiUyMGltcG9ydCUyMFBhdGglMEFpbXBvcnQlMjBkYXRldGltZSUwQWltcG9ydCUyMHRlbXBmaWxlJTBBaW1wb3J0JTIwanNvbiUwQWltcG9ydCUyMHNodXRpbCUwQWltcG9ydCUyMGltYWdlaW8lMEElMEFmcm9tJTIwd2FzYWJpJTIwaW1wb3J0JTIwUHJpbnRlciUwQSUwQW1zZyUyMCUzRCUyMFByaW50ZXIoKQ==",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi, upload_folder
<span class="hljs-keyword">from</span> huggingface_hub.repocard <span class="hljs-keyword">import</span> metadata_eval_result, metadata_save

<span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
<span class="hljs-keyword">import</span> datetime
<span class="hljs-keyword">import</span> tempfile
<span class="hljs-keyword">import</span> json
<span class="hljs-keyword">import</span> shutil
<span class="hljs-keyword">import</span> imageio

<span class="hljs-keyword">from</span> wasabi <span class="hljs-keyword">import</span> Printer

msg = Printer()`}}),Dl=new C({props:{code:"JTIzJTIwQWRkaW5nJTIwSHVnZ2luZ0ZhY2UlMjBhcmd1bWVudCUwQXBhcnNlci5hZGRfYXJndW1lbnQoJTBBJTIwJTIwJTIwJTIwJTIyLS1yZXBvLWlkJTIyJTJDJTBBJTIwJTIwJTIwJTIwdHlwZSUzRHN0ciUyQyUwQSUyMCUyMCUyMCUyMGRlZmF1bHQlM0QlMjJUaG9tYXNTaW1vbmluaSUyRnBwby1DYXJ0UG9sZS12MSUyMiUyQyUwQSUyMCUyMCUyMCUyMGhlbHAlM0QlMjJpZCUyMG9mJTIwdGhlJTIwbW9kZWwlMjByZXBvc2l0b3J5JTIwZnJvbSUyMHRoZSUyMEh1Z2dpbmclMjBGYWNlJTIwSHViJTIwJTdCdXNlcm5hbWUlMkZyZXBvX25hbWUlN0QlMjIlMkMlMEEp",highlighted:`<span class="hljs-comment"># Adding HuggingFace argument</span>
parser.add_argument(
    <span class="hljs-string">&quot;--repo-id&quot;</span>,
    <span class="hljs-built_in">type</span>=<span class="hljs-built_in">str</span>,
    default=<span class="hljs-string">&quot;ThomasSimonini/ppo-CartPole-v1&quot;</span>,
    <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;id of the model repository from the Hugging Face Hub {username/repo_name}&quot;</span>,
)`}}),Pl=new C({props:{code:" ",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">package_to_hub</span>(<span class="hljs-params">
    repo_id,
    model,
    hyperparameters,
    eval_env,
    video_fps=<span class="hljs-number">30</span>,
    commit_message=<span class="hljs-string">&quot;Push agent to the Hub&quot;</span>,
    token=<span class="hljs-literal">None</span>,
    logs=<span class="hljs-literal">None</span>,
</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Evaluate, Generate a video and Upload a model to Hugging Face Hub.
    This method does the complete pipeline:
    - It evaluates the model
    - It generates the model card
    - It generates a replay video of the agent
    - It pushes everything to the hub
    :param repo_id: id of the model repository from the Hugging Face Hub
    :param model: trained model
    :param eval_env: environment used to evaluate the agent
    :param fps: number of fps for rendering the video
    :param commit_message: commit message
    :param logs: directory on local machine of tensorboard logs you&#x27;d like to upload
    &quot;&quot;&quot;</span>
    msg.info(
        <span class="hljs-string">&quot;This function will save, evaluate, generate a video of your agent, &quot;</span>
        <span class="hljs-string">&quot;create a model card and push everything to the hub. &quot;</span>
        <span class="hljs-string">&quot;It might take up to 1min. \\n &quot;</span>
        <span class="hljs-string">&quot;This is a work in progress: if you encounter a bug, please open an issue.&quot;</span>
    )
    <span class="hljs-comment"># Step 1: Clone or create the repo</span>
    repo_url = HfApi().create_repo(
        repo_id=repo_id,
        token=token,
        private=<span class="hljs-literal">False</span>,
        exist_ok=<span class="hljs-literal">True</span>,
    )

    <span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmpdirname:
        tmpdirname = Path(tmpdirname)

        <span class="hljs-comment"># Step 2: Save the model</span>
        torch.save(model.state_dict(), tmpdirname / <span class="hljs-string">&quot;model.pt&quot;</span>)

        <span class="hljs-comment"># Step 3: Evaluate the model and build JSON</span>
        mean_reward, std_reward = _evaluate_agent(eval_env, <span class="hljs-number">10</span>, model)

        <span class="hljs-comment"># First get datetime</span>
        eval_datetime = datetime.datetime.now()
        eval_form_datetime = eval_datetime.isoformat()

        evaluate_data = {
            <span class="hljs-string">&quot;env_id&quot;</span>: hyperparameters.env_id,
            <span class="hljs-string">&quot;mean_reward&quot;</span>: mean_reward,
            <span class="hljs-string">&quot;std_reward&quot;</span>: std_reward,
            <span class="hljs-string">&quot;n_evaluation_episodes&quot;</span>: <span class="hljs-number">10</span>,
            <span class="hljs-string">&quot;eval_datetime&quot;</span>: eval_form_datetime,
        }

        <span class="hljs-comment"># Write a JSON file</span>
        <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(tmpdirname / <span class="hljs-string">&quot;results.json&quot;</span>, <span class="hljs-string">&quot;w&quot;</span>) <span class="hljs-keyword">as</span> outfile:
            json.dump(evaluate_data, outfile)

        <span class="hljs-comment"># Step 4: Generate a video</span>
        video_path = tmpdirname / <span class="hljs-string">&quot;replay.mp4&quot;</span>
        record_video(eval_env, model, video_path, video_fps)

        <span class="hljs-comment"># Step 5: Generate the model card</span>
        generated_model_card, metadata = _generate_model_card(
            <span class="hljs-string">&quot;PPO&quot;</span>, hyperparameters.env_id, mean_reward, std_reward, hyperparameters
        )
        _save_model_card(tmpdirname, generated_model_card, metadata)

        <span class="hljs-comment"># Step 6: Add logs if needed</span>
        <span class="hljs-keyword">if</span> logs:
            _add_logdir(tmpdirname, Path(logs))

        msg.info(<span class="hljs-string">f&quot;Pushing repo <span class="hljs-subst">{repo_id}</span> to the Hugging Face Hub&quot;</span>)

        repo_url = upload_folder(
            repo_id=repo_id,
            folder_path=tmpdirname,
            path_in_repo=<span class="hljs-string">&quot;&quot;</span>,
            commit_message=commit_message,
            token=token,
        )

        msg.info(<span class="hljs-string">f&quot;Your model is pushed to the Hub. You can view your model here: <span class="hljs-subst">{repo_url}</span>&quot;</span>)
    <span class="hljs-keyword">return</span> repo_url


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_evaluate_agent</span>(<span class="hljs-params">env, n_eval_episodes, policy</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Evaluate the agent for \`\`n_eval_episodes\`\` episodes and returns average reward and std of reward.
    :param env: The evaluation environment
    :param n_eval_episodes: Number of episode to evaluate the agent
    :param policy: The agent
    &quot;&quot;&quot;</span>
    episode_rewards = []
    <span class="hljs-keyword">for</span> episode <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_eval_episodes):
        state = env.reset()
        step = <span class="hljs-number">0</span>
        done = <span class="hljs-literal">False</span>
        total_rewards_ep = <span class="hljs-number">0</span>

        <span class="hljs-keyword">while</span> done <span class="hljs-keyword">is</span> <span class="hljs-literal">False</span>:
            state = torch.Tensor(state).to(device)
            action, _, _, _ = policy.get_action_and_value(state)
            new_state, reward, done, info = env.step(action.cpu().numpy())
            total_rewards_ep += reward
            <span class="hljs-keyword">if</span> done:
                <span class="hljs-keyword">break</span>
            state = new_state
        episode_rewards.append(total_rewards_ep)
    mean_reward = np.mean(episode_rewards)
    std_reward = np.std(episode_rewards)

    <span class="hljs-keyword">return</span> mean_reward, std_reward


<span class="hljs-keyword">def</span> <span class="hljs-title function_">record_video</span>(<span class="hljs-params">env, policy, out_directory, fps=<span class="hljs-number">30</span></span>):
    images = []
    done = <span class="hljs-literal">False</span>
    state = env.reset()
    img = env.render(mode=<span class="hljs-string">&quot;rgb_array&quot;</span>)
    images.append(img)
    <span class="hljs-keyword">while</span> <span class="hljs-keyword">not</span> done:
        state = torch.Tensor(state).to(device)
        <span class="hljs-comment"># Take the action (index) that have the maximum expected future reward given that state</span>
        action, _, _, _ = policy.get_action_and_value(state)
        state, reward, done, info = env.step(
            action.cpu().numpy()
        )  <span class="hljs-comment"># We directly put next_state = state for recording logic</span>
        img = env.render(mode=<span class="hljs-string">&quot;rgb_array&quot;</span>)
        images.append(img)
    imageio.mimsave(out_directory, [np.array(img) <span class="hljs-keyword">for</span> i, img <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(images)], fps=fps)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_generate_model_card</span>(<span class="hljs-params">model_name, env_id, mean_reward, std_reward, hyperparameters</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Generate the model card for the Hub
    :param model_name: name of the model
    :env_id: name of the environment
    :mean_reward: mean reward of the agent
    :std_reward: standard deviation of the mean reward of the agent
    :hyperparameters: training arguments
    &quot;&quot;&quot;</span>
    <span class="hljs-comment"># Step 1: Select the tags</span>
    metadata = generate_metadata(model_name, env_id, mean_reward, std_reward)

    <span class="hljs-comment"># Transform the hyperparams namespace to string</span>
    converted_dict = <span class="hljs-built_in">vars</span>(hyperparameters)
    converted_str = <span class="hljs-built_in">str</span>(converted_dict)
    converted_str = converted_str.split(<span class="hljs-string">&quot;, &quot;</span>)
    converted_str = <span class="hljs-string">&quot;\\n&quot;</span>.join(converted_str)

    <span class="hljs-comment"># Step 2: Generate the model card</span>
    model_card = <span class="hljs-string">f&quot;&quot;&quot;
  # PPO Agent Playing <span class="hljs-subst">{env_id}</span>

  This is a trained model of a PPO agent playing <span class="hljs-subst">{env_id}</span>.

  # Hyperparameters
  &quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> model_card, metadata


<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate_metadata</span>(<span class="hljs-params">model_name, env_id, mean_reward, std_reward</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Define the tags for the model card
    :param model_name: name of the model
    :param env_id: name of the environment
    :mean_reward: mean reward of the agent
    :std_reward: standard deviation of the mean reward of the agent
    &quot;&quot;&quot;</span>
    metadata = {}
    metadata[<span class="hljs-string">&quot;tags&quot;</span>] = [
        env_id,
        <span class="hljs-string">&quot;ppo&quot;</span>,
        <span class="hljs-string">&quot;deep-reinforcement-learning&quot;</span>,
        <span class="hljs-string">&quot;reinforcement-learning&quot;</span>,
        <span class="hljs-string">&quot;custom-implementation&quot;</span>,
        <span class="hljs-string">&quot;deep-rl-course&quot;</span>,
    ]

    <span class="hljs-comment"># Add metrics</span>
    <span class="hljs-built_in">eval</span> = metadata_eval_result(
        model_pretty_name=model_name,
        task_pretty_name=<span class="hljs-string">&quot;reinforcement-learning&quot;</span>,
        task_id=<span class="hljs-string">&quot;reinforcement-learning&quot;</span>,
        metrics_pretty_name=<span class="hljs-string">&quot;mean_reward&quot;</span>,
        metrics_id=<span class="hljs-string">&quot;mean_reward&quot;</span>,
        metrics_value=<span class="hljs-string">f&quot;<span class="hljs-subst">{mean_reward:<span class="hljs-number">.2</span>f}</span> +/- <span class="hljs-subst">{std_reward:<span class="hljs-number">.2</span>f}</span>&quot;</span>,
        dataset_pretty_name=env_id,
        dataset_id=env_id,
    )

    <span class="hljs-comment"># Merges both dictionaries</span>
    metadata = {**metadata, **<span class="hljs-built_in">eval</span>}

    <span class="hljs-keyword">return</span> metadata


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_save_model_card</span>(<span class="hljs-params">local_path, generated_model_card, metadata</span>):
    <span class="hljs-string">&quot;&quot;&quot;Saves a model card for the repository.
    :param local_path: repository directory
    :param generated_model_card: model card generated by _generate_model_card()
    :param metadata: metadata
    &quot;&quot;&quot;</span>
    readme_path = local_path / <span class="hljs-string">&quot;README.md&quot;</span>
    readme = <span class="hljs-string">&quot;&quot;</span>
    <span class="hljs-keyword">if</span> readme_path.exists():
        <span class="hljs-keyword">with</span> readme_path.<span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;r&quot;</span>, encoding=<span class="hljs-string">&quot;utf8&quot;</span>) <span class="hljs-keyword">as</span> f:
            readme = f.read()
    <span class="hljs-keyword">else</span>:
        readme = generated_model_card

    <span class="hljs-keyword">with</span> readme_path.<span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;w&quot;</span>, encoding=<span class="hljs-string">&quot;utf-8&quot;</span>) <span class="hljs-keyword">as</span> f:
        f.write(readme)

    <span class="hljs-comment"># Save our metrics to Readme metadata</span>
    metadata_save(readme_path, metadata)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_add_logdir</span>(<span class="hljs-params">local_path: Path, logdir: Path</span>):
    <span class="hljs-string">&quot;&quot;&quot;Adds a logdir to the repository.
    :param local_path: repository directory
    :param logdir: logdir directory
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">if</span> logdir.exists() <span class="hljs-keyword">and</span> logdir.is_dir():
        <span class="hljs-comment"># Add the logdir to the repository under new dir called logs</span>
        repo_logdir = local_path / <span class="hljs-string">&quot;logs&quot;</span>

        <span class="hljs-comment"># Delete current logs if they exist</span>
        <span class="hljs-keyword">if</span> repo_logdir.exists():
            shutil.rmtree(repo_logdir)

        <span class="hljs-comment"># Copy logdir into repo logdir</span>
        shutil.copytree(logdir, repo_logdir)`}}),Ol=new C({props:{code:"JTIzJTIwQ3JlYXRlJTIwdGhlJTIwZXZhbHVhdGlvbiUyMGVudmlyb25tZW50JTBBZXZhbF9lbnYlMjAlM0QlMjBneW0ubWFrZShhcmdzLmVudl9pZCklMEElMEFwYWNrYWdlX3RvX2h1YiglMEElMjAlMjAlMjAlMjByZXBvX2lkJTNEYXJncy5yZXBvX2lkJTJDJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0RhZ2VudCUyQyUyMCUyMCUyMyUyMFRoZSUyMG1vZGVsJTIwd2UlMjB3YW50JTIwdG8lMjBzYXZlJTBBJTIwJTIwJTIwJTIwaHlwZXJwYXJhbWV0ZXJzJTNEYXJncyUyQyUwQSUyMCUyMCUyMCUyMGV2YWxfZW52JTNEZ3ltLm1ha2UoYXJncy5lbnZfaWQpJTJDJTBBJTIwJTIwJTIwJTIwbG9ncyUzRGYlMjJydW5zJTJGJTdCcnVuX25hbWUlN0QlMjIlMkMlMEEp",highlighted:`<span class="hljs-comment"># Create the evaluation environment</span>
eval_env = gym.make(args.env_id)

package_to_hub(
    repo_id=args.repo_id,
    model=agent,  <span class="hljs-comment"># The model we want to save</span>
    hyperparameters=args,
    eval_env=gym.make(args.env_id),
    logs=<span class="hljs-string">f&quot;runs/<span class="hljs-subst">{run_name}</span>&quot;</span>,
)`}}),lM=new C({props:{code:" ",highlighted:`<span class="hljs-comment"># docs and experiment results can be found at https://docs.cleanrl.dev/rl-algorithms/ppo/#ppopy</span>

<span class="hljs-keyword">import</span> argparse
<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> random
<span class="hljs-keyword">import</span> time
<span class="hljs-keyword">from</span> distutils.util <span class="hljs-keyword">import</span> strtobool

<span class="hljs-keyword">import</span> gym
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torch.nn <span class="hljs-keyword">as</span> nn
<span class="hljs-keyword">import</span> torch.optim <span class="hljs-keyword">as</span> optim
<span class="hljs-keyword">from</span> torch.distributions.categorical <span class="hljs-keyword">import</span> Categorical
<span class="hljs-keyword">from</span> torch.utils.tensorboard <span class="hljs-keyword">import</span> SummaryWriter

<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi, upload_folder
<span class="hljs-keyword">from</span> huggingface_hub.repocard <span class="hljs-keyword">import</span> metadata_eval_result, metadata_save

<span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
<span class="hljs-keyword">import</span> datetime
<span class="hljs-keyword">import</span> tempfile
<span class="hljs-keyword">import</span> json
<span class="hljs-keyword">import</span> shutil
<span class="hljs-keyword">import</span> imageio

<span class="hljs-keyword">from</span> wasabi <span class="hljs-keyword">import</span> Printer

msg = Printer()


<span class="hljs-keyword">def</span> <span class="hljs-title function_">parse_args</span>():
    <span class="hljs-comment"># fmt: off</span>
    parser = argparse.ArgumentParser()
    parser.add_argument(<span class="hljs-string">&quot;--exp-name&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">str</span>, default=os.path.basename(__file__).rstrip(<span class="hljs-string">&quot;.py&quot;</span>),
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the name of this experiment&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--seed&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">int</span>, default=<span class="hljs-number">1</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;seed of the experiment&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--torch-deterministic&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">True</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;if toggled, \`torch.backends.cudnn.deterministic=False\`&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--cuda&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">True</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;if toggled, cuda will be enabled by default&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--track&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">False</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;if toggled, this experiment will be tracked with Weights and Biases&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--wandb-project-name&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">str</span>, default=<span class="hljs-string">&quot;cleanRL&quot;</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the wandb&#x27;s project name&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--wandb-entity&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">str</span>, default=<span class="hljs-literal">None</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the entity (team) of wandb&#x27;s project&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--capture-video&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">False</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;weather to capture videos of the agent performances (check out \`videos\` folder)&quot;</span>)

    <span class="hljs-comment"># Algorithm specific arguments</span>
    parser.add_argument(<span class="hljs-string">&quot;--env-id&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">str</span>, default=<span class="hljs-string">&quot;CartPole-v1&quot;</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the id of the environment&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--total-timesteps&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">int</span>, default=<span class="hljs-number">50000</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;total timesteps of the experiments&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--learning-rate&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">2.5e-4</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the learning rate of the optimizer&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--num-envs&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">int</span>, default=<span class="hljs-number">4</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the number of parallel game environments&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--num-steps&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">int</span>, default=<span class="hljs-number">128</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the number of steps to run in each environment per policy rollout&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--anneal-lr&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">True</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;Toggle learning rate annealing for policy and value networks&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--gae&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">True</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;Use GAE for advantage computation&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--gamma&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">0.99</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the discount factor gamma&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--gae-lambda&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">0.95</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the lambda for the general advantage estimation&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--num-minibatches&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">int</span>, default=<span class="hljs-number">4</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the number of mini-batches&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--update-epochs&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">int</span>, default=<span class="hljs-number">4</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the K epochs to update the policy&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--norm-adv&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">True</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;Toggles advantages normalization&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--clip-coef&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">0.2</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the surrogate clipping coefficient&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--clip-vloss&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">bool</span>(strtobool(x)), default=<span class="hljs-literal">True</span>, nargs=<span class="hljs-string">&quot;?&quot;</span>, const=<span class="hljs-literal">True</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;Toggles whether or not to use a clipped loss for the value function, as per the paper.&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--ent-coef&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">0.01</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;coefficient of the entropy&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--vf-coef&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">0.5</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;coefficient of the value function&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--max-grad-norm&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-number">0.5</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the maximum norm for the gradient clipping&quot;</span>)
    parser.add_argument(<span class="hljs-string">&quot;--target-kl&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">float</span>, default=<span class="hljs-literal">None</span>,
        <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;the target KL divergence threshold&quot;</span>)

    <span class="hljs-comment"># Adding HuggingFace argument</span>
    parser.add_argument(<span class="hljs-string">&quot;--repo-id&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-built_in">str</span>, default=<span class="hljs-string">&quot;ThomasSimonini/ppo-CartPole-v1&quot;</span>, <span class="hljs-built_in">help</span>=<span class="hljs-string">&quot;id of the model repository from the Hugging Face Hub {username/repo_name}&quot;</span>)

    args = parser.parse_args()
    args.batch_size = <span class="hljs-built_in">int</span>(args.num_envs * args.num_steps)
    args.minibatch_size = <span class="hljs-built_in">int</span>(args.batch_size // args.num_minibatches)
    <span class="hljs-comment"># fmt: on</span>
    <span class="hljs-keyword">return</span> args


<span class="hljs-keyword">def</span> <span class="hljs-title function_">package_to_hub</span>(<span class="hljs-params">
    repo_id,
    model,
    hyperparameters,
    eval_env,
    video_fps=<span class="hljs-number">30</span>,
    commit_message=<span class="hljs-string">&quot;Push agent to the Hub&quot;</span>,
    token=<span class="hljs-literal">None</span>,
    logs=<span class="hljs-literal">None</span>,
</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Evaluate, Generate a video and Upload a model to Hugging Face Hub.
    This method does the complete pipeline:
    - It evaluates the model
    - It generates the model card
    - It generates a replay video of the agent
    - It pushes everything to the hub
    :param repo_id: id of the model repository from the Hugging Face Hub
    :param model: trained model
    :param eval_env: environment used to evaluate the agent
    :param fps: number of fps for rendering the video
    :param commit_message: commit message
    :param logs: directory on local machine of tensorboard logs you&#x27;d like to upload
    &quot;&quot;&quot;</span>
    msg.info(
        <span class="hljs-string">&quot;This function will save, evaluate, generate a video of your agent, &quot;</span>
        <span class="hljs-string">&quot;create a model card and push everything to the hub. &quot;</span>
        <span class="hljs-string">&quot;It might take up to 1min. \\n &quot;</span>
        <span class="hljs-string">&quot;This is a work in progress: if you encounter a bug, please open an issue.&quot;</span>
    )
    <span class="hljs-comment"># Step 1: Clone or create the repo</span>
    repo_url = HfApi().create_repo(
        repo_id=repo_id,
        token=token,
        private=<span class="hljs-literal">False</span>,
        exist_ok=<span class="hljs-literal">True</span>,
    )

    <span class="hljs-keyword">with</span> tempfile.TemporaryDirectory() <span class="hljs-keyword">as</span> tmpdirname:
        tmpdirname = Path(tmpdirname)

        <span class="hljs-comment"># Step 2: Save the model</span>
        torch.save(model.state_dict(), tmpdirname / <span class="hljs-string">&quot;model.pt&quot;</span>)

        <span class="hljs-comment"># Step 3: Evaluate the model and build JSON</span>
        mean_reward, std_reward = _evaluate_agent(eval_env, <span class="hljs-number">10</span>, model)

        <span class="hljs-comment"># First get datetime</span>
        eval_datetime = datetime.datetime.now()
        eval_form_datetime = eval_datetime.isoformat()

        evaluate_data = {
            <span class="hljs-string">&quot;env_id&quot;</span>: hyperparameters.env_id,
            <span class="hljs-string">&quot;mean_reward&quot;</span>: mean_reward,
            <span class="hljs-string">&quot;std_reward&quot;</span>: std_reward,
            <span class="hljs-string">&quot;n_evaluation_episodes&quot;</span>: <span class="hljs-number">10</span>,
            <span class="hljs-string">&quot;eval_datetime&quot;</span>: eval_form_datetime,
        }

        <span class="hljs-comment"># Write a JSON file</span>
        <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(tmpdirname / <span class="hljs-string">&quot;results.json&quot;</span>, <span class="hljs-string">&quot;w&quot;</span>) <span class="hljs-keyword">as</span> outfile:
            json.dump(evaluate_data, outfile)

        <span class="hljs-comment"># Step 4: Generate a video</span>
        video_path = tmpdirname / <span class="hljs-string">&quot;replay.mp4&quot;</span>
        record_video(eval_env, model, video_path, video_fps)

        <span class="hljs-comment"># Step 5: Generate the model card</span>
        generated_model_card, metadata = _generate_model_card(
            <span class="hljs-string">&quot;PPO&quot;</span>, hyperparameters.env_id, mean_reward, std_reward, hyperparameters
        )
        _save_model_card(tmpdirname, generated_model_card, metadata)

        <span class="hljs-comment"># Step 6: Add logs if needed</span>
        <span class="hljs-keyword">if</span> logs:
            _add_logdir(tmpdirname, Path(logs))

        msg.info(<span class="hljs-string">f&quot;Pushing repo <span class="hljs-subst">{repo_id}</span> to the Hugging Face Hub&quot;</span>)

        repo_url = upload_folder(
            repo_id=repo_id,
            folder_path=tmpdirname,
            path_in_repo=<span class="hljs-string">&quot;&quot;</span>,
            commit_message=commit_message,
            token=token,
        )

        msg.info(<span class="hljs-string">f&quot;Your model is pushed to the Hub. You can view your model here: <span class="hljs-subst">{repo_url}</span>&quot;</span>)
    <span class="hljs-keyword">return</span> repo_url


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_evaluate_agent</span>(<span class="hljs-params">env, n_eval_episodes, policy</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Evaluate the agent for \`\`n_eval_episodes\`\` episodes and returns average reward and std of reward.
    :param env: The evaluation environment
    :param n_eval_episodes: Number of episode to evaluate the agent
    :param policy: The agent
    &quot;&quot;&quot;</span>
    episode_rewards = []
    <span class="hljs-keyword">for</span> episode <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_eval_episodes):
        state = env.reset()
        step = <span class="hljs-number">0</span>
        done = <span class="hljs-literal">False</span>
        total_rewards_ep = <span class="hljs-number">0</span>

        <span class="hljs-keyword">while</span> done <span class="hljs-keyword">is</span> <span class="hljs-literal">False</span>:
            state = torch.Tensor(state).to(device)
            action, _, _, _ = policy.get_action_and_value(state)
            new_state, reward, done, info = env.step(action.cpu().numpy())
            total_rewards_ep += reward
            <span class="hljs-keyword">if</span> done:
                <span class="hljs-keyword">break</span>
            state = new_state
        episode_rewards.append(total_rewards_ep)
    mean_reward = np.mean(episode_rewards)
    std_reward = np.std(episode_rewards)

    <span class="hljs-keyword">return</span> mean_reward, std_reward


<span class="hljs-keyword">def</span> <span class="hljs-title function_">record_video</span>(<span class="hljs-params">env, policy, out_directory, fps=<span class="hljs-number">30</span></span>):
    images = []
    done = <span class="hljs-literal">False</span>
    state = env.reset()
    img = env.render(mode=<span class="hljs-string">&quot;rgb_array&quot;</span>)
    images.append(img)
    <span class="hljs-keyword">while</span> <span class="hljs-keyword">not</span> done:
        state = torch.Tensor(state).to(device)
        <span class="hljs-comment"># Take the action (index) that have the maximum expected future reward given that state</span>
        action, _, _, _ = policy.get_action_and_value(state)
        state, reward, done, info = env.step(
            action.cpu().numpy()
        )  <span class="hljs-comment"># We directly put next_state = state for recording logic</span>
        img = env.render(mode=<span class="hljs-string">&quot;rgb_array&quot;</span>)
        images.append(img)
    imageio.mimsave(out_directory, [np.array(img) <span class="hljs-keyword">for</span> i, img <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(images)], fps=fps)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_generate_model_card</span>(<span class="hljs-params">model_name, env_id, mean_reward, std_reward, hyperparameters</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Generate the model card for the Hub
    :param model_name: name of the model
    :env_id: name of the environment
    :mean_reward: mean reward of the agent
    :std_reward: standard deviation of the mean reward of the agent
    :hyperparameters: training arguments
    &quot;&quot;&quot;</span>
    <span class="hljs-comment"># Step 1: Select the tags</span>
    metadata = generate_metadata(model_name, env_id, mean_reward, std_reward)

    <span class="hljs-comment"># Transform the hyperparams namespace to string</span>
    converted_dict = <span class="hljs-built_in">vars</span>(hyperparameters)
    converted_str = <span class="hljs-built_in">str</span>(converted_dict)
    converted_str = converted_str.split(<span class="hljs-string">&quot;, &quot;</span>)
    converted_str = <span class="hljs-string">&quot;\\n&quot;</span>.join(converted_str)

    <span class="hljs-comment"># Step 2: Generate the model card</span>
    model_card = <span class="hljs-string">f&quot;&quot;&quot;
  # PPO Agent Playing <span class="hljs-subst">{env_id}</span>

  This is a trained model of a PPO agent playing <span class="hljs-subst">{env_id}</span>.

  # Hyperparameters
  &quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> model_card, metadata


<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate_metadata</span>(<span class="hljs-params">model_name, env_id, mean_reward, std_reward</span>):
    <span class="hljs-string">&quot;&quot;&quot;
    Define the tags for the model card
    :param model_name: name of the model
    :param env_id: name of the environment
    :mean_reward: mean reward of the agent
    :std_reward: standard deviation of the mean reward of the agent
    &quot;&quot;&quot;</span>
    metadata = {}
    metadata[<span class="hljs-string">&quot;tags&quot;</span>] = [
        env_id,
        <span class="hljs-string">&quot;ppo&quot;</span>,
        <span class="hljs-string">&quot;deep-reinforcement-learning&quot;</span>,
        <span class="hljs-string">&quot;reinforcement-learning&quot;</span>,
        <span class="hljs-string">&quot;custom-implementation&quot;</span>,
        <span class="hljs-string">&quot;deep-rl-course&quot;</span>,
    ]

    <span class="hljs-comment"># Add metrics</span>
    <span class="hljs-built_in">eval</span> = metadata_eval_result(
        model_pretty_name=model_name,
        task_pretty_name=<span class="hljs-string">&quot;reinforcement-learning&quot;</span>,
        task_id=<span class="hljs-string">&quot;reinforcement-learning&quot;</span>,
        metrics_pretty_name=<span class="hljs-string">&quot;mean_reward&quot;</span>,
        metrics_id=<span class="hljs-string">&quot;mean_reward&quot;</span>,
        metrics_value=<span class="hljs-string">f&quot;<span class="hljs-subst">{mean_reward:<span class="hljs-number">.2</span>f}</span> +/- <span class="hljs-subst">{std_reward:<span class="hljs-number">.2</span>f}</span>&quot;</span>,
        dataset_pretty_name=env_id,
        dataset_id=env_id,
    )

    <span class="hljs-comment"># Merges both dictionaries</span>
    metadata = {**metadata, **<span class="hljs-built_in">eval</span>}

    <span class="hljs-keyword">return</span> metadata


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_save_model_card</span>(<span class="hljs-params">local_path, generated_model_card, metadata</span>):
    <span class="hljs-string">&quot;&quot;&quot;Saves a model card for the repository.
    :param local_path: repository directory
    :param generated_model_card: model card generated by _generate_model_card()
    :param metadata: metadata
    &quot;&quot;&quot;</span>
    readme_path = local_path / <span class="hljs-string">&quot;README.md&quot;</span>
    readme = <span class="hljs-string">&quot;&quot;</span>
    <span class="hljs-keyword">if</span> readme_path.exists():
        <span class="hljs-keyword">with</span> readme_path.<span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;r&quot;</span>, encoding=<span class="hljs-string">&quot;utf8&quot;</span>) <span class="hljs-keyword">as</span> f:
            readme = f.read()
    <span class="hljs-keyword">else</span>:
        readme = generated_model_card

    <span class="hljs-keyword">with</span> readme_path.<span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;w&quot;</span>, encoding=<span class="hljs-string">&quot;utf-8&quot;</span>) <span class="hljs-keyword">as</span> f:
        f.write(readme)

    <span class="hljs-comment"># Save our metrics to Readme metadata</span>
    metadata_save(readme_path, metadata)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">_add_logdir</span>(<span class="hljs-params">local_path: Path, logdir: Path</span>):
    <span class="hljs-string">&quot;&quot;&quot;Adds a logdir to the repository.
    :param local_path: repository directory
    :param logdir: logdir directory
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">if</span> logdir.exists() <span class="hljs-keyword">and</span> logdir.is_dir():
        <span class="hljs-comment"># Add the logdir to the repository under new dir called logs</span>
        repo_logdir = local_path / <span class="hljs-string">&quot;logs&quot;</span>

        <span class="hljs-comment"># Delete current logs if they exist</span>
        <span class="hljs-keyword">if</span> repo_logdir.exists():
            shutil.rmtree(repo_logdir)

        <span class="hljs-comment"># Copy logdir into repo logdir</span>
        shutil.copytree(logdir, repo_logdir)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">make_env</span>(<span class="hljs-params">env_id, seed, idx, capture_video, run_name</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">thunk</span>():
        env = gym.make(env_id)
        env = gym.wrappers.RecordEpisodeStatistics(env)
        <span class="hljs-keyword">if</span> capture_video:
            <span class="hljs-keyword">if</span> idx == <span class="hljs-number">0</span>:
                env = gym.wrappers.RecordVideo(env, <span class="hljs-string">f&quot;videos/<span class="hljs-subst">{run_name}</span>&quot;</span>)
        env.seed(seed)
        env.action_space.seed(seed)
        env.observation_space.seed(seed)
        <span class="hljs-keyword">return</span> env

    <span class="hljs-keyword">return</span> thunk


<span class="hljs-keyword">def</span> <span class="hljs-title function_">layer_init</span>(<span class="hljs-params">layer, std=np.sqrt(<span class="hljs-params"><span class="hljs-number">2</span></span>), bias_const=<span class="hljs-number">0.0</span></span>):
    torch.nn.init.orthogonal_(layer.weight, std)
    torch.nn.init.constant_(layer.bias, bias_const)
    <span class="hljs-keyword">return</span> layer


<span class="hljs-keyword">class</span> <span class="hljs-title class_">Agent</span>(nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, envs</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.critic = nn.Sequential(
            layer_init(nn.Linear(np.array(envs.single_observation_space.shape).prod(), <span class="hljs-number">64</span>)),
            nn.Tanh(),
            layer_init(nn.Linear(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>)),
            nn.Tanh(),
            layer_init(nn.Linear(<span class="hljs-number">64</span>, <span class="hljs-number">1</span>), std=<span class="hljs-number">1.0</span>),
        )
        self.actor = nn.Sequential(
            layer_init(nn.Linear(np.array(envs.single_observation_space.shape).prod(), <span class="hljs-number">64</span>)),
            nn.Tanh(),
            layer_init(nn.Linear(<span class="hljs-number">64</span>, <span class="hljs-number">64</span>)),
            nn.Tanh(),
            layer_init(nn.Linear(<span class="hljs-number">64</span>, envs.single_action_space.n), std=<span class="hljs-number">0.01</span>),
        )

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_value</span>(<span class="hljs-params">self, x</span>):
        <span class="hljs-keyword">return</span> self.critic(x)

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_action_and_value</span>(<span class="hljs-params">self, x, action=<span class="hljs-literal">None</span></span>):
        logits = self.actor(x)
        probs = Categorical(logits=logits)
        <span class="hljs-keyword">if</span> action <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
            action = probs.sample()
        <span class="hljs-keyword">return</span> action, probs.log_prob(action), probs.entropy(), self.critic(x)


<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&quot;__main__&quot;</span>:
    args = parse_args()
    run_name = <span class="hljs-string">f&quot;<span class="hljs-subst">{args.env_id}</span>__<span class="hljs-subst">{args.exp_name}</span>__<span class="hljs-subst">{args.seed}</span>__<span class="hljs-subst">{<span class="hljs-built_in">int</span>(time.time())}</span>&quot;</span>
    <span class="hljs-keyword">if</span> args.track:
        <span class="hljs-keyword">import</span> wandb

        wandb.init(
            project=args.wandb_project_name,
            entity=args.wandb_entity,
            sync_tensorboard=<span class="hljs-literal">True</span>,
            config=<span class="hljs-built_in">vars</span>(args),
            name=run_name,
            monitor_gym=<span class="hljs-literal">True</span>,
            save_code=<span class="hljs-literal">True</span>,
        )
    writer = SummaryWriter(<span class="hljs-string">f&quot;runs/<span class="hljs-subst">{run_name}</span>&quot;</span>)
    writer.add_text(
        <span class="hljs-string">&quot;hyperparameters&quot;</span>,
        <span class="hljs-string">&quot;|param|value|\\n|-|-|\\n%s&quot;</span> % (<span class="hljs-string">&quot;\\n&quot;</span>.join([<span class="hljs-string">f&quot;|<span class="hljs-subst">{key}</span>|<span class="hljs-subst">{value}</span>|&quot;</span> <span class="hljs-keyword">for</span> key, value <span class="hljs-keyword">in</span> <span class="hljs-built_in">vars</span>(args).items()])),
    )

    <span class="hljs-comment"># TRY NOT TO MODIFY: seeding</span>
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.backends.cudnn.deterministic = args.torch_deterministic

    device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">and</span> args.cuda <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>)

    <span class="hljs-comment"># env setup</span>
    envs = gym.vector.SyncVectorEnv(
        [make_env(args.env_id, args.seed + i, i, args.capture_video, run_name) <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(args.num_envs)]
    )
    <span class="hljs-keyword">assert</span> <span class="hljs-built_in">isinstance</span>(envs.single_action_space, gym.spaces.Discrete), <span class="hljs-string">&quot;only discrete action space is supported&quot;</span>

    agent = Agent(envs).to(device)
    optimizer = optim.Adam(agent.parameters(), lr=args.learning_rate, eps=<span class="hljs-number">1e-5</span>)

    <span class="hljs-comment"># ALGO Logic: Storage setup</span>
    obs = torch.zeros((args.num_steps, args.num_envs) + envs.single_observation_space.shape).to(device)
    actions = torch.zeros((args.num_steps, args.num_envs) + envs.single_action_space.shape).to(device)
    logprobs = torch.zeros((args.num_steps, args.num_envs)).to(device)
    rewards = torch.zeros((args.num_steps, args.num_envs)).to(device)
    dones = torch.zeros((args.num_steps, args.num_envs)).to(device)
    values = torch.zeros((args.num_steps, args.num_envs)).to(device)

    <span class="hljs-comment"># TRY NOT TO MODIFY: start the game</span>
    global_step = <span class="hljs-number">0</span>
    start_time = time.time()
    next_obs = torch.Tensor(envs.reset()).to(device)
    next_done = torch.zeros(args.num_envs).to(device)
    num_updates = args.total_timesteps // args.batch_size

    <span class="hljs-keyword">for</span> update <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>, num_updates + <span class="hljs-number">1</span>):
        <span class="hljs-comment"># Annealing the rate if instructed to do so.</span>
        <span class="hljs-keyword">if</span> args.anneal_lr:
            frac = <span class="hljs-number">1.0</span> - (update - <span class="hljs-number">1.0</span>) / num_updates
            lrnow = frac * args.learning_rate
            optimizer.param_groups[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;lr&quot;</span>] = lrnow

        <span class="hljs-keyword">for</span> step <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, args.num_steps):
            global_step += <span class="hljs-number">1</span> * args.num_envs
            obs[step] = next_obs
            dones[step] = next_done

            <span class="hljs-comment"># ALGO LOGIC: action logic</span>
            <span class="hljs-keyword">with</span> torch.no_grad():
                action, logprob, _, value = agent.get_action_and_value(next_obs)
                values[step] = value.flatten()
            actions[step] = action
            logprobs[step] = logprob

            <span class="hljs-comment"># TRY NOT TO MODIFY: execute the game and log data.</span>
            next_obs, reward, done, info = envs.step(action.cpu().numpy())
            rewards[step] = torch.tensor(reward).to(device).view(-<span class="hljs-number">1</span>)
            next_obs, next_done = torch.Tensor(next_obs).to(device), torch.Tensor(done).to(device)

            <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> info:
                <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;episode&quot;</span> <span class="hljs-keyword">in</span> item.keys():
                    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;global_step=<span class="hljs-subst">{global_step}</span>, episodic_return=<span class="hljs-subst">{item[<span class="hljs-string">&#x27;episode&#x27;</span>][<span class="hljs-string">&#x27;r&#x27;</span>]}</span>&quot;</span>)
                    writer.add_scalar(<span class="hljs-string">&quot;charts/episodic_return&quot;</span>, item[<span class="hljs-string">&quot;episode&quot;</span>][<span class="hljs-string">&quot;r&quot;</span>], global_step)
                    writer.add_scalar(<span class="hljs-string">&quot;charts/episodic_length&quot;</span>, item[<span class="hljs-string">&quot;episode&quot;</span>][<span class="hljs-string">&quot;l&quot;</span>], global_step)
                    <span class="hljs-keyword">break</span>

        <span class="hljs-comment"># bootstrap value if not done</span>
        <span class="hljs-keyword">with</span> torch.no_grad():
            next_value = agent.get_value(next_obs).reshape(<span class="hljs-number">1</span>, -<span class="hljs-number">1</span>)
            <span class="hljs-keyword">if</span> args.gae:
                advantages = torch.zeros_like(rewards).to(device)
                lastgaelam = <span class="hljs-number">0</span>
                <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> <span class="hljs-built_in">reversed</span>(<span class="hljs-built_in">range</span>(args.num_steps)):
                    <span class="hljs-keyword">if</span> t == args.num_steps - <span class="hljs-number">1</span>:
                        nextnonterminal = <span class="hljs-number">1.0</span> - next_done
                        nextvalues = next_value
                    <span class="hljs-keyword">else</span>:
                        nextnonterminal = <span class="hljs-number">1.0</span> - dones[t + <span class="hljs-number">1</span>]
                        nextvalues = values[t + <span class="hljs-number">1</span>]
                    delta = rewards[t] + args.gamma * nextvalues * nextnonterminal - values[t]
                    advantages[t] = lastgaelam = delta + args.gamma * args.gae_lambda * nextnonterminal * lastgaelam
                returns = advantages + values
            <span class="hljs-keyword">else</span>:
                returns = torch.zeros_like(rewards).to(device)
                <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> <span class="hljs-built_in">reversed</span>(<span class="hljs-built_in">range</span>(args.num_steps)):
                    <span class="hljs-keyword">if</span> t == args.num_steps - <span class="hljs-number">1</span>:
                        nextnonterminal = <span class="hljs-number">1.0</span> - next_done
                        next_return = next_value
                    <span class="hljs-keyword">else</span>:
                        nextnonterminal = <span class="hljs-number">1.0</span> - dones[t + <span class="hljs-number">1</span>]
                        next_return = returns[t + <span class="hljs-number">1</span>]
                    returns[t] = rewards[t] + args.gamma * nextnonterminal * next_return
                advantages = returns - values

        <span class="hljs-comment"># flatten the batch</span>
        b_obs = obs.reshape((-<span class="hljs-number">1</span>,) + envs.single_observation_space.shape)
        b_logprobs = logprobs.reshape(-<span class="hljs-number">1</span>)
        b_actions = actions.reshape((-<span class="hljs-number">1</span>,) + envs.single_action_space.shape)
        b_advantages = advantages.reshape(-<span class="hljs-number">1</span>)
        b_returns = returns.reshape(-<span class="hljs-number">1</span>)
        b_values = values.reshape(-<span class="hljs-number">1</span>)

        <span class="hljs-comment"># Optimizing the policy and value network</span>
        b_inds = np.arange(args.batch_size)
        clipfracs = []
        <span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(args.update_epochs):
            np.random.shuffle(b_inds)
            <span class="hljs-keyword">for</span> start <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, args.batch_size, args.minibatch_size):
                end = start + args.minibatch_size
                mb_inds = b_inds[start:end]

                _, newlogprob, entropy, newvalue = agent.get_action_and_value(
                    b_obs[mb_inds], b_actions.long()[mb_inds]
                )
                logratio = newlogprob - b_logprobs[mb_inds]
                ratio = logratio.exp()

                <span class="hljs-keyword">with</span> torch.no_grad():
                    <span class="hljs-comment"># calculate approx_kl http://joschu.net/blog/kl-approx.html</span>
                    old_approx_kl = (-logratio).mean()
                    approx_kl = ((ratio - <span class="hljs-number">1</span>) - logratio).mean()
                    clipfracs += [((ratio - <span class="hljs-number">1.0</span>).<span class="hljs-built_in">abs</span>() &gt; args.clip_coef).<span class="hljs-built_in">float</span>().mean().item()]

                mb_advantages = b_advantages[mb_inds]
                <span class="hljs-keyword">if</span> args.norm_adv:
                    mb_advantages = (mb_advantages - mb_advantages.mean()) / (mb_advantages.std() + <span class="hljs-number">1e-8</span>)

                <span class="hljs-comment"># Policy loss</span>
                pg_loss1 = -mb_advantages * ratio
                pg_loss2 = -mb_advantages * torch.clamp(ratio, <span class="hljs-number">1</span> - args.clip_coef, <span class="hljs-number">1</span> + args.clip_coef)
                pg_loss = torch.<span class="hljs-built_in">max</span>(pg_loss1, pg_loss2).mean()

                <span class="hljs-comment"># Value loss</span>
                newvalue = newvalue.view(-<span class="hljs-number">1</span>)
                <span class="hljs-keyword">if</span> args.clip_vloss:
                    v_loss_unclipped = (newvalue - b_returns[mb_inds]) ** <span class="hljs-number">2</span>
                    v_clipped = b_values[mb_inds] + torch.clamp(
                        newvalue - b_values[mb_inds],
                        -args.clip_coef,
                        args.clip_coef,
                    )
                    v_loss_clipped = (v_clipped - b_returns[mb_inds]) ** <span class="hljs-number">2</span>
                    v_loss_max = torch.<span class="hljs-built_in">max</span>(v_loss_unclipped, v_loss_clipped)
                    v_loss = <span class="hljs-number">0.5</span> * v_loss_max.mean()
                <span class="hljs-keyword">else</span>:
                    v_loss = <span class="hljs-number">0.5</span> * ((newvalue - b_returns[mb_inds]) ** <span class="hljs-number">2</span>).mean()

                entropy_loss = entropy.mean()
                loss = pg_loss - args.ent_coef * entropy_loss + v_loss * args.vf_coef

                optimizer.zero_grad()
                loss.backward()
                nn.utils.clip_grad_norm_(agent.parameters(), args.max_grad_norm)
                optimizer.step()

            <span class="hljs-keyword">if</span> args.target_kl <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                <span class="hljs-keyword">if</span> approx_kl &gt; args.target_kl:
                    <span class="hljs-keyword">break</span>

        y_pred, y_true = b_values.cpu().numpy(), b_returns.cpu().numpy()
        var_y = np.var(y_true)
        explained_var = np.nan <span class="hljs-keyword">if</span> var_y == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-number">1</span> - np.var(y_true - y_pred) / var_y

        <span class="hljs-comment"># TRY NOT TO MODIFY: record rewards for plotting purposes</span>
        writer.add_scalar(<span class="hljs-string">&quot;charts/learning_rate&quot;</span>, optimizer.param_groups[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;lr&quot;</span>], global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/value_loss&quot;</span>, v_loss.item(), global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/policy_loss&quot;</span>, pg_loss.item(), global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/entropy&quot;</span>, entropy_loss.item(), global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/old_approx_kl&quot;</span>, old_approx_kl.item(), global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/approx_kl&quot;</span>, approx_kl.item(), global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/clipfrac&quot;</span>, np.mean(clipfracs), global_step)
        writer.add_scalar(<span class="hljs-string">&quot;losses/explained_variance&quot;</span>, explained_var, global_step)
        <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;SPS:&quot;</span>, <span class="hljs-built_in">int</span>(global_step / (time.time() - start_time)))
        writer.add_scalar(<span class="hljs-string">&quot;charts/SPS&quot;</span>, <span class="hljs-built_in">int</span>(global_step / (time.time() - start_time)), global_step)

    envs.close()
    writer.close()

    <span class="hljs-comment"># Create the evaluation environment</span>
    eval_env = gym.make(args.env_id)

    package_to_hub(
        repo_id=args.repo_id,
        model=agent,  <span class="hljs-comment"># The model we want to save</span>
        hyperparameters=args,
        eval_env=gym.make(args.env_id),
        logs=<span class="hljs-string">f&quot;runs/<span class="hljs-subst">{run_name}</span>&quot;</span>,
    )`}}),tM=new C({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBbm90ZWJvb2tfbG9naW4oKSUwQSFnaXQlMjBjb25maWclMjAtLWdsb2JhbCUyMGNyZWRlbnRpYWwuaGVscGVyJTIwc3RvcmU=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login
notebook_login()
!git config --<span class="hljs-keyword">global</span> credential.helper store`}}),nM=new m({}),wM=new C({props:{code:"IXB5dGhvbiUyMHBwby5weSUyMC0tZW52LWlkJTNEJTIyTHVuYXJMYW5kZXItdjIlMjIlMjAtLXJlcG8taWQlM0QlMjJZT1VSX1JFUE9fSUQlMjIlMjAtLXRvdGFsLXRpbWVzdGVwcyUzRDUwMDAw",highlighted:'!python ppo.py --env-<span class="hljs-built_in">id</span>=<span class="hljs-string">&quot;LunarLander-v2&quot;</span> --repo-<span class="hljs-built_in">id</span>=<span class="hljs-string">&quot;YOUR_REPO_ID&quot;</span> --total-timesteps=<span class="hljs-number">50000</span>'}}),oM=new m({}),pM=new m({}),{c(){A=a("meta"),Ua=w(),Z=a("h1"),V=a("a"),DM=a("span"),p(ol.$$.fragment),at=w(),$M=a("span"),et=J("Hands-on"),ja=w(),p(pl.$$.fragment),wa=w(),Tl=a("p"),tt=J("Now that we studied the theory behind PPO, the best way to understand how it works\xA0"),PM=a("strong"),yt=J("is to implement it from scratch."),oa=w(),TM=a("p"),nt=J("Implementing an architecture from scratch is the best way to understand it, and it\u2019s a good habit. We have already done it for a value-based method with Q-Learning and a Policy-based method with Reinforce."),pa=w(),rM=a("p"),Jt=J("So, to be able to code it, we\u2019re going to use two resources:"),Ta=w(),F=a("ul"),B=a("li"),Ut=J("A tutorial made by "),rl=a("a"),jt=J("Costa Huang"),wt=J(". Costa is behind "),il=a("a"),ot=J("CleanRL"),pt=J(", a Deep Reinforcement Learning library that provides high-quality single-file implementation with research-friendly features."),Tt=w(),iM=a("li"),rt=J("In addition to the tutorial, to go deeper, you can read the 13 core implementation details:\xA0"),cl=a("a"),it=J("https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/"),ra=w(),cM=a("p"),ct=J("Then, to test its robustness, we\u2019re going to train it in:"),ia=w(),IM=a("ul"),OM=a("li"),Il=a("a"),It=J("LunarLander-v2"),ca=w(),dl=a("figure"),d=a("video"),dM=a("source"),dt=J(`
  &lt;/video>`),Ia=w(),hM=a("p"),ht=J("And finally, we will push the trained model to the Hub to evaluate and visualize your agent playing."),da=w(),hl=a("p"),mt=J("LunarLander-v2 is the first environment you used when you started this course. At that time, you didn\u2019t know how it worked, and now you can code it from scratch and train it.\xA0"),ls=a("strong"),Ct=J("How incredible is that \u{1F929}."),ha=w(),h=a("iframe"),mM=a("p"),CM=a("a"),ut=J("via GIPHY"),ma=w(),uM=a("p"),bt=J("Let\u2019s get started! \u{1F680}"),Ca=w(),bM=a("p"),At=J("The colab notebook:"),ua=w(),AM=a("p"),ml=a("a"),ZM=a("img"),ba=w(),G=a("h1"),N=a("a"),Ms=a("span"),p(Cl.$$.fragment),Zt=w(),ss=a("span"),Bt=J("Unit 8: Proximal Policy Gradient (PPO) with PyTorch \u{1F916}"),Aa=w(),ul=a("img"),Za=w(),H=a("p"),Gt=J("In this notebook, you\u2019ll learn to "),as=a("strong"),ft=J("code your PPO agent from scratch with PyTorch using CleanRL implementation as model"),Wt=J("."),Ba=w(),BM=a("p"),vt=J("To test its robustness, we\u2019re going to train it in:"),Ga=w(),GM=a("ul"),es=a("li"),bl=a("a"),Xt=J("LunarLander-v2 \u{1F680}"),fa=w(),u=a("p"),gt=J("We\u2019re constantly trying to improve our tutorials, so "),ts=a("strong"),_t=J("if you find some issues in this notebook"),kt=J(", please "),Al=a("a"),Et=J("open an issue on the GitHub Repo"),Rt=J("."),Wa=w(),f=a("h2"),z=a("a"),ys=a("span"),p(Zl.$$.fragment),Yt=w(),ns=a("span"),St=J("Objectives of this notebook \u{1F3C6}"),va=w(),fM=a("p"),Qt=J("At the end of the notebook, you will:"),Xa=w(),q=a("ul"),Bl=a("li"),Vt=J("Be able to "),Js=a("strong"),Ft=J("code your PPO agent from scratch using PyTorch"),Nt=J("."),Ht=w(),Gl=a("li"),zt=J("Be able to "),Us=a("strong"),qt=J("push your trained agent and the code to the Hub"),xt=J(" with a nice video replay and an evaluation score \u{1F525}."),ga=w(),W=a("h2"),x=a("a"),js=a("span"),p(fl.$$.fragment),Lt=w(),ws=a("span"),Kt=J("Prerequisites \u{1F3D7}\uFE0F"),_a=w(),WM=a("p"),Dt=J("Before diving into the notebook, you need to:"),ka=w(),L=a("p"),$t=J("\u{1F532} \u{1F4DA} Study "),Wl=a("a"),Pt=J("PPO by reading Unit 8"),Ot=J(" \u{1F917}"),Ea=w(),b=a("p"),ly=J("To validate this hands-on for the "),vl=a("a"),My=J("certification process"),sy=J(", you need to push one model, we don\u2019t ask for a minimal result but we "),os=a("strong"),ay=J("advise you to try different hyperparameters settings to get better results"),ey=J("."),Ra=w(),Xl=a("p"),ty=J("If you don\u2019t find your model, "),ps=a("strong"),yy=J("go to the bottom of the page and click on the refresh button"),Ya=w(),gl=a("p"),ny=J("For more information about the certification process, check this section \u{1F449} "),_l=a("a"),Jy=J("https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),Sa=w(),v=a("h2"),K=a("a"),Ts=a("span"),p(kl.$$.fragment),Uy=w(),rs=a("span"),jy=J("Set the GPU \u{1F4AA}"),Qa=w(),vM=a("ul"),D=a("li"),wy=J("To "),is=a("strong"),oy=J("accelerate the agent\u2019s training, we\u2019ll use a GPU"),py=J(". To do that, go to "),cs=a("code"),Ty=J("Runtime > Change Runtime type"),Va=w(),El=a("img"),Fa=w(),XM=a("ul"),Is=a("li"),ds=a("code"),ry=J("Hardware Accelerator > GPU"),Na=w(),Rl=a("img"),Ha=w(),X=a("h2"),$=a("a"),hs=a("span"),p(Yl.$$.fragment),iy=w(),ms=a("span"),cy=J("Create a virtual display \u{1F53D}"),za=w(),P=a("p"),Iy=J("During the notebook, we\u2019ll need to generate a replay video. To do so, with colab, "),Cs=a("strong"),dy=J("we need to have a virtual screen to be able to render the environment"),hy=J(" (and thus record the frames)."),qa=w(),gM=a("p"),my=J("Hence the following cell will install the librairies and create and run a virtual screen \u{1F5A5}"),xa=w(),p(Sl.$$.fragment),La=w(),p(Ql.$$.fragment),Ka=w(),g=a("h2"),O=a("a"),us=a("span"),p(Vl.$$.fragment),Cy=w(),bs=a("span"),uy=J("Install dependencies \u{1F53D}"),Da=J(`

For this exercise, we use \`gym==0.21\` because the video was recorded with Gym.

	`),p(Fl.$$.fragment),$a=w(),_=a("h2"),ll=a("a"),As=a("span"),p(Nl.$$.fragment),by=w(),Zs=a("span"),Ay=J("Let's code PPO from scratch with Costa Huang's tutorial"),Pa=J(`

- For the core implementation of PPO we're going to use the excellent [Costa Huang](https://costa.sh/) tutorial.
- In addition to the tutorial, to go deeper you can read the 37 core implementation details: https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/
`),Hl=a("p"),Zy=J("\u{1F449} The video tutorial: "),zl=a("a"),By=J("https://youtu.be/MEt6rrxH8W4"),Oa=w(),p(ql.$$.fragment),le=w(),k=a("h2"),Ml=a("a"),Bs=a("span"),p(xl.$$.fragment),Gy=w(),Gs=a("span"),fy=J("Add the Hugging Face Integration \u{1F917}"),Me=J(`

- In order to push our model to the Hub, we need to define a function \`package_to_hub\`
`),_M=a("ul"),fs=a("li"),Wy=J("Add dependencies we need to push our model to the Hub"),se=w(),p(Ll.$$.fragment),ae=w(),kM=a("ul"),Kl=a("li"),vy=J("Add new argument in "),Ws=a("code"),Xy=J("parse_args()"),gy=J(" function to define the repo-id where we want to push the model."),ee=w(),p(Dl.$$.fragment),te=w(),sl=a("ul"),vs=a("li"),Xs=a("p"),_y=J("Next, we add the methods needed to push the model to the Hub"),ky=w(),$l=a("li"),gs=a("p"),Ey=J("These methods will:"),Ry=w(),E=a("ul"),EM=a("li"),_s=a("code"),Yy=J("_evalutate_agent()"),Sy=J(": evaluate the agent."),Qy=w(),RM=a("li"),ks=a("code"),Vy=J("_generate_model_card()"),Fy=J(": generate the model card of your agent."),Ny=w(),YM=a("li"),Es=a("code"),Hy=J("_record_video()"),zy=J(": record a video of your agent."),ye=w(),p(Pl.$$.fragment),ne=w(),SM=a("ul"),Rs=a("li"),qy=J("Finally, we call this function at the end of the PPO training"),Je=w(),p(Ol.$$.fragment),Ue=w(),QM=a("ul"),Ys=a("li"),xy=J("Here\u2019s what the final ppo.py file looks like:"),je=w(),p(lM.$$.fragment),we=w(),VM=a("p"),Ly=J("To be able to share your model with the community there are three more steps to follow:"),oe=w(),MM=a("p"),Ky=J("1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),sM=a("a"),Dy=J("https://huggingface.co/join"),pe=w(),FM=a("p"),$y=J("2\uFE0F\u20E3 Sign in and get your authentication token from the Hugging Face website."),Te=w(),NM=a("ul"),al=a("li"),Py=J("Create a new token ("),aM=a("a"),Oy=J("https://huggingface.co/settings/tokens"),ln=J(") "),Ss=a("strong"),Mn=J("with write role"),re=w(),eM=a("img"),ie=w(),el=a("ul"),Qs=a("li"),sn=J("Copy the token"),an=w(),Vs=a("li"),en=J("Run the cell below and paste the token"),ce=w(),p(tM.$$.fragment),Ie=w(),yM=a("p"),tn=J("If you don\u2019t want to use Google Colab or a Jupyter Notebook, you need to use this command instead: "),Fs=a("code"),yn=J("huggingface-cli login"),de=w(),R=a("h2"),tl=a("a"),Ns=a("span"),p(nM.$$.fragment),nn=w(),Hs=a("span"),Jn=J("Let's start the training \u{1F525}"),he=w(),JM=a("p"),Un=J("\u26A0\uFE0F \u26A0\uFE0F \u26A0\uFE0F  Don\u2019t use "),zs=a("strong"),jn=J("the same repo id with the one you used for the Unit 1"),me=w(),yl=a("ul"),qs=a("li"),xs=a("p"),wn=J("Now that you\u2019ve coded PPO from scratch and added the Hugging Face Integration, we\u2019re ready to start the training \u{1F525}"),on=w(),Ls=a("li"),HM=a("p"),pn=J("First, you need to copy all your code to a file you create called "),Ks=a("code"),Tn=J("ppo.py"),Ce=w(),UM=a("img"),ue=w(),jM=a("img"),be=w(),nl=a("ul"),Ds=a("li"),Jl=a("p"),rn=J("Now we just need to run this python script using "),$s=a("code"),cn=J("python <name-of-python-script>.py"),In=J(" with the additional parameters we defined using "),Ps=a("code"),dn=J("argparse"),hn=w(),Os=a("li"),la=a("p"),mn=J("You should modify more hyperparameters otherwise the training will not be super stable."),Ae=w(),p(wM.$$.fragment),Ze=w(),Y=a("h2"),Ul=a("a"),Ma=a("span"),p(oM.$$.fragment),Cn=w(),sa=a("span"),un=J("Some additional challenges \u{1F3C6}"),Be=w(),jl=a("p"),bn=J("The best way to learn "),aa=a("strong"),An=J("is to try things on your own"),Zn=J("! Why not try another environment? Or why not trying to modify the implementation to work with Gymnasium?"),Ge=w(),zM=a("p"),Bn=J("See you in Unit 8, part 2 where we\u2019re going to train agents to play Doom \u{1F525}"),fe=w(),S=a("h2"),wl=a("a"),ea=a("span"),p(pM.$$.fragment),Gn=w(),ta=a("span"),fn=J("Keep learning, stay awesome \u{1F917}"),this.h()},l(l){const y=wj('[data-svelte="svelte-1phssyn"]',document.head);A=e(y,"META",{name:!0,content:!0}),y.forEach(M),Ua=o(l),Z=e(l,"H1",{class:!0});var ve=t(Z);V=e(ve,"A",{id:!0,class:!0,href:!0});var lJ=t(V);DM=e(lJ,"SPAN",{});var MJ=t(DM);T(ol.$$.fragment,MJ),MJ.forEach(M),lJ.forEach(M),at=o(ve),$M=e(ve,"SPAN",{});var sJ=t($M);et=U(sJ,"Hands-on"),sJ.forEach(M),ve.forEach(M),ja=o(l),T(pl.$$.fragment,l),wa=o(l),Tl=e(l,"P",{});var Wn=t(Tl);tt=U(Wn,"Now that we studied the theory behind PPO, the best way to understand how it works\xA0"),PM=e(Wn,"STRONG",{});var aJ=t(PM);yt=U(aJ,"is to implement it from scratch."),aJ.forEach(M),Wn.forEach(M),oa=o(l),TM=e(l,"P",{});var eJ=t(TM);nt=U(eJ,"Implementing an architecture from scratch is the best way to understand it, and it\u2019s a good habit. We have already done it for a value-based method with Q-Learning and a Policy-based method with Reinforce."),eJ.forEach(M),pa=o(l),rM=e(l,"P",{});var tJ=t(rM);Jt=U(tJ,"So, to be able to code it, we\u2019re going to use two resources:"),tJ.forEach(M),Ta=o(l),F=e(l,"UL",{});var Xe=t(F);B=e(Xe,"LI",{});var qM=t(B);Ut=U(qM,"A tutorial made by "),rl=e(qM,"A",{href:!0,rel:!0});var yJ=t(rl);jt=U(yJ,"Costa Huang"),yJ.forEach(M),wt=U(qM,". Costa is behind "),il=e(qM,"A",{href:!0,rel:!0});var nJ=t(il);ot=U(nJ,"CleanRL"),nJ.forEach(M),pt=U(qM,", a Deep Reinforcement Learning library that provides high-quality single-file implementation with research-friendly features."),qM.forEach(M),Tt=o(Xe),iM=e(Xe,"LI",{});var vn=t(iM);rt=U(vn,"In addition to the tutorial, to go deeper, you can read the 13 core implementation details:\xA0"),cl=e(vn,"A",{href:!0,rel:!0});var JJ=t(cl);it=U(JJ,"https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/"),JJ.forEach(M),vn.forEach(M),Xe.forEach(M),ra=o(l),cM=e(l,"P",{});var UJ=t(cM);ct=U(UJ,"Then, to test its robustness, we\u2019re going to train it in:"),UJ.forEach(M),ia=o(l),IM=e(l,"UL",{});var jJ=t(IM);OM=e(jJ,"LI",{});var wJ=t(OM);Il=e(wJ,"A",{href:!0,rel:!0});var oJ=t(Il);It=U(oJ,"LunarLander-v2"),oJ.forEach(M),wJ.forEach(M),jJ.forEach(M),ca=o(l),dl=e(l,"FIGURE",{class:!0});var pJ=t(dl);d=e(pJ,"VIDEO",{alt:!0,style:!0,autobuffer:!0});var Xn=t(d);dM=e(Xn,"SOURCE",{src:!0,type:!0}),dt=U(Xn,`
  &lt;/video>`),Xn.forEach(M),pJ.forEach(M),Ia=o(l),hM=e(l,"P",{});var TJ=t(hM);ht=U(TJ,"And finally, we will push the trained model to the Hub to evaluate and visualize your agent playing."),TJ.forEach(M),da=o(l),hl=e(l,"P",{});var gn=t(hl);mt=U(gn,"LunarLander-v2 is the first environment you used when you started this course. At that time, you didn\u2019t know how it worked, and now you can code it from scratch and train it.\xA0"),ls=e(gn,"STRONG",{});var rJ=t(ls);Ct=U(rJ,"How incredible is that \u{1F929}."),rJ.forEach(M),gn.forEach(M),ha=o(l),h=e(l,"IFRAME",{src:!0,width:!0,height:!0,frameborder:!0,class:!0}),t(h).forEach(M),mM=e(l,"P",{});var iJ=t(mM);CM=e(iJ,"A",{href:!0});var cJ=t(CM);ut=U(cJ,"via GIPHY"),cJ.forEach(M),iJ.forEach(M),ma=o(l),uM=e(l,"P",{});var IJ=t(uM);bt=U(IJ,"Let\u2019s get started! \u{1F680}"),IJ.forEach(M),Ca=o(l),bM=e(l,"P",{});var dJ=t(bM);At=U(dJ,"The colab notebook:"),dJ.forEach(M),ua=o(l),AM=e(l,"P",{});var hJ=t(AM);ml=e(hJ,"A",{href:!0,rel:!0});var mJ=t(ml);ZM=e(mJ,"IMG",{src:!0,alt:!0}),mJ.forEach(M),hJ.forEach(M),ba=o(l),G=e(l,"H1",{class:!0});var ge=t(G);N=e(ge,"A",{id:!0,class:!0,href:!0});var CJ=t(N);Ms=e(CJ,"SPAN",{});var uJ=t(Ms);T(Cl.$$.fragment,uJ),uJ.forEach(M),CJ.forEach(M),Zt=o(ge),ss=e(ge,"SPAN",{});var bJ=t(ss);Bt=U(bJ,"Unit 8: Proximal Policy Gradient (PPO) with PyTorch \u{1F916}"),bJ.forEach(M),ge.forEach(M),Aa=o(l),ul=e(l,"IMG",{src:!0,alt:!0}),Za=o(l),H=e(l,"P",{});var _e=t(H);Gt=U(_e,"In this notebook, you\u2019ll learn to "),as=e(_e,"STRONG",{});var AJ=t(as);ft=U(AJ,"code your PPO agent from scratch with PyTorch using CleanRL implementation as model"),AJ.forEach(M),Wt=U(_e,"."),_e.forEach(M),Ba=o(l),BM=e(l,"P",{});var ZJ=t(BM);vt=U(ZJ,"To test its robustness, we\u2019re going to train it in:"),ZJ.forEach(M),Ga=o(l),GM=e(l,"UL",{});var BJ=t(GM);es=e(BJ,"LI",{});var GJ=t(es);bl=e(GJ,"A",{href:!0,rel:!0});var fJ=t(bl);Xt=U(fJ,"LunarLander-v2 \u{1F680}"),fJ.forEach(M),GJ.forEach(M),BJ.forEach(M),fa=o(l),u=e(l,"P",{});var xM=t(u);gt=U(xM,"We\u2019re constantly trying to improve our tutorials, so "),ts=e(xM,"STRONG",{});var WJ=t(ts);_t=U(WJ,"if you find some issues in this notebook"),WJ.forEach(M),kt=U(xM,", please "),Al=e(xM,"A",{href:!0,rel:!0});var vJ=t(Al);Et=U(vJ,"open an issue on the GitHub Repo"),vJ.forEach(M),Rt=U(xM,"."),xM.forEach(M),Wa=o(l),f=e(l,"H2",{class:!0});var ke=t(f);z=e(ke,"A",{id:!0,class:!0,href:!0});var XJ=t(z);ys=e(XJ,"SPAN",{});var gJ=t(ys);T(Zl.$$.fragment,gJ),gJ.forEach(M),XJ.forEach(M),Yt=o(ke),ns=e(ke,"SPAN",{});var _J=t(ns);St=U(_J,"Objectives of this notebook \u{1F3C6}"),_J.forEach(M),ke.forEach(M),va=o(l),fM=e(l,"P",{});var kJ=t(fM);Qt=U(kJ,"At the end of the notebook, you will:"),kJ.forEach(M),Xa=o(l),q=e(l,"UL",{});var Ee=t(q);Bl=e(Ee,"LI",{});var Re=t(Bl);Vt=U(Re,"Be able to "),Js=e(Re,"STRONG",{});var EJ=t(Js);Ft=U(EJ,"code your PPO agent from scratch using PyTorch"),EJ.forEach(M),Nt=U(Re,"."),Re.forEach(M),Ht=o(Ee),Gl=e(Ee,"LI",{});var Ye=t(Gl);zt=U(Ye,"Be able to "),Us=e(Ye,"STRONG",{});var RJ=t(Us);qt=U(RJ,"push your trained agent and the code to the Hub"),RJ.forEach(M),xt=U(Ye," with a nice video replay and an evaluation score \u{1F525}."),Ye.forEach(M),Ee.forEach(M),ga=o(l),W=e(l,"H2",{class:!0});var Se=t(W);x=e(Se,"A",{id:!0,class:!0,href:!0});var YJ=t(x);js=e(YJ,"SPAN",{});var SJ=t(js);T(fl.$$.fragment,SJ),SJ.forEach(M),YJ.forEach(M),Lt=o(Se),ws=e(Se,"SPAN",{});var QJ=t(ws);Kt=U(QJ,"Prerequisites \u{1F3D7}\uFE0F"),QJ.forEach(M),Se.forEach(M),_a=o(l),WM=e(l,"P",{});var VJ=t(WM);Dt=U(VJ,"Before diving into the notebook, you need to:"),VJ.forEach(M),ka=o(l),L=e(l,"P",{});var Qe=t(L);$t=U(Qe,"\u{1F532} \u{1F4DA} Study "),Wl=e(Qe,"A",{href:!0,rel:!0});var FJ=t(Wl);Pt=U(FJ,"PPO by reading Unit 8"),FJ.forEach(M),Ot=U(Qe," \u{1F917}"),Qe.forEach(M),Ea=o(l),b=e(l,"P",{});var LM=t(b);ly=U(LM,"To validate this hands-on for the "),vl=e(LM,"A",{href:!0,rel:!0});var NJ=t(vl);My=U(NJ,"certification process"),NJ.forEach(M),sy=U(LM,", you need to push one model, we don\u2019t ask for a minimal result but we "),os=e(LM,"STRONG",{});var HJ=t(os);ay=U(HJ,"advise you to try different hyperparameters settings to get better results"),HJ.forEach(M),ey=U(LM,"."),LM.forEach(M),Ra=o(l),Xl=e(l,"P",{});var _n=t(Xl);ty=U(_n,"If you don\u2019t find your model, "),ps=e(_n,"STRONG",{});var zJ=t(ps);yy=U(zJ,"go to the bottom of the page and click on the refresh button"),zJ.forEach(M),_n.forEach(M),Ya=o(l),gl=e(l,"P",{});var kn=t(gl);ny=U(kn,"For more information about the certification process, check this section \u{1F449} "),_l=e(kn,"A",{href:!0,rel:!0});var qJ=t(_l);Jy=U(qJ,"https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),qJ.forEach(M),kn.forEach(M),Sa=o(l),v=e(l,"H2",{class:!0});var Ve=t(v);K=e(Ve,"A",{id:!0,class:!0,href:!0});var xJ=t(K);Ts=e(xJ,"SPAN",{});var LJ=t(Ts);T(kl.$$.fragment,LJ),LJ.forEach(M),xJ.forEach(M),Uy=o(Ve),rs=e(Ve,"SPAN",{});var KJ=t(rs);jy=U(KJ,"Set the GPU \u{1F4AA}"),KJ.forEach(M),Ve.forEach(M),Qa=o(l),vM=e(l,"UL",{});var DJ=t(vM);D=e(DJ,"LI",{});var ya=t(D);wy=U(ya,"To "),is=e(ya,"STRONG",{});var $J=t(is);oy=U($J,"accelerate the agent\u2019s training, we\u2019ll use a GPU"),$J.forEach(M),py=U(ya,". To do that, go to "),cs=e(ya,"CODE",{});var PJ=t(cs);Ty=U(PJ,"Runtime > Change Runtime type"),PJ.forEach(M),ya.forEach(M),DJ.forEach(M),Va=o(l),El=e(l,"IMG",{src:!0,alt:!0}),Fa=o(l),XM=e(l,"UL",{});var OJ=t(XM);Is=e(OJ,"LI",{});var lU=t(Is);ds=e(lU,"CODE",{});var MU=t(ds);ry=U(MU,"Hardware Accelerator > GPU"),MU.forEach(M),lU.forEach(M),OJ.forEach(M),Na=o(l),Rl=e(l,"IMG",{src:!0,alt:!0}),Ha=o(l),X=e(l,"H2",{class:!0});var Fe=t(X);$=e(Fe,"A",{id:!0,class:!0,href:!0});var sU=t($);hs=e(sU,"SPAN",{});var aU=t(hs);T(Yl.$$.fragment,aU),aU.forEach(M),sU.forEach(M),iy=o(Fe),ms=e(Fe,"SPAN",{});var eU=t(ms);cy=U(eU,"Create a virtual display \u{1F53D}"),eU.forEach(M),Fe.forEach(M),za=o(l),P=e(l,"P",{});var Ne=t(P);Iy=U(Ne,"During the notebook, we\u2019ll need to generate a replay video. To do so, with colab, "),Cs=e(Ne,"STRONG",{});var tU=t(Cs);dy=U(tU,"we need to have a virtual screen to be able to render the environment"),tU.forEach(M),hy=U(Ne," (and thus record the frames)."),Ne.forEach(M),qa=o(l),gM=e(l,"P",{});var yU=t(gM);my=U(yU,"Hence the following cell will install the librairies and create and run a virtual screen \u{1F5A5}"),yU.forEach(M),xa=o(l),T(Sl.$$.fragment,l),La=o(l),T(Ql.$$.fragment,l),Ka=o(l),g=e(l,"H2",{class:!0});var He=t(g);O=e(He,"A",{id:!0,class:!0,href:!0});var nU=t(O);us=e(nU,"SPAN",{});var JU=t(us);T(Vl.$$.fragment,JU),JU.forEach(M),nU.forEach(M),Cy=o(He),bs=e(He,"SPAN",{});var UU=t(bs);uy=U(UU,"Install dependencies \u{1F53D}"),UU.forEach(M),He.forEach(M),Da=U(l,`

For this exercise, we use \`gym==0.21\` because the video was recorded with Gym.

	`),T(Fl.$$.fragment,l),$a=o(l),_=e(l,"H2",{class:!0});var ze=t(_);ll=e(ze,"A",{id:!0,class:!0,href:!0});var jU=t(ll);As=e(jU,"SPAN",{});var wU=t(As);T(Nl.$$.fragment,wU),wU.forEach(M),jU.forEach(M),by=o(ze),Zs=e(ze,"SPAN",{});var oU=t(Zs);Ay=U(oU,"Let's code PPO from scratch with Costa Huang's tutorial"),oU.forEach(M),ze.forEach(M),Pa=U(l,`

- For the core implementation of PPO we're going to use the excellent [Costa Huang](https://costa.sh/) tutorial.
- In addition to the tutorial, to go deeper you can read the 37 core implementation details: https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/
`),Hl=e(l,"P",{});var En=t(Hl);Zy=U(En,"\u{1F449} The video tutorial: "),zl=e(En,"A",{href:!0,rel:!0});var pU=t(zl);By=U(pU,"https://youtu.be/MEt6rrxH8W4"),pU.forEach(M),En.forEach(M),Oa=o(l),T(ql.$$.fragment,l),le=o(l),k=e(l,"H2",{class:!0});var qe=t(k);Ml=e(qe,"A",{id:!0,class:!0,href:!0});var TU=t(Ml);Bs=e(TU,"SPAN",{});var rU=t(Bs);T(xl.$$.fragment,rU),rU.forEach(M),TU.forEach(M),Gy=o(qe),Gs=e(qe,"SPAN",{});var iU=t(Gs);fy=U(iU,"Add the Hugging Face Integration \u{1F917}"),iU.forEach(M),qe.forEach(M),Me=U(l,`

- In order to push our model to the Hub, we need to define a function \`package_to_hub\`
`),_M=e(l,"UL",{});var cU=t(_M);fs=e(cU,"LI",{});var IU=t(fs);Wy=U(IU,"Add dependencies we need to push our model to the Hub"),IU.forEach(M),cU.forEach(M),se=o(l),T(Ll.$$.fragment,l),ae=o(l),kM=e(l,"UL",{});var dU=t(kM);Kl=e(dU,"LI",{});var xe=t(Kl);vy=U(xe,"Add new argument in "),Ws=e(xe,"CODE",{});var hU=t(Ws);Xy=U(hU,"parse_args()"),hU.forEach(M),gy=U(xe," function to define the repo-id where we want to push the model."),xe.forEach(M),dU.forEach(M),ee=o(l),T(Dl.$$.fragment,l),te=o(l),sl=e(l,"UL",{});var Le=t(sl);vs=e(Le,"LI",{});var mU=t(vs);Xs=e(mU,"P",{});var CU=t(Xs);_y=U(CU,"Next, we add the methods needed to push the model to the Hub"),CU.forEach(M),mU.forEach(M),ky=o(Le),$l=e(Le,"LI",{});var Ke=t($l);gs=e(Ke,"P",{});var uU=t(gs);Ey=U(uU,"These methods will:"),uU.forEach(M),Ry=o(Ke),E=e(Ke,"UL",{});var KM=t(E);EM=e(KM,"LI",{});var Rn=t(EM);_s=e(Rn,"CODE",{});var bU=t(_s);Yy=U(bU,"_evalutate_agent()"),bU.forEach(M),Sy=U(Rn,": evaluate the agent."),Rn.forEach(M),Qy=o(KM),RM=e(KM,"LI",{});var Yn=t(RM);ks=e(Yn,"CODE",{});var AU=t(ks);Vy=U(AU,"_generate_model_card()"),AU.forEach(M),Fy=U(Yn,": generate the model card of your agent."),Yn.forEach(M),Ny=o(KM),YM=e(KM,"LI",{});var Sn=t(YM);Es=e(Sn,"CODE",{});var ZU=t(Es);Hy=U(ZU,"_record_video()"),ZU.forEach(M),zy=U(Sn,": record a video of your agent."),Sn.forEach(M),KM.forEach(M),Ke.forEach(M),Le.forEach(M),ye=o(l),T(Pl.$$.fragment,l),ne=o(l),SM=e(l,"UL",{});var BU=t(SM);Rs=e(BU,"LI",{});var GU=t(Rs);qy=U(GU,"Finally, we call this function at the end of the PPO training"),GU.forEach(M),BU.forEach(M),Je=o(l),T(Ol.$$.fragment,l),Ue=o(l),QM=e(l,"UL",{});var fU=t(QM);Ys=e(fU,"LI",{});var WU=t(Ys);xy=U(WU,"Here\u2019s what the final ppo.py file looks like:"),WU.forEach(M),fU.forEach(M),je=o(l),T(lM.$$.fragment,l),we=o(l),VM=e(l,"P",{});var vU=t(VM);Ly=U(vU,"To be able to share your model with the community there are three more steps to follow:"),vU.forEach(M),oe=o(l),MM=e(l,"P",{});var Qn=t(MM);Ky=U(Qn,"1\uFE0F\u20E3 (If it\u2019s not already done) create an account to HF \u27A1 "),sM=e(Qn,"A",{href:!0,rel:!0});var XU=t(sM);Dy=U(XU,"https://huggingface.co/join"),XU.forEach(M),Qn.forEach(M),pe=o(l),FM=e(l,"P",{});var gU=t(FM);$y=U(gU,"2\uFE0F\u20E3 Sign in and get your authentication token from the Hugging Face website."),gU.forEach(M),Te=o(l),NM=e(l,"UL",{});var _U=t(NM);al=e(_U,"LI",{});var na=t(al);Py=U(na,"Create a new token ("),aM=e(na,"A",{href:!0,rel:!0});var kU=t(aM);Oy=U(kU,"https://huggingface.co/settings/tokens"),kU.forEach(M),ln=U(na,") "),Ss=e(na,"STRONG",{});var EU=t(Ss);Mn=U(EU,"with write role"),EU.forEach(M),na.forEach(M),_U.forEach(M),re=o(l),eM=e(l,"IMG",{src:!0,alt:!0}),ie=o(l),el=e(l,"UL",{});var De=t(el);Qs=e(De,"LI",{});var RU=t(Qs);sn=U(RU,"Copy the token"),RU.forEach(M),an=o(De),Vs=e(De,"LI",{});var YU=t(Vs);en=U(YU,"Run the cell below and paste the token"),YU.forEach(M),De.forEach(M),ce=o(l),T(tM.$$.fragment,l),Ie=o(l),yM=e(l,"P",{});var Vn=t(yM);tn=U(Vn,"If you don\u2019t want to use Google Colab or a Jupyter Notebook, you need to use this command instead: "),Fs=e(Vn,"CODE",{});var SU=t(Fs);yn=U(SU,"huggingface-cli login"),SU.forEach(M),Vn.forEach(M),de=o(l),R=e(l,"H2",{class:!0});var $e=t(R);tl=e($e,"A",{id:!0,class:!0,href:!0});var QU=t(tl);Ns=e(QU,"SPAN",{});var VU=t(Ns);T(nM.$$.fragment,VU),VU.forEach(M),QU.forEach(M),nn=o($e),Hs=e($e,"SPAN",{});var FU=t(Hs);Jn=U(FU,"Let's start the training \u{1F525}"),FU.forEach(M),$e.forEach(M),he=o(l),JM=e(l,"P",{});var Fn=t(JM);Un=U(Fn,"\u26A0\uFE0F \u26A0\uFE0F \u26A0\uFE0F  Don\u2019t use "),zs=e(Fn,"STRONG",{});var NU=t(zs);jn=U(NU,"the same repo id with the one you used for the Unit 1"),NU.forEach(M),Fn.forEach(M),me=o(l),yl=e(l,"UL",{});var Pe=t(yl);qs=e(Pe,"LI",{});var HU=t(qs);xs=e(HU,"P",{});var zU=t(xs);wn=U(zU,"Now that you\u2019ve coded PPO from scratch and added the Hugging Face Integration, we\u2019re ready to start the training \u{1F525}"),zU.forEach(M),HU.forEach(M),on=o(Pe),Ls=e(Pe,"LI",{});var qU=t(Ls);HM=e(qU,"P",{});var Nn=t(HM);pn=U(Nn,"First, you need to copy all your code to a file you create called "),Ks=e(Nn,"CODE",{});var xU=t(Ks);Tn=U(xU,"ppo.py"),xU.forEach(M),Nn.forEach(M),qU.forEach(M),Pe.forEach(M),Ce=o(l),UM=e(l,"IMG",{src:!0,alt:!0}),ue=o(l),jM=e(l,"IMG",{src:!0,alt:!0}),be=o(l),nl=e(l,"UL",{});var Oe=t(nl);Ds=e(Oe,"LI",{});var LU=t(Ds);Jl=e(LU,"P",{});var Ja=t(Jl);rn=U(Ja,"Now we just need to run this python script using "),$s=e(Ja,"CODE",{});var KU=t($s);cn=U(KU,"python <name-of-python-script>.py"),KU.forEach(M),In=U(Ja," with the additional parameters we defined using "),Ps=e(Ja,"CODE",{});var DU=t(Ps);dn=U(DU,"argparse"),DU.forEach(M),Ja.forEach(M),LU.forEach(M),hn=o(Oe),Os=e(Oe,"LI",{});var $U=t(Os);la=e($U,"P",{});var PU=t(la);mn=U(PU,"You should modify more hyperparameters otherwise the training will not be super stable."),PU.forEach(M),$U.forEach(M),Oe.forEach(M),Ae=o(l),T(wM.$$.fragment,l),Ze=o(l),Y=e(l,"H2",{class:!0});var lt=t(Y);Ul=e(lt,"A",{id:!0,class:!0,href:!0});var OU=t(Ul);Ma=e(OU,"SPAN",{});var lj=t(Ma);T(oM.$$.fragment,lj),lj.forEach(M),OU.forEach(M),Cn=o(lt),sa=e(lt,"SPAN",{});var Mj=t(sa);un=U(Mj,"Some additional challenges \u{1F3C6}"),Mj.forEach(M),lt.forEach(M),Be=o(l),jl=e(l,"P",{});var Mt=t(jl);bn=U(Mt,"The best way to learn "),aa=e(Mt,"STRONG",{});var sj=t(aa);An=U(sj,"is to try things on your own"),sj.forEach(M),Zn=U(Mt,"! Why not try another environment? Or why not trying to modify the implementation to work with Gymnasium?"),Mt.forEach(M),Ge=o(l),zM=e(l,"P",{});var aj=t(zM);Bn=U(aj,"See you in Unit 8, part 2 where we\u2019re going to train agents to play Doom \u{1F525}"),aj.forEach(M),fe=o(l),S=e(l,"H2",{class:!0});var st=t(S);wl=e(st,"A",{id:!0,class:!0,href:!0});var ej=t(wl);ea=e(ej,"SPAN",{});var tj=t(ea);T(pM.$$.fragment,tj),tj.forEach(M),ej.forEach(M),Gn=o(st),ta=e(st,"SPAN",{});var yj=t(ta);fn=U(yj,"Keep learning, stay awesome \u{1F917}"),yj.forEach(M),st.forEach(M),this.h()},h(){j(A,"name","hf:doc:metadata"),j(A,"content",JSON.stringify(ij)),j(V,"id","handson"),j(V,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(V,"href","#handson"),j(Z,"class","relative group"),j(rl,"href","https://github.com/vwxyzjn"),j(rl,"rel","nofollow"),j(il,"href","https://github.com/vwxyzjn/cleanrl"),j(il,"rel","nofollow"),j(cl,"href","https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/"),j(cl,"rel","nofollow"),j(Il,"href","https://www.gymlibrary.ml/environments/box2d/lunar_lander/"),j(Il,"rel","nofollow"),Q(dM.src,zn="assets/63_deep_rl_intro/lunarlander.mp4")||j(dM,"src",zn),j(dM,"type","video/mp4"),j(d,"alt","LunarLander"),nj(d,"max-width","70%"),nj(d,"margin","auto"),d.autoplay=!0,d.loop=!0,j(d,"autobuffer",""),d.muted=!0,d.playsInline=!0,j(dl,"class","image table text-center m-0 w-full"),Q(h.src,qn="https://giphy.com/embed/pynZagVcYxVUk")||j(h,"src",qn),j(h,"width","480"),j(h,"height","480"),j(h,"frameborder","0"),j(h,"class","giphy-embed"),h.allowFullscreen=!0,j(CM,"href","https://giphy.com/gifs/the-office-michael-heartbreak-pynZagVcYxVUk"),Q(ZM.src,xn="https://colab.research.google.com/assets/colab-badge.svg")||j(ZM,"src",xn),j(ZM,"alt","Open In Colab"),j(ml,"href","https://colab.research.google.com/github/huggingface/deep-rl-class/blob/master/notebooks/unit8/unit8_part1.ipynb"),j(ml,"rel","nofollow"),j(N,"id","unit-8-proximal-policy-gradient-ppo-with-pytorch"),j(N,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(N,"href","#unit-8-proximal-policy-gradient-ppo-with-pytorch"),j(G,"class","relative group"),Q(ul.src,Ln="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit9/thumbnail.png")||j(ul,"src",Ln),j(ul,"alt","Unit 8"),j(bl,"href","https://www.gymlibrary.dev/environments/box2d/lunar_lander/"),j(bl,"rel","nofollow"),j(Al,"href","https://github.com/huggingface/deep-rl-class/issues"),j(Al,"rel","nofollow"),j(z,"id","objectives-of-this-notebook"),j(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(z,"href","#objectives-of-this-notebook"),j(f,"class","relative group"),j(x,"id","prerequisites"),j(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(x,"href","#prerequisites"),j(W,"class","relative group"),j(Wl,"href","https://huggingface.co/deep-rl-course/unit8/introduction"),j(Wl,"rel","nofollow"),j(vl,"href","https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),j(vl,"rel","nofollow"),j(_l,"href","https://huggingface.co/deep-rl-course/en/unit0/introduction#certification-process"),j(_l,"rel","nofollow"),j(K,"id","set-the-gpu"),j(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(K,"href","#set-the-gpu"),j(v,"class","relative group"),Q(El.src,Kn="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step1.jpg")||j(El,"src",Kn),j(El,"alt","GPU Step 1"),Q(Rl.src,Dn="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/gpu-step2.jpg")||j(Rl,"src",Dn),j(Rl,"alt","GPU Step 2"),j($,"id","create-a-virtual-display"),j($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j($,"href","#create-a-virtual-display"),j(X,"class","relative group"),j(O,"id","install-dependencies"),j(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(O,"href","#install-dependencies"),j(g,"class","relative group"),j(ll,"id","lets-code-ppo-from-scratch-with-costa-huangs-tutorial"),j(ll,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(ll,"href","#lets-code-ppo-from-scratch-with-costa-huangs-tutorial"),j(_,"class","relative group"),j(zl,"href","https://youtu.be/MEt6rrxH8W4"),j(zl,"rel","nofollow"),j(Ml,"id","add-the-hugging-face-integration"),j(Ml,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(Ml,"href","#add-the-hugging-face-integration"),j(k,"class","relative group"),j(sM,"href","https://huggingface.co/join"),j(sM,"rel","nofollow"),j(aM,"href","https://huggingface.co/settings/tokens"),j(aM,"rel","nofollow"),Q(eM.src,$n="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/notebooks/create-token.jpg")||j(eM,"src",$n),j(eM,"alt","Create HF Token"),j(tl,"id","lets-start-the-training"),j(tl,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(tl,"href","#lets-start-the-training"),j(R,"class","relative group"),Q(UM.src,Pn="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit9/step1.png")||j(UM,"src",Pn),j(UM,"alt","PPO"),Q(jM.src,On="https://huggingface.co/datasets/huggingface-deep-rl-course/course-images/resolve/main/en/unit9/step2.png")||j(jM,"src",On),j(jM,"alt","PPO"),j(Ul,"id","some-additional-challenges"),j(Ul,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(Ul,"href","#some-additional-challenges"),j(Y,"class","relative group"),j(wl,"id","keep-learning-stay-awesome"),j(wl,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(wl,"href","#keep-learning-stay-awesome"),j(S,"class","relative group")},m(l,y){s(document.head,A),n(l,Ua,y),n(l,Z,y),s(Z,V),s(V,DM),r(ol,DM,null),s(Z,at),s(Z,$M),s($M,et),n(l,ja,y),r(pl,l,y),n(l,wa,y),n(l,Tl,y),s(Tl,tt),s(Tl,PM),s(PM,yt),n(l,oa,y),n(l,TM,y),s(TM,nt),n(l,pa,y),n(l,rM,y),s(rM,Jt),n(l,Ta,y),n(l,F,y),s(F,B),s(B,Ut),s(B,rl),s(rl,jt),s(B,wt),s(B,il),s(il,ot),s(B,pt),s(F,Tt),s(F,iM),s(iM,rt),s(iM,cl),s(cl,it),n(l,ra,y),n(l,cM,y),s(cM,ct),n(l,ia,y),n(l,IM,y),s(IM,OM),s(OM,Il),s(Il,It),n(l,ca,y),n(l,dl,y),s(dl,d),s(d,dM),s(d,dt),n(l,Ia,y),n(l,hM,y),s(hM,ht),n(l,da,y),n(l,hl,y),s(hl,mt),s(hl,ls),s(ls,Ct),n(l,ha,y),n(l,h,y),n(l,mM,y),s(mM,CM),s(CM,ut),n(l,ma,y),n(l,uM,y),s(uM,bt),n(l,Ca,y),n(l,bM,y),s(bM,At),n(l,ua,y),n(l,AM,y),s(AM,ml),s(ml,ZM),n(l,ba,y),n(l,G,y),s(G,N),s(N,Ms),r(Cl,Ms,null),s(G,Zt),s(G,ss),s(ss,Bt),n(l,Aa,y),n(l,ul,y),n(l,Za,y),n(l,H,y),s(H,Gt),s(H,as),s(as,ft),s(H,Wt),n(l,Ba,y),n(l,BM,y),s(BM,vt),n(l,Ga,y),n(l,GM,y),s(GM,es),s(es,bl),s(bl,Xt),n(l,fa,y),n(l,u,y),s(u,gt),s(u,ts),s(ts,_t),s(u,kt),s(u,Al),s(Al,Et),s(u,Rt),n(l,Wa,y),n(l,f,y),s(f,z),s(z,ys),r(Zl,ys,null),s(f,Yt),s(f,ns),s(ns,St),n(l,va,y),n(l,fM,y),s(fM,Qt),n(l,Xa,y),n(l,q,y),s(q,Bl),s(Bl,Vt),s(Bl,Js),s(Js,Ft),s(Bl,Nt),s(q,Ht),s(q,Gl),s(Gl,zt),s(Gl,Us),s(Us,qt),s(Gl,xt),n(l,ga,y),n(l,W,y),s(W,x),s(x,js),r(fl,js,null),s(W,Lt),s(W,ws),s(ws,Kt),n(l,_a,y),n(l,WM,y),s(WM,Dt),n(l,ka,y),n(l,L,y),s(L,$t),s(L,Wl),s(Wl,Pt),s(L,Ot),n(l,Ea,y),n(l,b,y),s(b,ly),s(b,vl),s(vl,My),s(b,sy),s(b,os),s(os,ay),s(b,ey),n(l,Ra,y),n(l,Xl,y),s(Xl,ty),s(Xl,ps),s(ps,yy),n(l,Ya,y),n(l,gl,y),s(gl,ny),s(gl,_l),s(_l,Jy),n(l,Sa,y),n(l,v,y),s(v,K),s(K,Ts),r(kl,Ts,null),s(v,Uy),s(v,rs),s(rs,jy),n(l,Qa,y),n(l,vM,y),s(vM,D),s(D,wy),s(D,is),s(is,oy),s(D,py),s(D,cs),s(cs,Ty),n(l,Va,y),n(l,El,y),n(l,Fa,y),n(l,XM,y),s(XM,Is),s(Is,ds),s(ds,ry),n(l,Na,y),n(l,Rl,y),n(l,Ha,y),n(l,X,y),s(X,$),s($,hs),r(Yl,hs,null),s(X,iy),s(X,ms),s(ms,cy),n(l,za,y),n(l,P,y),s(P,Iy),s(P,Cs),s(Cs,dy),s(P,hy),n(l,qa,y),n(l,gM,y),s(gM,my),n(l,xa,y),r(Sl,l,y),n(l,La,y),r(Ql,l,y),n(l,Ka,y),n(l,g,y),s(g,O),s(O,us),r(Vl,us,null),s(g,Cy),s(g,bs),s(bs,uy),n(l,Da,y),r(Fl,l,y),n(l,$a,y),n(l,_,y),s(_,ll),s(ll,As),r(Nl,As,null),s(_,by),s(_,Zs),s(Zs,Ay),n(l,Pa,y),n(l,Hl,y),s(Hl,Zy),s(Hl,zl),s(zl,By),n(l,Oa,y),r(ql,l,y),n(l,le,y),n(l,k,y),s(k,Ml),s(Ml,Bs),r(xl,Bs,null),s(k,Gy),s(k,Gs),s(Gs,fy),n(l,Me,y),n(l,_M,y),s(_M,fs),s(fs,Wy),n(l,se,y),r(Ll,l,y),n(l,ae,y),n(l,kM,y),s(kM,Kl),s(Kl,vy),s(Kl,Ws),s(Ws,Xy),s(Kl,gy),n(l,ee,y),r(Dl,l,y),n(l,te,y),n(l,sl,y),s(sl,vs),s(vs,Xs),s(Xs,_y),s(sl,ky),s(sl,$l),s($l,gs),s(gs,Ey),s($l,Ry),s($l,E),s(E,EM),s(EM,_s),s(_s,Yy),s(EM,Sy),s(E,Qy),s(E,RM),s(RM,ks),s(ks,Vy),s(RM,Fy),s(E,Ny),s(E,YM),s(YM,Es),s(Es,Hy),s(YM,zy),n(l,ye,y),r(Pl,l,y),n(l,ne,y),n(l,SM,y),s(SM,Rs),s(Rs,qy),n(l,Je,y),r(Ol,l,y),n(l,Ue,y),n(l,QM,y),s(QM,Ys),s(Ys,xy),n(l,je,y),r(lM,l,y),n(l,we,y),n(l,VM,y),s(VM,Ly),n(l,oe,y),n(l,MM,y),s(MM,Ky),s(MM,sM),s(sM,Dy),n(l,pe,y),n(l,FM,y),s(FM,$y),n(l,Te,y),n(l,NM,y),s(NM,al),s(al,Py),s(al,aM),s(aM,Oy),s(al,ln),s(al,Ss),s(Ss,Mn),n(l,re,y),n(l,eM,y),n(l,ie,y),n(l,el,y),s(el,Qs),s(Qs,sn),s(el,an),s(el,Vs),s(Vs,en),n(l,ce,y),r(tM,l,y),n(l,Ie,y),n(l,yM,y),s(yM,tn),s(yM,Fs),s(Fs,yn),n(l,de,y),n(l,R,y),s(R,tl),s(tl,Ns),r(nM,Ns,null),s(R,nn),s(R,Hs),s(Hs,Jn),n(l,he,y),n(l,JM,y),s(JM,Un),s(JM,zs),s(zs,jn),n(l,me,y),n(l,yl,y),s(yl,qs),s(qs,xs),s(xs,wn),s(yl,on),s(yl,Ls),s(Ls,HM),s(HM,pn),s(HM,Ks),s(Ks,Tn),n(l,Ce,y),n(l,UM,y),n(l,ue,y),n(l,jM,y),n(l,be,y),n(l,nl,y),s(nl,Ds),s(Ds,Jl),s(Jl,rn),s(Jl,$s),s($s,cn),s(Jl,In),s(Jl,Ps),s(Ps,dn),s(nl,hn),s(nl,Os),s(Os,la),s(la,mn),n(l,Ae,y),r(wM,l,y),n(l,Ze,y),n(l,Y,y),s(Y,Ul),s(Ul,Ma),r(oM,Ma,null),s(Y,Cn),s(Y,sa),s(sa,un),n(l,Be,y),n(l,jl,y),s(jl,bn),s(jl,aa),s(aa,An),s(jl,Zn),n(l,Ge,y),n(l,zM,y),s(zM,Bn),n(l,fe,y),n(l,S,y),s(S,wl),s(wl,ea),r(pM,ea,null),s(S,Gn),s(S,ta),s(ta,fn),We=!0},p:oj,i(l){We||(i(ol.$$.fragment,l),i(pl.$$.fragment,l),i(Cl.$$.fragment,l),i(Zl.$$.fragment,l),i(fl.$$.fragment,l),i(kl.$$.fragment,l),i(Yl.$$.fragment,l),i(Sl.$$.fragment,l),i(Ql.$$.fragment,l),i(Vl.$$.fragment,l),i(Fl.$$.fragment,l),i(Nl.$$.fragment,l),i(ql.$$.fragment,l),i(xl.$$.fragment,l),i(Ll.$$.fragment,l),i(Dl.$$.fragment,l),i(Pl.$$.fragment,l),i(Ol.$$.fragment,l),i(lM.$$.fragment,l),i(tM.$$.fragment,l),i(nM.$$.fragment,l),i(wM.$$.fragment,l),i(oM.$$.fragment,l),i(pM.$$.fragment,l),We=!0)},o(l){c(ol.$$.fragment,l),c(pl.$$.fragment,l),c(Cl.$$.fragment,l),c(Zl.$$.fragment,l),c(fl.$$.fragment,l),c(kl.$$.fragment,l),c(Yl.$$.fragment,l),c(Sl.$$.fragment,l),c(Ql.$$.fragment,l),c(Vl.$$.fragment,l),c(Fl.$$.fragment,l),c(Nl.$$.fragment,l),c(ql.$$.fragment,l),c(xl.$$.fragment,l),c(Ll.$$.fragment,l),c(Dl.$$.fragment,l),c(Pl.$$.fragment,l),c(Ol.$$.fragment,l),c(lM.$$.fragment,l),c(tM.$$.fragment,l),c(nM.$$.fragment,l),c(wM.$$.fragment,l),c(oM.$$.fragment,l),c(pM.$$.fragment,l),We=!1},d(l){M(A),l&&M(Ua),l&&M(Z),I(ol),l&&M(ja),I(pl,l),l&&M(wa),l&&M(Tl),l&&M(oa),l&&M(TM),l&&M(pa),l&&M(rM),l&&M(Ta),l&&M(F),l&&M(ra),l&&M(cM),l&&M(ia),l&&M(IM),l&&M(ca),l&&M(dl),l&&M(Ia),l&&M(hM),l&&M(da),l&&M(hl),l&&M(ha),l&&M(h),l&&M(mM),l&&M(ma),l&&M(uM),l&&M(Ca),l&&M(bM),l&&M(ua),l&&M(AM),l&&M(ba),l&&M(G),I(Cl),l&&M(Aa),l&&M(ul),l&&M(Za),l&&M(H),l&&M(Ba),l&&M(BM),l&&M(Ga),l&&M(GM),l&&M(fa),l&&M(u),l&&M(Wa),l&&M(f),I(Zl),l&&M(va),l&&M(fM),l&&M(Xa),l&&M(q),l&&M(ga),l&&M(W),I(fl),l&&M(_a),l&&M(WM),l&&M(ka),l&&M(L),l&&M(Ea),l&&M(b),l&&M(Ra),l&&M(Xl),l&&M(Ya),l&&M(gl),l&&M(Sa),l&&M(v),I(kl),l&&M(Qa),l&&M(vM),l&&M(Va),l&&M(El),l&&M(Fa),l&&M(XM),l&&M(Na),l&&M(Rl),l&&M(Ha),l&&M(X),I(Yl),l&&M(za),l&&M(P),l&&M(qa),l&&M(gM),l&&M(xa),I(Sl,l),l&&M(La),I(Ql,l),l&&M(Ka),l&&M(g),I(Vl),l&&M(Da),I(Fl,l),l&&M($a),l&&M(_),I(Nl),l&&M(Pa),l&&M(Hl),l&&M(Oa),I(ql,l),l&&M(le),l&&M(k),I(xl),l&&M(Me),l&&M(_M),l&&M(se),I(Ll,l),l&&M(ae),l&&M(kM),l&&M(ee),I(Dl,l),l&&M(te),l&&M(sl),l&&M(ye),I(Pl,l),l&&M(ne),l&&M(SM),l&&M(Je),I(Ol,l),l&&M(Ue),l&&M(QM),l&&M(je),I(lM,l),l&&M(we),l&&M(VM),l&&M(oe),l&&M(MM),l&&M(pe),l&&M(FM),l&&M(Te),l&&M(NM),l&&M(re),l&&M(eM),l&&M(ie),l&&M(el),l&&M(ce),I(tM,l),l&&M(Ie),l&&M(yM),l&&M(de),l&&M(R),I(nM),l&&M(he),l&&M(JM),l&&M(me),l&&M(yl),l&&M(Ce),l&&M(UM),l&&M(ue),l&&M(jM),l&&M(be),l&&M(nl),l&&M(Ae),I(wM,l),l&&M(Ze),l&&M(Y),I(oM),l&&M(Be),l&&M(jl),l&&M(Ge),l&&M(zM),l&&M(fe),l&&M(S),I(pM)}}}const ij={local:"unit-8-proximal-policy-gradient-ppo-with-pytorch",sections:[{local:"objectives-of-this-notebook",title:"Objectives of this notebook \u{1F3C6}"},{local:"prerequisites",title:"Prerequisites \u{1F3D7}\uFE0F"},{local:"set-the-gpu",title:"Set the GPU \u{1F4AA}"},{local:"create-a-virtual-display",title:"Create a virtual display \u{1F53D}"},{local:"install-dependencies",title:"Install dependencies \u{1F53D}"},{local:"lets-code-ppo-from-scratch-with-costa-huangs-tutorial",title:"Let's code PPO from scratch with Costa Huang's tutorial"},{local:"add-the-hugging-face-integration",title:"Add the Hugging Face Integration \u{1F917}"},{local:"lets-start-the-training",title:"Let's start the training \u{1F525}"},{local:"some-additional-challenges",title:"Some additional challenges \u{1F3C6}"},{local:"keep-learning-stay-awesome",title:"Keep learning, stay awesome \u{1F917}"}],title:"Unit 8: Proximal Policy Gradient (PPO) with PyTorch \u{1F916}"};function cj(Hn){return pj(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Cj extends Jj{constructor(A){super();Uj(this,A,cj,rj,jj,{})}}export{Cj as default,ij as metadata};
