import{S as rr,i as ar,s as nr,e as r,k as d,w as Ee,t as i,M as ir,c as a,d as t,m as f,a as n,x as ke,h as s,b as p,G as o,g as h,y as $e,L as sr,q as Ae,o as Pe,B as Le,v as lr}from"../../chunks/vendor-hf-doc-builder.js";import{I as Te}from"../../chunks/IconCopyLink-hf-doc-builder.js";function hr(vo){let w,Se,g,E,ae,N,lt,ne,ht,xe,H,pt,ie,dt,Me,k,ft,se,ct,ut,Ne,Q,mt,He,$,vt,q,wt,gt,qe,A,bt,C,yt,_t,Ce,V,Rt,Ge,b,P,le,G,Et,he,kt,Ie,X,$t,Be,c,At,I,pe,Pt,Lt,de,Tt,St,fe,xt,Mt,ce,Nt,Ht,ue,qt,Ct,ze,v,Gt,B,It,Bt,z,zt,Ot,Oe,O,me,Dt,Ft,De,Y,Wt,Fe,Z,Jt,We,y,L,ve,D,Ut,we,jt,Je,T,Kt,ge,Qt,Vt,Ue,ee,Xt,je,_,S,be,F,Yt,ye,Zt,Ke,te,eo,Qe,u,to,W,oo,ro,J,ao,no,U,io,so,Ve,R,x,_e,j,lo,Re,ho,Xe,K,po,oe,fo,Ye;return N=new Te({}),G=new Te({}),D=new Te({}),F=new Te({}),j=new Te({}),{c(){w=r("meta"),Se=d(),g=r("h1"),E=r("a"),ae=r("span"),Ee(N.$$.fragment),lt=d(),ne=r("span"),ht=i("Brief introduction to RL documentation"),xe=d(),H=r("p"),pt=i("In this advanced topic, we address the question: "),ie=r("strong"),dt=i(`how should we monitor and keep track of powerful reinforcement learning agents that we are training in the real world and
interfacing with humans?`),Me=d(),k=r("p"),ft=i("As machine learning systems have increasingly impacted modern life, the "),se=r("strong"),ct=i("call for the documentation of these systems has grown"),ut=i("."),Ne=d(),Q=r("p"),mt=i(`Such documentation can cover aspects such as the training data used \u2014 where it is stored, when it was collected, who was involved, etc.
\u2014 or the model optimization framework \u2014 the architecture, evaluation metrics, relevant papers, etc. \u2014 and more.`),He=d(),$=r("p"),vt=i(`Today, model cards and datasheets are becoming increasingly available. For example, on the Hub
(see documentation `),q=r("a"),wt=i("here"),gt=i(")."),qe=d(),A=r("p"),bt=i("If you click on a "),C=r("a"),yt=i("popular model on the Hub"),_t=i(", you can learn about its creation process."),Ce=d(),V=r("p"),Rt=i(`These model and data specific logs are designed to be completed when the model or dataset are created, leaving them to go un-updated when these models are built into evolving systems in the future.
\u200B`),Ge=d(),b=r("h2"),P=r("a"),le=r("span"),Ee(G.$$.fragment),Et=d(),he=r("span"),kt=i("Motivating Reward Reports"),Ie=d(),X=r("p"),$t=i(`Reinforcement learning systems are fundamentally designed to optimize based on measurements of reward and time.
While the notion of a reward function can be mapped nicely to many well-understood fields of supervised learning (via a loss function),
understanding of how machine learning systems evolve over time is limited.`),Be=d(),c=r("p"),At=i("To that end, the authors introduce "),I=r("a"),pe=r("em"),Pt=i("Reward Reports for Reinforcement Learning"),Lt=i(" (the pithy naming is designed to mirror the popular papers "),de=r("em"),Tt=i("Model Cards for Model Reporting"),St=i(" and "),fe=r("em"),xt=i("Datasheets for Datasets"),Mt=i(`).
The goal is to propose a type of documentation focused on the `),ce=r("strong"),Nt=i("human factors of reward"),Ht=i(" and "),ue=r("strong"),qt=i("time-varying feedback systems"),Ct=i("."),ze=d(),v=r("p"),Gt=i("Building on the documentation frameworks for "),B=r("a"),It=i("model cards"),Bt=i(" and "),z=r("a"),zt=i("datasheets"),Ot=i(" proposed by Mitchell et al. and Gebru et al., we argue the need for Reward Reports for AI systems."),Oe=d(),O=r("p"),me=r("strong"),Dt=i("Reward Reports"),Ft=i(" are living documents for proposed RL deployments that demarcate design choices."),De=d(),Y=r("p"),Wt=i(`However, many questions remain about the applicability of this framework to different RL applications, roadblocks to system interpretability,
and the resonances between deployed supervised machine learning systems and the sequential decision-making utilized in RL.`),Fe=d(),Z=r("p"),Jt=i(`At a minimum, Reward Reports are an opportunity for RL practitioners to deliberate on these questions and begin the work of deciding how to resolve them in practice.
\u200B`),We=d(),y=r("h2"),L=r("a"),ve=r("span"),Ee(D.$$.fragment),Ut=d(),we=r("span"),jt=i("Capturing temporal behavior with documentation"),Je=d(),T=r("p"),Kt=i("The core piece specific to documentation designed for RL and feedback-driven ML systems is a "),ge=r("em"),Qt=i("change-log"),Vt=i(`. The change-log updates information
from the designer (changed training parameters, data, etc.) along with noticed changes from the user (harmful behavior, unexpected responses, etc.).`),Ue=d(),ee=r("p"),Xt=i("The change log is accompanied by update triggers that encourage monitoring these effects."),je=d(),_=r("h2"),S=r("a"),be=r("span"),Ee(F.$$.fragment),Yt=d(),ye=r("span"),Zt=i("Contributing"),Ke=d(),te=r("p"),eo=i(`Some of the most impactful RL-driven systems are multi-stakeholder in nature and behind the closed doors of private corporations.
These corporations are largely without regulation, so the burden of documentation falls on the public.`),Qe=d(),u=r("p"),to=i(`If you are interested in contributing, we are building Reward Reports for popular machine learning systems on a public
record on `),W=r("a"),oo=i("GitHub"),ro=i(`.
\u200B
For further reading, you can visit the Reward Reports `),J=r("a"),ao=i("paper"),no=i(`
or look `),U=r("a"),io=i("an example report"),so=i("."),Ve=d(),R=r("h2"),x=r("a"),_e=r("span"),Ee(j.$$.fragment),lo=d(),Re=r("span"),ho=i("Author"),Xe=d(),K=r("p"),po=i("This section was written by "),oe=r("a"),fo=i("Nathan Lambert"),this.h()},l(e){const l=ir('[data-svelte="svelte-1phssyn"]',document.head);w=a(l,"META",{name:!0,content:!0}),l.forEach(t),Se=f(e),g=a(e,"H1",{class:!0});var Ze=n(g);E=a(Ze,"A",{id:!0,class:!0,href:!0});var wo=n(E);ae=a(wo,"SPAN",{});var go=n(ae);ke(N.$$.fragment,go),go.forEach(t),wo.forEach(t),lt=f(Ze),ne=a(Ze,"SPAN",{});var bo=n(ne);ht=s(bo,"Brief introduction to RL documentation"),bo.forEach(t),Ze.forEach(t),xe=f(e),H=a(e,"P",{});var co=n(H);pt=s(co,"In this advanced topic, we address the question: "),ie=a(co,"STRONG",{});var yo=n(ie);dt=s(yo,`how should we monitor and keep track of powerful reinforcement learning agents that we are training in the real world and
interfacing with humans?`),yo.forEach(t),co.forEach(t),Me=f(e),k=a(e,"P",{});var et=n(k);ft=s(et,"As machine learning systems have increasingly impacted modern life, the "),se=a(et,"STRONG",{});var _o=n(se);ct=s(_o,"call for the documentation of these systems has grown"),_o.forEach(t),ut=s(et,"."),et.forEach(t),Ne=f(e),Q=a(e,"P",{});var Ro=n(Q);mt=s(Ro,`Such documentation can cover aspects such as the training data used \u2014 where it is stored, when it was collected, who was involved, etc.
\u2014 or the model optimization framework \u2014 the architecture, evaluation metrics, relevant papers, etc. \u2014 and more.`),Ro.forEach(t),He=f(e),$=a(e,"P",{});var tt=n($);vt=s(tt,`Today, model cards and datasheets are becoming increasingly available. For example, on the Hub
(see documentation `),q=a(tt,"A",{href:!0,rel:!0});var Eo=n(q);wt=s(Eo,"here"),Eo.forEach(t),gt=s(tt,")."),tt.forEach(t),qe=f(e),A=a(e,"P",{});var ot=n(A);bt=s(ot,"If you click on a "),C=a(ot,"A",{href:!0,rel:!0});var ko=n(C);yt=s(ko,"popular model on the Hub"),ko.forEach(t),_t=s(ot,", you can learn about its creation process."),ot.forEach(t),Ce=f(e),V=a(e,"P",{});var $o=n(V);Rt=s($o,`These model and data specific logs are designed to be completed when the model or dataset are created, leaving them to go un-updated when these models are built into evolving systems in the future.
\u200B`),$o.forEach(t),Ge=f(e),b=a(e,"H2",{class:!0});var rt=n(b);P=a(rt,"A",{id:!0,class:!0,href:!0});var Ao=n(P);le=a(Ao,"SPAN",{});var Po=n(le);ke(G.$$.fragment,Po),Po.forEach(t),Ao.forEach(t),Et=f(rt),he=a(rt,"SPAN",{});var Lo=n(he);kt=s(Lo,"Motivating Reward Reports"),Lo.forEach(t),rt.forEach(t),Ie=f(e),X=a(e,"P",{});var To=n(X);$t=s(To,`Reinforcement learning systems are fundamentally designed to optimize based on measurements of reward and time.
While the notion of a reward function can be mapped nicely to many well-understood fields of supervised learning (via a loss function),
understanding of how machine learning systems evolve over time is limited.`),To.forEach(t),Be=f(e),c=a(e,"P",{});var m=n(c);At=s(m,"To that end, the authors introduce "),I=a(m,"A",{href:!0,rel:!0});var So=n(I);pe=a(So,"EM",{});var xo=n(pe);Pt=s(xo,"Reward Reports for Reinforcement Learning"),xo.forEach(t),So.forEach(t),Lt=s(m," (the pithy naming is designed to mirror the popular papers "),de=a(m,"EM",{});var Mo=n(de);Tt=s(Mo,"Model Cards for Model Reporting"),Mo.forEach(t),St=s(m," and "),fe=a(m,"EM",{});var No=n(fe);xt=s(No,"Datasheets for Datasets"),No.forEach(t),Mt=s(m,`).
The goal is to propose a type of documentation focused on the `),ce=a(m,"STRONG",{});var Ho=n(ce);Nt=s(Ho,"human factors of reward"),Ho.forEach(t),Ht=s(m," and "),ue=a(m,"STRONG",{});var qo=n(ue);qt=s(qo,"time-varying feedback systems"),qo.forEach(t),Ct=s(m,"."),m.forEach(t),ze=f(e),v=a(e,"P",{});var re=n(v);Gt=s(re,"Building on the documentation frameworks for "),B=a(re,"A",{href:!0,rel:!0});var Co=n(B);It=s(Co,"model cards"),Co.forEach(t),Bt=s(re," and "),z=a(re,"A",{href:!0,rel:!0});var Go=n(z);zt=s(Go,"datasheets"),Go.forEach(t),Ot=s(re," proposed by Mitchell et al. and Gebru et al., we argue the need for Reward Reports for AI systems."),re.forEach(t),Oe=f(e),O=a(e,"P",{});var uo=n(O);me=a(uo,"STRONG",{});var Io=n(me);Dt=s(Io,"Reward Reports"),Io.forEach(t),Ft=s(uo," are living documents for proposed RL deployments that demarcate design choices."),uo.forEach(t),De=f(e),Y=a(e,"P",{});var Bo=n(Y);Wt=s(Bo,`However, many questions remain about the applicability of this framework to different RL applications, roadblocks to system interpretability,
and the resonances between deployed supervised machine learning systems and the sequential decision-making utilized in RL.`),Bo.forEach(t),Fe=f(e),Z=a(e,"P",{});var zo=n(Z);Jt=s(zo,`At a minimum, Reward Reports are an opportunity for RL practitioners to deliberate on these questions and begin the work of deciding how to resolve them in practice.
\u200B`),zo.forEach(t),We=f(e),y=a(e,"H2",{class:!0});var at=n(y);L=a(at,"A",{id:!0,class:!0,href:!0});var Oo=n(L);ve=a(Oo,"SPAN",{});var Do=n(ve);ke(D.$$.fragment,Do),Do.forEach(t),Oo.forEach(t),Ut=f(at),we=a(at,"SPAN",{});var Fo=n(we);jt=s(Fo,"Capturing temporal behavior with documentation"),Fo.forEach(t),at.forEach(t),Je=f(e),T=a(e,"P",{});var nt=n(T);Kt=s(nt,"The core piece specific to documentation designed for RL and feedback-driven ML systems is a "),ge=a(nt,"EM",{});var Wo=n(ge);Qt=s(Wo,"change-log"),Wo.forEach(t),Vt=s(nt,`. The change-log updates information
from the designer (changed training parameters, data, etc.) along with noticed changes from the user (harmful behavior, unexpected responses, etc.).`),nt.forEach(t),Ue=f(e),ee=a(e,"P",{});var Jo=n(ee);Xt=s(Jo,"The change log is accompanied by update triggers that encourage monitoring these effects."),Jo.forEach(t),je=f(e),_=a(e,"H2",{class:!0});var it=n(_);S=a(it,"A",{id:!0,class:!0,href:!0});var Uo=n(S);be=a(Uo,"SPAN",{});var jo=n(be);ke(F.$$.fragment,jo),jo.forEach(t),Uo.forEach(t),Yt=f(it),ye=a(it,"SPAN",{});var Ko=n(ye);Zt=s(Ko,"Contributing"),Ko.forEach(t),it.forEach(t),Ke=f(e),te=a(e,"P",{});var Qo=n(te);eo=s(Qo,`Some of the most impactful RL-driven systems are multi-stakeholder in nature and behind the closed doors of private corporations.
These corporations are largely without regulation, so the burden of documentation falls on the public.`),Qo.forEach(t),Qe=f(e),u=a(e,"P",{});var M=n(u);to=s(M,`If you are interested in contributing, we are building Reward Reports for popular machine learning systems on a public
record on `),W=a(M,"A",{href:!0,rel:!0});var Vo=n(W);oo=s(Vo,"GitHub"),Vo.forEach(t),ro=s(M,`.
\u200B
For further reading, you can visit the Reward Reports `),J=a(M,"A",{href:!0,rel:!0});var Xo=n(J);ao=s(Xo,"paper"),Xo.forEach(t),no=s(M,`
or look `),U=a(M,"A",{href:!0,rel:!0});var Yo=n(U);io=s(Yo,"an example report"),Yo.forEach(t),so=s(M,"."),M.forEach(t),Ve=f(e),R=a(e,"H2",{class:!0});var st=n(R);x=a(st,"A",{id:!0,class:!0,href:!0});var Zo=n(x);_e=a(Zo,"SPAN",{});var er=n(_e);ke(j.$$.fragment,er),er.forEach(t),Zo.forEach(t),lo=f(st),Re=a(st,"SPAN",{});var tr=n(Re);ho=s(tr,"Author"),tr.forEach(t),st.forEach(t),Xe=f(e),K=a(e,"P",{});var mo=n(K);po=s(mo,"This section was written by "),oe=a(mo,"A",{href:!0});var or=n(oe);fo=s(or,"Nathan Lambert"),or.forEach(t),mo.forEach(t),this.h()},h(){p(w,"name","hf:doc:metadata"),p(w,"content",JSON.stringify(pr)),p(E,"id","brief-introduction-to-rl-documentation"),p(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(E,"href","#brief-introduction-to-rl-documentation"),p(g,"class","relative group"),p(q,"href","https://huggingface.co/docs/hub/model-cards"),p(q,"rel","nofollow"),p(C,"href","https://huggingface.co/models"),p(C,"rel","nofollow"),p(P,"id","motivating-reward-reports"),p(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(P,"href","#motivating-reward-reports"),p(b,"class","relative group"),p(I,"href","https://www.notion.so/Brief-introduction-to-RL-documentation-b8cbda5a6f5242338e0756e6bef72af4"),p(I,"rel","nofollow"),p(B,"href","https://arxiv.org/abs/1810.03993"),p(B,"rel","nofollow"),p(z,"href","https://arxiv.org/abs/1803.09010"),p(z,"rel","nofollow"),p(L,"id","capturing-temporal-behavior-with-documentation"),p(L,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(L,"href","#capturing-temporal-behavior-with-documentation"),p(y,"class","relative group"),p(S,"id","contributing"),p(S,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(S,"href","#contributing"),p(_,"class","relative group"),p(W,"href","https://github.com/RewardReports/reward-reports"),p(W,"rel","nofollow"),p(J,"href","https://arxiv.org/abs/2204.10817"),p(J,"rel","nofollow"),p(U,"href","https://github.com/RewardReports/reward-reports/tree/main/examples"),p(U,"rel","nofollow"),p(x,"id","author"),p(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),p(x,"href","#author"),p(R,"class","relative group"),p(oe,"href","https://twitter.com/natolambert")},m(e,l){o(document.head,w),h(e,Se,l),h(e,g,l),o(g,E),o(E,ae),$e(N,ae,null),o(g,lt),o(g,ne),o(ne,ht),h(e,xe,l),h(e,H,l),o(H,pt),o(H,ie),o(ie,dt),h(e,Me,l),h(e,k,l),o(k,ft),o(k,se),o(se,ct),o(k,ut),h(e,Ne,l),h(e,Q,l),o(Q,mt),h(e,He,l),h(e,$,l),o($,vt),o($,q),o(q,wt),o($,gt),h(e,qe,l),h(e,A,l),o(A,bt),o(A,C),o(C,yt),o(A,_t),h(e,Ce,l),h(e,V,l),o(V,Rt),h(e,Ge,l),h(e,b,l),o(b,P),o(P,le),$e(G,le,null),o(b,Et),o(b,he),o(he,kt),h(e,Ie,l),h(e,X,l),o(X,$t),h(e,Be,l),h(e,c,l),o(c,At),o(c,I),o(I,pe),o(pe,Pt),o(c,Lt),o(c,de),o(de,Tt),o(c,St),o(c,fe),o(fe,xt),o(c,Mt),o(c,ce),o(ce,Nt),o(c,Ht),o(c,ue),o(ue,qt),o(c,Ct),h(e,ze,l),h(e,v,l),o(v,Gt),o(v,B),o(B,It),o(v,Bt),o(v,z),o(z,zt),o(v,Ot),h(e,Oe,l),h(e,O,l),o(O,me),o(me,Dt),o(O,Ft),h(e,De,l),h(e,Y,l),o(Y,Wt),h(e,Fe,l),h(e,Z,l),o(Z,Jt),h(e,We,l),h(e,y,l),o(y,L),o(L,ve),$e(D,ve,null),o(y,Ut),o(y,we),o(we,jt),h(e,Je,l),h(e,T,l),o(T,Kt),o(T,ge),o(ge,Qt),o(T,Vt),h(e,Ue,l),h(e,ee,l),o(ee,Xt),h(e,je,l),h(e,_,l),o(_,S),o(S,be),$e(F,be,null),o(_,Yt),o(_,ye),o(ye,Zt),h(e,Ke,l),h(e,te,l),o(te,eo),h(e,Qe,l),h(e,u,l),o(u,to),o(u,W),o(W,oo),o(u,ro),o(u,J),o(J,ao),o(u,no),o(u,U),o(U,io),o(u,so),h(e,Ve,l),h(e,R,l),o(R,x),o(x,_e),$e(j,_e,null),o(R,lo),o(R,Re),o(Re,ho),h(e,Xe,l),h(e,K,l),o(K,po),o(K,oe),o(oe,fo),Ye=!0},p:sr,i(e){Ye||(Ae(N.$$.fragment,e),Ae(G.$$.fragment,e),Ae(D.$$.fragment,e),Ae(F.$$.fragment,e),Ae(j.$$.fragment,e),Ye=!0)},o(e){Pe(N.$$.fragment,e),Pe(G.$$.fragment,e),Pe(D.$$.fragment,e),Pe(F.$$.fragment,e),Pe(j.$$.fragment,e),Ye=!1},d(e){t(w),e&&t(Se),e&&t(g),Le(N),e&&t(xe),e&&t(H),e&&t(Me),e&&t(k),e&&t(Ne),e&&t(Q),e&&t(He),e&&t($),e&&t(qe),e&&t(A),e&&t(Ce),e&&t(V),e&&t(Ge),e&&t(b),Le(G),e&&t(Ie),e&&t(X),e&&t(Be),e&&t(c),e&&t(ze),e&&t(v),e&&t(Oe),e&&t(O),e&&t(De),e&&t(Y),e&&t(Fe),e&&t(Z),e&&t(We),e&&t(y),Le(D),e&&t(Je),e&&t(T),e&&t(Ue),e&&t(ee),e&&t(je),e&&t(_),Le(F),e&&t(Ke),e&&t(te),e&&t(Qe),e&&t(u),e&&t(Ve),e&&t(R),Le(j),e&&t(Xe),e&&t(K)}}}const pr={local:"brief-introduction-to-rl-documentation",sections:[{local:"motivating-reward-reports",title:"Motivating Reward Reports"},{local:"capturing-temporal-behavior-with-documentation",title:"Capturing temporal behavior with documentation"},{local:"contributing",title:"Contributing"},{local:"author",title:"Author"}],title:"Brief introduction to RL documentation"};function dr(vo){return lr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ur extends rr{constructor(w){super();ar(this,w,dr,hr,nr,{})}}export{ur as default,pr as metadata};
