mhjensen
209 lines
16 KiB
HTML
209 lines
16 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/hplgit/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: Reinforcement Learning">
|
|
|
|
<title>Data Analysis and Machine Learning: Reinforcement Learning</title>
|
|
|
|
|
|
<style type="text/css">
|
|
/* bloodish style */
|
|
|
|
body {
|
|
font-family: Helvetica, Verdana, Arial, Sans-serif;
|
|
color: #404040;
|
|
background: #ffffff;
|
|
}
|
|
h1 { font-size: 1.8em; color: #8A0808; }
|
|
h2 { font-size: 1.6em; color: #8A0808; }
|
|
h3 { font-size: 1.4em; color: #8A0808; }
|
|
h4 { color: #8A0808; }
|
|
a { color: #8A0808; text-decoration:none; }
|
|
tt { font-family: "Courier New", Courier; }
|
|
/* pre style removed because it will interfer with pygments */
|
|
p { text-indent: 0px; }
|
|
hr { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
p.caption { width: 80%; font-style: normal; text-align: left; }
|
|
hr.figure { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
|
|
div { text-align: justify; text-justify: inter-word; }
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Reinforcement Learning: Overarching view', 2, None, '___sec0'),
|
|
('Code example', 2, None, '___sec1')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
<!-- ------------------- main content ---------------------- -->
|
|
|
|
|
|
|
|
<center><h1>Data Analysis and Machine Learning: Reinforcement Learning </h1></center> <!-- document title -->
|
|
|
|
<p>
|
|
<!-- author(s): Morten Hjorth-Jensen -->
|
|
|
|
<center>
|
|
<b>Morten Hjorth-Jensen</b> [1, 2]
|
|
</center>
|
|
|
|
<p>
|
|
<!-- institution(s) -->
|
|
|
|
<center>[1] <b>Department of Physics, University of Oslo</b></center>
|
|
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
|
<br>
|
|
<p>
|
|
<center><h4>Dec 26, 2018</h4></center> <!-- date -->
|
|
<br>
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec0">Reinforcement Learning: Overarching view </h2>
|
|
|
|
<p>
|
|
Reinforcement Learning (RL) is one of the most exciting fields of Machine Learning today, and also one
|
|
of the oldest. It has been around since the 1950s, producing many interesting applications over the years.
|
|
|
|
<p>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
|
|
<h2 id="___sec1">Code example </h2>
|
|
<p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #BA2121; font-style: italic">"""</span>
|
|
<span style="color: #BA2121; font-style: italic">A simple example for Reinforcement Learning using table lookup Q-learning method.</span>
|
|
<span style="color: #BA2121; font-style: italic">An agent "o" is on the left of a 1 dimensional world, the treasure is on the rightmost location.</span>
|
|
<span style="color: #BA2121; font-style: italic">Run this program and to see how the agent will improve its strategy of finding the treasure.</span>
|
|
<span style="color: #BA2121; font-style: italic">View more on my tutorial page: https://morvanzhou.github.io/tutorials/</span>
|
|
<span style="color: #BA2121; font-style: italic">"""</span>
|
|
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">time</span>
|
|
|
|
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">2</span>) <span style="color: #408080; font-style: italic"># reproducible</span>
|
|
|
|
|
|
N_STATES <span style="color: #666666">=</span> <span style="color: #666666">6</span> <span style="color: #408080; font-style: italic"># the length of the 1 dimensional world</span>
|
|
ACTIONS <span style="color: #666666">=</span> [<span style="color: #BA2121">'left'</span>, <span style="color: #BA2121">'right'</span>] <span style="color: #408080; font-style: italic"># available actions</span>
|
|
EPSILON <span style="color: #666666">=</span> <span style="color: #666666">0.9</span> <span style="color: #408080; font-style: italic"># greedy police</span>
|
|
ALPHA <span style="color: #666666">=</span> <span style="color: #666666">0.1</span> <span style="color: #408080; font-style: italic"># learning rate</span>
|
|
GAMMA <span style="color: #666666">=</span> <span style="color: #666666">0.9</span> <span style="color: #408080; font-style: italic"># discount factor</span>
|
|
MAX_EPISODES <span style="color: #666666">=</span> <span style="color: #666666">13</span> <span style="color: #408080; font-style: italic"># maximum episodes</span>
|
|
FRESH_TIME <span style="color: #666666">=</span> <span style="color: #666666">0.3</span> <span style="color: #408080; font-style: italic"># fresh time for one move</span>
|
|
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">build_q_table</span>(n_states, actions):
|
|
table <span style="color: #666666">=</span> pd<span style="color: #666666">.</span>DataFrame(
|
|
np<span style="color: #666666">.</span>zeros((n_states, <span style="color: #008000">len</span>(actions))), <span style="color: #408080; font-style: italic"># q_table initial values</span>
|
|
columns<span style="color: #666666">=</span>actions, <span style="color: #408080; font-style: italic"># actions's name</span>
|
|
)
|
|
<span style="color: #408080; font-style: italic"># print(table) # show table</span>
|
|
<span style="color: #008000; font-weight: bold">return</span> table
|
|
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">choose_action</span>(state, q_table):
|
|
<span style="color: #408080; font-style: italic"># This is how to choose an action</span>
|
|
state_actions <span style="color: #666666">=</span> q_table<span style="color: #666666">.</span>iloc[state, :]
|
|
<span style="color: #008000; font-weight: bold">if</span> (np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>uniform() <span style="color: #666666">></span> EPSILON) <span style="color: #AA22FF; font-weight: bold">or</span> ((state_actions <span style="color: #666666">==</span> <span style="color: #666666">0</span>)<span style="color: #666666">.</span>all()): <span style="color: #408080; font-style: italic"># act non-greedy or state-action have no value</span>
|
|
action_name <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(ACTIONS)
|
|
<span style="color: #008000; font-weight: bold">else</span>: <span style="color: #408080; font-style: italic"># act greedy</span>
|
|
action_name <span style="color: #666666">=</span> state_actions<span style="color: #666666">.</span>idxmax() <span style="color: #408080; font-style: italic"># replace argmax to idxmax as argmax means a different function in newer version of pandas</span>
|
|
<span style="color: #008000; font-weight: bold">return</span> action_name
|
|
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">get_env_feedback</span>(S, A):
|
|
<span style="color: #408080; font-style: italic"># This is how agent will interact with the environment</span>
|
|
<span style="color: #008000; font-weight: bold">if</span> A <span style="color: #666666">==</span> <span style="color: #BA2121">'right'</span>: <span style="color: #408080; font-style: italic"># move right</span>
|
|
<span style="color: #008000; font-weight: bold">if</span> S <span style="color: #666666">==</span> N_STATES <span style="color: #666666">-</span> <span style="color: #666666">2</span>: <span style="color: #408080; font-style: italic"># terminate</span>
|
|
S_ <span style="color: #666666">=</span> <span style="color: #BA2121">'terminal'</span>
|
|
R <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
|
<span style="color: #008000; font-weight: bold">else</span>:
|
|
S_ <span style="color: #666666">=</span> S <span style="color: #666666">+</span> <span style="color: #666666">1</span>
|
|
R <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
|
<span style="color: #008000; font-weight: bold">else</span>: <span style="color: #408080; font-style: italic"># move left</span>
|
|
R <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
|
<span style="color: #008000; font-weight: bold">if</span> S <span style="color: #666666">==</span> <span style="color: #666666">0</span>:
|
|
S_ <span style="color: #666666">=</span> S <span style="color: #408080; font-style: italic"># reach the wall</span>
|
|
<span style="color: #008000; font-weight: bold">else</span>:
|
|
S_ <span style="color: #666666">=</span> S <span style="color: #666666">-</span> <span style="color: #666666">1</span>
|
|
<span style="color: #008000; font-weight: bold">return</span> S_, R
|
|
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">update_env</span>(S, episode, step_counter):
|
|
<span style="color: #408080; font-style: italic"># This is how environment be updated</span>
|
|
env_list <span style="color: #666666">=</span> [<span style="color: #BA2121">'-'</span>]<span style="color: #666666">*</span>(N_STATES<span style="color: #666666">-1</span>) <span style="color: #666666">+</span> [<span style="color: #BA2121">'T'</span>] <span style="color: #408080; font-style: italic"># '---------T' our environment</span>
|
|
<span style="color: #008000; font-weight: bold">if</span> S <span style="color: #666666">==</span> <span style="color: #BA2121">'terminal'</span>:
|
|
interaction <span style="color: #666666">=</span> <span style="color: #BA2121">'Episode </span><span style="color: #BB6688; font-weight: bold">%s</span><span style="color: #BA2121">: total_steps = </span><span style="color: #BB6688; font-weight: bold">%s</span><span style="color: #BA2121">'</span> <span style="color: #666666">%</span> (episode<span style="color: #666666">+1</span>, step_counter)
|
|
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'</span><span style="color: #BB6622; font-weight: bold">\r</span><span style="color: #BA2121">{}'</span><span style="color: #666666">.</span>format(interaction), end<span style="color: #666666">=</span><span style="color: #BA2121">''</span>)
|
|
time<span style="color: #666666">.</span>sleep(<span style="color: #666666">2</span>)
|
|
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'</span><span style="color: #BB6622; font-weight: bold">\r</span><span style="color: #BA2121"> '</span>, end<span style="color: #666666">=</span><span style="color: #BA2121">''</span>)
|
|
<span style="color: #008000; font-weight: bold">else</span>:
|
|
env_list[S] <span style="color: #666666">=</span> <span style="color: #BA2121">'o'</span>
|
|
interaction <span style="color: #666666">=</span> <span style="color: #BA2121">''</span><span style="color: #666666">.</span>join(env_list)
|
|
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'</span><span style="color: #BB6622; font-weight: bold">\r</span><span style="color: #BA2121">{}'</span><span style="color: #666666">.</span>format(interaction), end<span style="color: #666666">=</span><span style="color: #BA2121">''</span>)
|
|
time<span style="color: #666666">.</span>sleep(FRESH_TIME)
|
|
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">rl</span>():
|
|
<span style="color: #408080; font-style: italic"># main part of RL loop</span>
|
|
q_table <span style="color: #666666">=</span> build_q_table(N_STATES, ACTIONS)
|
|
<span style="color: #008000; font-weight: bold">for</span> episode <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(MAX_EPISODES):
|
|
step_counter <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
|
S <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
|
is_terminated <span style="color: #666666">=</span> <span style="color: #008000">False</span>
|
|
update_env(S, episode, step_counter)
|
|
<span style="color: #008000; font-weight: bold">while</span> <span style="color: #AA22FF; font-weight: bold">not</span> is_terminated:
|
|
|
|
A <span style="color: #666666">=</span> choose_action(S, q_table)
|
|
S_, R <span style="color: #666666">=</span> get_env_feedback(S, A) <span style="color: #408080; font-style: italic"># take action & get next state and reward</span>
|
|
q_predict <span style="color: #666666">=</span> q_table<span style="color: #666666">.</span>loc[S, A]
|
|
<span style="color: #008000; font-weight: bold">if</span> S_ <span style="color: #666666">!=</span> <span style="color: #BA2121">'terminal'</span>:
|
|
q_target <span style="color: #666666">=</span> R <span style="color: #666666">+</span> GAMMA <span style="color: #666666">*</span> q_table<span style="color: #666666">.</span>iloc[S_, :]<span style="color: #666666">.</span>max() <span style="color: #408080; font-style: italic"># next state is not terminal</span>
|
|
<span style="color: #008000; font-weight: bold">else</span>:
|
|
q_target <span style="color: #666666">=</span> R <span style="color: #408080; font-style: italic"># next state is terminal</span>
|
|
is_terminated <span style="color: #666666">=</span> <span style="color: #008000">True</span> <span style="color: #408080; font-style: italic"># terminate this episode</span>
|
|
|
|
q_table<span style="color: #666666">.</span>loc[S, A] <span style="color: #666666">+=</span> ALPHA <span style="color: #666666">*</span> (q_target <span style="color: #666666">-</span> q_predict) <span style="color: #408080; font-style: italic"># update</span>
|
|
S <span style="color: #666666">=</span> S_ <span style="color: #408080; font-style: italic"># move to next state</span>
|
|
|
|
update_env(S, episode, step_counter<span style="color: #666666">+1</span>)
|
|
step_counter <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
|
<span style="color: #008000; font-weight: bold">return</span> q_table
|
|
|
|
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #19177C">__name__</span> <span style="color: #666666">==</span> <span style="color: #BA2121">"__main__"</span>:
|
|
q_table <span style="color: #666666">=</span> rl()
|
|
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">'</span><span style="color: #BB6622; font-weight: bold">\r\n</span><span style="color: #BA2121">Q-table:</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">'</span>)
|
|
<span style="color: #008000; font-weight: bold">print</span>(q_table)
|
|
</pre></div>
|
|
<p>
|
|
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright --> © 1999-2018, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
|
</center>
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|