13 self.
seen_size = 0 # total experiences
seen cumulatively
14 # declare what data keys to store
15 self.data_keys = ['states', 'actions',
'rewards', 'next_states', 'dones']
16 [Link]()
17
18 @lab_api
19 def reset(self):
20 '''Resets the memory. Also used to
initialize memory vars'''
21 for k in self.data_keys:
22 setattr(self, k, [])
23 self.cur_epi_data = {k: [] for k in
self.data_keys}
24 self.most_recent = (None,) *
len(self.data_keys)
25 [Link] = 0
Code 2.7: OnPolicyReplay: reset
Memory update The update function serves as an API
method to the memory. Adding an experience to memory is
mostly straightforward. The only tricky part is keeping track of
the episode boundaries. The steps in Code 2.8 can be broken
down as follows:
1. Add the experience to the current episode (line:14-15).
2. Check if the episode is finished (line:17). This is given by
the done variable which will be 1 if the episode is
finished, 0 otherwise.
3. If the episode is finished, add the entire set of
experiences for the episode to the main containers in the
memory (line:18-19).
WOW! eBook
[Link]
4. If the episode is finished, clear the current episode
dictionary (line:20) so that the memory is ready to store
the next episode.
5. If the desired number of episodes has been collected set
the train flag to 1 in the agent (line:23-24). This signals
that the agent should train this time step.
1 # slm_lab/agent/memory/[Link]
2
3 class OnPolicyReplay(Memory):
4 ...
5
6 @lab_api
7 def update(self, state, action, reward,
next_state, done):
8 '''Interface method to update memory'''
9 self.add_experience(state, action,
reward, next_state, done)
10
11 def add_experience(self, state, action,
reward, next_state, done):
12 '''Interface helper method for update()
to add experience to memory'''
13 self.most_recent = (state, action,
reward, next_state, done)
14 for idx, k in enumerate(self.data_keys):
15
self.cur_epi_data[k].append(self.most_re
cent[idx])
16 # If episode ended, add to memory and
clear cur_epi_data
17 if util.epi_done(done):
18 for k in self.data_keys:
19 getattr(self,
k).append(self.cur_epi_data[k])
WOW! eBook
[Link]
20 self.cur_epi_data = {k: [] for k in
self.data_keys}
21 # If agent has collected the desired
number of episodes, it is ready
↪ to train
22 # length is num of epis due to nested
structure
23 if len([Link]) ==
[Link].training_frequency:
24 [Link].to_train
= 1
25 # Track memory size and num experiences
26 [Link] += 1
27 self.seen_size += 1
Code 2.8: OnPolicyReplay: add experience
Memory sample sample in Code 2.9 simply returns all of the
completed episodes packaged into a batch dictionary (line:6).
Then it resets the memory (line:7) as the stored experiences will
no longer be valid to use once the agent has completed a
training step.
1 # slm_lab/agent/memory/[Link]
2
3 class OnPolicyReplay(Memory):
4
5 def sample(self):
6 batch = {k: getattr(self, k) for k in
self.data_keys}
7 [Link]()
8 return batch
Code 2.9: OnPolicyReplay: sample
WOW! eBook
[Link]
2.7 TRAINING A REINFORCE AGENT
Deep RL algorithms often have many parameters. For example,
the network type, architecture, activation functions, optimizer
and learning rate need to be specified. More advanced neural
network functionality can involve gradient clipping and learning
rate decay schedules. This is only the “deep” part! RL
algorithms also involve hyperparameter choices. For example,
the discount rate, γ and how frequently to train an agent. To
make this more manageable, all of the hyperparameter choices
are specified in a json file in SLM Lab known as a spec (stands
for specification). For more details on spec file see Chapter 11.
Code 2.10 contains an example spec file for REINFORCE. The
file is also available in SLM Lab at
slm_lab/spec/benchmark/reinforce/reinforce_cart
[Link].
1 #
slm_lab/spec/benchmark/reinforce/reinforce_cartpole.
json
2
3 {
4 "reinforce_cartpole": {
5 "agent": [{
6 "name": "Reinforce",
7 "algorithm": {
8 "name": "Reinforce",
9 "action_pdtype": "default",
10 "action_policy": "default",
11 "center_return": true,
12 "explore_var_spec": null,
13 "gamma": 0.99,
14 "entropy_coef_spec": {
WOW! eBook
[Link]
15 "name": "linear_decay",
16 "start_val": 0.01,
17 "end_val": 0.001,
18 "start_step": 0,
19 "end_step": 20000,
20 },
21 "training_frequency": 1
22 },
23 "memory": {
24 "name": "OnPolicyReplay"
25 },
26 "net": {
27 "type": "MLPNet",
28 "hid_layers": [64],
29 "hid_layers_activation": "selu",
30 "clip_grad_val": null,
31 "loss_spec": {
32 "name": "MSELoss"
33 },
34 "optim_spec": {
35 "name": "Adam",
36 "lr": 0.002
37 },
38 "lr_scheduler_spec": null
39 }
40 }],
41 "env": [{
42 "name": "CartPole-v0",
43 "max_t": null,
44 "max_frame": 100000,
45 }],
46 "body": {
47 "product": "outer",
48 "num": 1
49 },
50 "meta": {
51 "distributed": false,
52 "eval_frequency": 2000,
53 "max_session": 4,
WOW! eBook
[Link]