prioritized_memory.py

# Code from https://github.com/rlcode/per

import random
import numpy as np
from SumTree import SumTree

class Memory:  # stored as ( s, a, r, s_ ) in SumTree
    e = 0.01
    a = 0.6
    beta = 0.4
    beta_increment_per_sampling = 0.001

    def __init__(self, capacity):
        self.tree = SumTree(capacity)
        self.capacity = capacity

    def _get_priority(self, weight):
        return (np.abs(weight) + self.e) ** self.a

    def add_per(self, weight, sample):
        p = self._get_priority(weight)
        self.tree.add(p, sample)

    def add(self, weight, sample):
        p = weight + 0.0000001 if weight%10==0 else weight
        self.tree.add(p, sample)

    def sample(self, n):
        batch = []
        idxs = []
        segment = self.tree.total() / n
        priorities = []

        self.beta = np.min([1., self.beta + self.beta_increment_per_sampling])

        for i in range(n):
            a = segment * i
            b = segment * (i + 1)

            s = random.uniform(a, b)
            (idx, p, data) = self.tree.get(s)
            priorities.append(p)
            batch.append(data)
            idxs.append(idx)

        sampling_probabilities = priorities / self.tree.total()
        weights = np.power(self.tree.n_entries * sampling_probabilities, -self.beta)
        weights /= weights.max()


        return batch, idxs, weights, self.tree.total()

    def update(self, idx, weight):
        p = self._get_priority(weight)
        self.tree.update(idx, p)