import numpy as np
[docs]class Activation:
[docs] def __call__(self, incoming):
raise NotImplementedError
[docs] def delta(self, incoming, outgoing, above):
"""
Compute the derivative of the cost with respect to the input of this
activation function. Outgoing is what this function returned in the
forward pass and above is the derivative of the cost with respect to
the outgoing activation.
"""
raise NotImplementedError
[docs]class Identity(Activation):
[docs] def __call__(self, incoming):
return incoming
[docs] def delta(self, incoming, outgoing, above):
delta = np.ones(incoming.shape).astype(float)
return delta * above
[docs]class Sigmoid(Activation):
[docs] def __call__(self, incoming):
return 1 / (1 + np.exp(-incoming))
[docs] def delta(self, incoming, outgoing, above):
delta = outgoing * (1 - outgoing)
return delta * above
[docs]class Relu(Activation):
[docs] def __call__(self, incoming):
return np.maximum(incoming, 0)
[docs] def delta(self, incoming, outgoing, above):
delta = np.greater(incoming, 0).astype(float)
return delta * above
[docs]class Softmax(Activation):
[docs] def __call__(self, incoming):
# The constant doesn't change the expression but prevents overflows.
constant = np.max(incoming)
exps = np.exp(incoming - constant)
return exps / exps.sum()
[docs] def delta(self, incoming, outgoing, above):
delta = outgoing * above
sum_ = delta.sum(axis=delta.ndim - 1, keepdims=True)
delta -= outgoing * sum_
return delta
[docs]class SparseField(Activation):
def __init__(self, inhibition=0.05, leaking=0.0):
self.inhibition = inhibition
self.leaking = leaking
[docs] def __call__(self, incoming):
count = len(incoming)
length = int(np.sqrt(count))
assert length ** 2 == count, 'layer size must be a square'
field = incoming.copy().reshape((length, length))
radius = int(np.sqrt(self.inhibition * count)) // 2
assert radius, 'no inhibition due to small factor'
outgoing = np.zeros(field.shape)
while True:
x, y = np.unravel_index(field.argmax(), field.shape)
if field[x, y] <= 0:
break
outgoing[x, y] = 1
surrounding = np.s_[
max(x - radius, 0):min(x + radius + 1, length),
max(y - radius, 0):min(y + radius + 1, length)]
field[surrounding] = 0
assert field[x, y] == 0
outgoing = outgoing.reshape(count)
outgoing = np.maximum(outgoing, self.leaking * incoming)
return outgoing
[docs] def delta(self, incoming, outgoing, above):
delta = np.greater(outgoing, 0).astype(float)
return delta * above
[docs]class SparseRange(Activation):
"""
E%-Max Winner-Take-All.
Binary activation. First, the activation function is applied. Then all
neurons within the specified range below the strongest neuron are set to
one. All others are set to zero. The gradient is the one of the activation
function for active neurons and zero otherwise.
See: A Second Function of Gamma Frequency Oscillations: An E%-Max
Winner-Take-All Mechanism Selects Which Cells Fire. (2009)
"""
def __init__(self, range_=0.3, function=Sigmoid()):
assert 0 < range_ < 1
self._range = range_
self._function = function
[docs] def __call__(self, incoming):
incoming = self._function(incoming)
threshold = self._threshold(incoming)
active = (incoming >= threshold)
outgoing = np.zeros(incoming.shape)
outgoing[active] = 1
# width = active.sum() * 80 / 1000
# print('|', '#' * width, ' ' * (80 - width), '|')
return outgoing
[docs] def delta(self, incoming, outgoing, above):
# return self._function.delta(incoming, outgoing, outgoing * above)
return outgoing * self._function.delta(incoming, outgoing, above)
def _threshold(self, incoming):
min_, max_ = incoming.min(), incoming.max()
threshold = min_ + (max_ - min_) * (1 - self._range)
return threshold