# -*- coding: utf-8 -*-
"""activation-functions-for-deep-learning.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1VgnoWZiglcLr_3JI2obyb73Nb3j8ioVy

# Most popular activation functions for deep learning

Source code for creating the figure for each activation function. 
More detail on this page: [Most popular activation functions for deep learning](https://lucidar.me/en/neural-networks/most-popular-activation-functions-for-deep-learning/).
"""

import numpy as np
import math
import matplotlib.pyplot as plt

#plt.style.use('classic')
import matplotlib.pyplot as plt
plt.rcParams.update({'font.size': 16})

"""## Linear activation function"""

def plot (x,y,dfdx, title):
  # Plot function
  fig, ax = plt.subplots()
  f = ax.plot (x, y, label=title, linewidth=3)
  df = ax.plot (x, dfdx, '--', label='Derivative - df/dt', linewidth=2.5)

  
  fig = plt.gcf()
  plt.grid()
  # Set figure size in inches
  fig.set_size_inches(16, 9)
  leg = ax.legend(loc='upper left');

  plt.title(title)
  plt.xlabel('x')
  plt.ylabel('y')
  plt.figure(facecolor="white")

  ax.axhline(y=0, color='k')
  ax.axvline(x=0, color='k')

# Set x data
x = np.linspace(-6,6,100)

# Linear activation function
def linear_function(x):
  return x

# Derivative of the linear activation function
def linear_derivative(x):
  return [1] * len(x)

plot (x,linear_function(x), linear_derivative(x), 'Linear activation function')

"""## Sigmoid activation function"""

# Set x data
x = np.linspace(-8,8,100)

# Sigmoid activation function
def sigmoid_function(x):
  return 1/(1+np.exp(-x))

# Derivative of the sigmoid activation function
def sigmoid_derivative(x):
  return np.exp(-x) / (1+ np.exp(-x))**2
  
plot (x,sigmoid_function(x), sigmoid_derivative(x), 'Sigmoid activation function')

"""## Hyperbolic tangent activation function"""

# Set x data
x = np.linspace(-6,6,100)

# Tanh activation function
def tanh_function(x):
  return np.tanh(x)

# Derivative of the tanh activation function
def tanh_derivative(x):
  return 1 - np.tanh(x)**2
  
plot (x,tanh_function(x), tanh_derivative(x), 'Tanh activation function')

"""# ReLU activation function"""

# Set x data
x = np.linspace(-2,2,200)

# ReLU activation function
def ReLU_function(x):
  return np.where(x <= 0, 0, x)

# Derivative of the ReLU activation function
def ReLU_derivative(x):
  return np.where(x <= 0, 0, 1)
  
plot (x, ReLU_function(x), ReLU_derivative(x), 'ReLU activation function')

"""# Leaky ReLU"""

# Set x data
x = np.linspace(-5,2,200)

# Leaky ReLU activation function
def leakyReLU_function(x):
  return np.where(x <= 0, 0.01*x, x)

# Derivative of the leaky ReLU activation function
def leakyReLU_derivative(x):
  return np.where(x <= 0, 0.01, 1)
  
plot (x, leakyReLU_function(x), leakyReLU_derivative(x), 'Leaky ReLU activation function')

"""# Parameterised ReLU"""

# Set x data
x = np.linspace(-5,2,200)

# Parameterised ReLU activation function
def parameterised_ReLU_function(x,a):
  return np.where(x <= 0, a*x, x)

# Derivative of the parameterised ReLU activation function
def parameterised_ReLU_derivative(x,a):
  return np.where(x <= 0, a, 1)
  
plot (x, parameterised_ReLU_function(x, 0.2), parameterised_ReLU_derivative(x, 0.2), 'Parameterised ReLU activation function (a=0.2)')

"""# Exponential Linear Unit (ELU)"""

# Set x data
x = np.linspace(-5,2,200)

# ELU activation function
def ELU_function(x,a):
  return np.where(x <= 0, a*(np.exp(x) - 1), x)

# Derivative of the ELU activation function
def ELU_derivative(x,a):
  return np.where(x <= 0, a*np.exp(x), 1)
  
plot (x, ELU_function(x, 1), ELU_derivative(x, 1), 'ELU activation function (a=1)')