Feedforward multilayer neural network. More...

#include <Net.h>

Inheritance diagram for OpenANN::Net:

Public Member Functions
	Net ()
	Create feedforward neural network. More...

virtual	~Net ()

Architecture Definition
These functions must be called to define the architecture of the network.
Net &	inputLayer (int dim1, int dim2=1, int dim3=1)
	Add an input layer. More...

Net &	alphaBetaFilterLayer (double deltaT, double stdDev=0.05)
	Add a alpha-beta filter layer. More...

Net &	fullyConnectedLayer (int units, ActivationFunction act, double stdDev=0.05, bool bias=true)
	Add a fully connected hidden layer. More...

Net &	restrictedBoltzmannMachineLayer (int H, int cdN=1, double stdDev=0.01, bool backprop=true)
	Add a layer that contains an RBM. More...

Net &	sparseAutoEncoderLayer (int H, double beta, double rho, ActivationFunction act)
	Add a sparse auto-encoder. More...

Net &	compressedLayer (int units, int params, ActivationFunction act, const std::string &compression, double stdDev=0.05, bool bias=true)
	Add a compressed fully connected hidden layer. More...

Net &	extremeLayer (int units, ActivationFunction act, double stdDev=5.0, bool bias=true)
	Add a fully connected hidden layer with fixed weights. More...

Net &	intrinsicPlasticityLayer (double targetMean, double stdDev=1.0)
	Add an intrinsic plasticity layer that is able to learn the parameters of a logistic activation function. More...

Net &	convolutionalLayer (int featureMaps, int kernelRows, int kernelCols, ActivationFunction act, double stdDev=0.05, bool bias=true)
	Add a convolutional layer. More...

Net &	subsamplingLayer (int kernelRows, int kernelCols, ActivationFunction act, double stdDev=0.05, bool bias=true)
	Add a subsampling layer. More...

Net &	maxPoolingLayer (int kernelRows, int kernelCols)
	Add a max-pooling layer. More...

Net &	localReponseNormalizationLayer (double k, int n, double alpha, double beta)
	Add a local response normalization layer. More...

Net &	dropoutLayer (double dropoutProbability)
	Add a dropout layer. More...

Net &	outputLayer (int units, ActivationFunction act, double stdDev=0.05, bool bias=true)
	Add a fully connected output layer. More...

Net &	compressedOutputLayer (int units, int params, ActivationFunction act, const std::string &compression, double stdDev=0.05, bool bias=true)
	Add a compressed output layer. More...

Net &	addLayer (Layer *layer)
	Add a new layer to this deep neural network. More...

Net &	addOutputLayer (Layer *layer)
	Add a new output layer to this deep neural network. More...

Access Internal Structure
unsigned int	numberOflayers ()
	Request number of layers. More...

Layer &	getLayer (unsigned int l)
	Access layer. More...

OutputInfo	getOutputInfo (unsigned int l)
	Request information about output of a given layer. More...

DataSet *	propagateDataSet (DataSet &dataSet, int l)
	Propagate data set through the first l layers. More...

Persistence
void	save (const std::string &fileName)
	Save network. More...

void	save (std::ostream &stream)
	Save network. More...

void	load (const std::string &fileName)
	Load network from file. More...

void	load (std::istream &stream)
	Load network from stream. More...

Optimization Contol
Net &	setRegularization (double l1Penalty=0.0, double l2Penalty=0.0, double maxSquaredWeightNorm=0.0)

Net &	setErrorFunction (ErrorFunction errorFunction)
	Set the error function. More...

Net &	useDropout (bool activate=true)
	Toggle dropout. More...

Inherited Functions
virtual Eigen::VectorXd	operator() (const Eigen::VectorXd &x)
	Make a prediction. More...

virtual Eigen::MatrixXd	operator() (const Eigen::MatrixXd &X)
	Make predictions. More...

virtual unsigned int	dimension ()
	Request the number of optimizable parameters. More...

virtual const Eigen::VectorXd &	currentParameters ()
	Request the current parameters. More...

virtual void	setParameters (const Eigen::VectorXd &parameters)
	Set new parameters. More...

virtual bool	providesInitialization ()
	Check if the object knows how to initialize its parameters. More...

virtual void	initialize ()
	Initialize the optimizable parameters. More...

virtual unsigned int	examples ()
	Request number of training examples. More...

virtual double	error (unsigned int n)

virtual double	error ()
	Compute error on training set. More...

virtual bool	providesGradient ()
	Check if the object provides a gradient of the error function with respect to its parameters. More...

virtual Eigen::VectorXd	gradient (unsigned int n)

virtual Eigen::VectorXd	gradient ()
	Compute gradient of the error function with respect to the parameters. More...

virtual void	errorGradient (int n, double &value, Eigen::VectorXd &grad)
	Calculates the function value and gradient of a training example. More...

virtual void	errorGradient (double &value, Eigen::VectorXd &grad)
	Calculates the function value and gradient of all training examples. More...

virtual void	errorGradient (std::vector< int >::const_iterator startN, std::vector< int >::const_iterator endN, double &value, Eigen::VectorXd &grad)
	Calculates the accumulated gradient and error of given training examples. More...

virtual void	finishedIteration ()
	This callback is called after each optimization algorithm iteration. More...

Public Member Functions inherited from OpenANN::Learner
	Learner ()

virtual	~Learner ()

virtual Learner &	trainingSet (Eigen::MatrixXd &input, Eigen::MatrixXd &output)
	Set training set. More...

virtual Learner &	trainingSet (DataSet &trainingSet)
	Set training set. More...

virtual Learner &	removeTrainingSet ()
	Remove the training set from the learner. More...

virtual Learner &	validationSet (Eigen::MatrixXd &input, Eigen::MatrixXd &output)
	Set validation set. More...

virtual Learner &	validationSet (DataSet &validationSet)
	Set validation set. More...

virtual Learner &	removeValidationSet ()
	Remove the validation set from the learner. More...

Public Member Functions inherited from OpenANN::Optimizable
virtual	~Optimizable ()

virtual double	error (unsigned n)
	Compute error of a given training example. More...

virtual Eigen::VectorXd	gradient (unsigned n)
	Compute gradient of a given training example. More...

virtual Eigen::VectorXd	error (std::vector< int >::const_iterator startN, std::vector< int >::const_iterator endN)
	Calculates the errors of given training examples. More...

virtual Eigen::VectorXd	gradient (std::vector< int >::const_iterator startN, std::vector< int >::const_iterator endN)
	Calculates the accumulated gradient of given training examples. More...

Protected Member Functions
void	initializeNetwork ()

void	forwardPropagate (double *error)

void	backpropagate ()

Protected Attributes
std::vector< OutputInfo >	infos

std::vector< Layer * >	layers

std::vector< double * >	parameters

std::vector< double * >	derivatives

Regularization	regularization

ErrorFunction	errorFunction

bool	dropout

bool	initialized

int	P

int	L

Eigen::VectorXd	parameterVector

Eigen::VectorXd	tempGradient

Eigen::MatrixXd	tempInput

Eigen::MatrixXd	tempOutput

Eigen::MatrixXd	tempError

std::stringstream	architecture

Protected Attributes inherited from OpenANN::Learner
DataSet *	trainSet

DataSet *	validSet

bool	deleteTrainSet

bool	deleteValidSet

int	N

Detailed Description

Feedforward multilayer neural network.

You can specify many different types of layers and choose the architecture almost arbitrary.

Constructor & Destructor Documentation

OpenANN::Net::Net ( )

Create feedforward neural network.

virtual OpenANN::Net::~Net ( )

virtual

Member Function Documentation

Net& OpenANN::Net::addLayer ( Layer * layer)

Add a new layer to this deep neural network.

Never free/delete the added layer outside of this class. Its cleaned up by Net's destructor automatically.

Parameters

layer pointer to an instance that implements the Layer interface

Returns: this for chaining

Net& OpenANN::Net::addOutputLayer ( Layer * layer)

Add a new output layer to this deep neural network.

Never free/delete the added layer outside of this class. Its cleaned up by Net's destructor automatically.

Parameters

layer pointer to an instance that implements the Layer interface

Returns: this for chaining

Net& OpenANN::Net::alphaBetaFilterLayer	(	double	deltaT,
		double	stdDev = `0.05`
	)

Add a alpha-beta filter layer.

Parameters

deltaT	temporal difference between two steps
stdDev	standard deviation of the Gaussian distributed initial weights

Returns: this for chaining

void OpenANN::Net::backpropagate ( )

protected

Net& OpenANN::Net::compressedLayer	(	int	units,
		int	params,
		ActivationFunction	act,
		const std::string &	compression,
		double	stdDev = `0.05`,
		bool	bias = `true`
	)

Add a compressed fully connected hidden layer.

Parameters

units	number of nodes (neurons)
params	number of parameters to represent the incoming weights of a neuron in this layer
act	activation function
compression	type of compression matrix, possible values are dct, gaussian, sparse, average, edge
stdDev	standard deviation of the Gaussian distributed initial weights
bias	add bias term

Returns: this for chaining

Net& OpenANN::Net::compressedOutputLayer	(	int	units,
		int	params,
		ActivationFunction	act,
		const std::string &	compression,
		double	stdDev = `0.05`,
		bool	bias = `true`
	)

Add a compressed output layer.

This will initialize the network.

Parameters

units	number of nodes (neurons)
params	number of parameters to represent the incoming weights of a neuron in this layer
act	activation function
compression	type of compression matrix, possible values are dct, gaussian, sparse, average, edge
stdDev	standard deviation of the Gaussian distributed initial weights
bias	add bias term

Returns: this for chaining

Net& OpenANN::Net::convolutionalLayer	(	int	featureMaps,
		int	kernelRows,
		int	kernelCols,
		ActivationFunction	act,
		double	stdDev = `0.05`,
		bool	bias = `true`
	)

Add a convolutional layer.

Parameters

featureMaps	number of feature maps
kernelRows	number of kernel rows
kernelCols	number of kernel columns
act	activation function
stdDev	standard deviation of the Gaussian distributed initial weights
bias	add bias term

Returns: this for chaining

virtual const Eigen::VectorXd& OpenANN::Net::currentParameters ( )

virtual

Request the current parameters.

Returns: current parameters

Implements OpenANN::Optimizable.

virtual unsigned int OpenANN::Net::dimension ( )

virtual

Request the number of optimizable parameters.

Returns: number of optimizable parameters

Implements OpenANN::Optimizable.

Net& OpenANN::Net::dropoutLayer ( double dropoutProbability)

Add a dropout layer.

Parameters

dropoutProbability probability of suppression during training

virtual double OpenANN::Net::error ( unsigned int n)

virtual

virtual double OpenANN::Net::error ( )

virtual

Compute error on training set.

Returns: current error on training set or objective function value

Implements OpenANN::Optimizable.

virtual void OpenANN::Net::errorGradient	(	int	n,
		double &	value,
		Eigen::VectorXd &	grad
	)

virtual

Calculates the function value and gradient of a training example.

Parameters

n	index of training example
value	function value
grad	gradient of the function, lenght must be dimension()

Reimplemented from OpenANN::Optimizable.

virtual void OpenANN::Net::errorGradient	(	double &	value,
		Eigen::VectorXd &	grad
	)

virtual

Calculates the function value and gradient of all training examples.

Parameters

value	function value
grad	gradient of the function, lenght must be dimension()

Reimplemented from OpenANN::Optimizable.

virtual void OpenANN::Net::errorGradient	(	std::vector< int >::const_iterator	startN,
		std::vector< int >::const_iterator	endN,
		double &	value,
		Eigen::VectorXd &	grad
	)

virtual

Calculates the accumulated gradient and error of given training examples.

Parameters

startN	iterator over index vector
endN	iterator over index vector
value	function value
grad	gradient of the function, lenght must be dimension()

Returns: each row contains the gradient for one training example

Reimplemented from OpenANN::Optimizable.

virtual unsigned int OpenANN::Net::examples ( )

virtual

Request number of training examples.

Returns: number of training examples

Reimplemented from OpenANN::Optimizable.

Net& OpenANN::Net::extremeLayer	(	int	units,
		ActivationFunction	act,
		double	stdDev = `5.0`,
		bool	bias = `true`
	)

Add a fully connected hidden layer with fixed weights.

Parameters

units	number of nodes (neurons)
act	activation function
stdDev	standard deviation of the Gaussian distributed initial weights
bias	add bias term

Returns: this for chaining

virtual void OpenANN::Net::finishedIteration ( )

virtual

This callback is called after each optimization algorithm iteration.

Reimplemented from OpenANN::Optimizable.

void OpenANN::Net::forwardPropagate ( double * error)

protected

Net& OpenANN::Net::fullyConnectedLayer	(	int	units,
		ActivationFunction	act,
		double	stdDev = `0.05`,
		bool	bias = `true`
	)

Add a fully connected hidden layer.

Parameters

units	number of nodes (neurons)
act	activation function
stdDev	standard deviation of the Gaussian distributed initial weights
bias	add bias term

Returns: this for chaining

Layer& OpenANN::Net::getLayer ( unsigned int l)

Access layer.

Parameters

l	index of layer

Returns: l-th layer

OutputInfo OpenANN::Net::getOutputInfo ( unsigned int l)

Request information about output of a given layer.

Parameters

l	index of the layer

Returns: output information

virtual Eigen::VectorXd OpenANN::Net::gradient ( unsigned int n)

virtual

virtual Eigen::VectorXd OpenANN::Net::gradient ( )

virtual

Compute gradient of the error function with respect to the parameters.

Returns: gradient

Implements OpenANN::Optimizable.

virtual void OpenANN::Net::initialize ( )

virtual

Initialize the optimizable parameters.

Implements OpenANN::Optimizable.

void OpenANN::Net::initializeNetwork ( )

protected

Net& OpenANN::Net::inputLayer	(	int	dim1,
		int	dim2 = `1`,
		int	dim3 = `1`
	)

Add an input layer.

Parameters

dim1	first dimension of the input, e. g. number of color channels of an image
dim2	second dimension, e. g. number of rows of an image
dim3	third dimension, e. g. number of columns of an image

Returns: this for chaining

Net& OpenANN::Net::intrinsicPlasticityLayer	(	double	targetMean,
		double	stdDev = `1.0`
	)

Add an intrinsic plasticity layer that is able to learn the parameters of a logistic activation function.

Parameters

targetMean	desired mean of the output distribution, must be within [0, 1], e.g. 0.2
stdDev	standard deviation of the Gaussian distributed initial biases

Returns: this for chaining

void OpenANN::Net::load ( const std::string & fileName)

Load network from file.

Parameters

fileName name of the file

void OpenANN::Net::load ( std::istream & stream)

Load network from stream.

Note: Note that we cannot ensure that the network will be reconstructed correctly in case it contains either an extreme layer, compressed layer or a compressed output layer because these types of layers internally generate random matrices that will not be stored. To ensure that these matrices will contain the same values, you will have to set the seed for the random number generator, e.g.
OpenANN::RandomNumberGenerator().seed(0);

Net net;

// Construct and train network

net.save("mlnn.net");

OpenANN::RandomNumberGenerator().seed(0);

Net net2;

net2.load("mlnn.net");

Parameters

stream input stream

Net& OpenANN::Net::localReponseNormalizationLayer	(	double	k,
		int	n,
		double	alpha,
		double	beta
	)

Add a local response normalization layer.

$y^i_{rc} = x^i_{rc} / \left( k + \alpha \sum_{j=max(0, i-n/2)}^{min(N-1, i+n/2)} x^j_{rc} \right)^{\beta}$

Parameters

k	hyperparameter, k >= 1, e.g. 1 or 2
n	number of adjacent feature maps
alpha	controls strength of inhibition, alpha > 0, e.g. 1e-4
beta	controls strength of inhibition, beta > 0, e.g. 0.75

Returns: this for chaining

Net& OpenANN::Net::maxPoolingLayer	(	int	kernelRows,
		int	kernelCols
	)

Add a max-pooling layer.

Parameters

kernelRows	number of kernel rows
kernelCols	number of kernel columns

Returns: this for chaining

unsigned int OpenANN::Net::numberOflayers ( )

Request number of layers.

Returns: number of layers in this neural network

virtual Eigen::VectorXd OpenANN::Net::operator() ( const Eigen::VectorXd & x)

virtual

Make a prediction.

Parameters

x	Input vector.

Returns: Prediction.

Implements OpenANN::Learner.

virtual Eigen::MatrixXd OpenANN::Net::operator() ( const Eigen::MatrixXd & X)

virtual

Make predictions.

Parameters

X	Each row represents an input vector.

Returns: Each row represents a prediction.

Implements OpenANN::Learner.

Net& OpenANN::Net::outputLayer	(	int	units,
		ActivationFunction	act,
		double	stdDev = `0.05`,
		bool	bias = `true`
	)

Add a fully connected output layer.

This will initialize the network.

Parameters

units	number of nodes (neurons)
act	activation function
stdDev	standard deviation of the Gaussian distributed initial weights
bias	add bias term

Returns: this for chaining

DataSet* OpenANN::Net::propagateDataSet	(	DataSet &	dataSet,
		int	l
	)

Propagate data set through the first l layers.

Parameters

dataSet	original dataset
l	index of the layer

Returns: transformed data set (has to be deleted manually)

virtual bool OpenANN::Net::providesGradient ( )

virtual

Check if the object provides a gradient of the error function with respect to its parameters.

Returns: does the optimizable provide a gradient?

Implements OpenANN::Optimizable.

virtual bool OpenANN::Net::providesInitialization ( )

virtual

Check if the object knows how to initialize its parameters.

Returns: does the optimizable object provide a parameter initialization?

Implements OpenANN::Optimizable.

Net& OpenANN::Net::restrictedBoltzmannMachineLayer	(	int	H,
		int	cdN = `1`,
		double	stdDev = `0.01`,
		bool	backprop = `true`
	)

Add a layer that contains an RBM.

Parameters

H	number of nodes (neurons)
cdN	number of gibbs sampling steps for pretraining
stdDev	standard deviation of the Gaussian distributed initial weights
backprop	finetune weights with backpropagation

Returns: this for chaining

void OpenANN::Net::save ( const std::string & fileName)

Save network.

Parameters

fileName name of the file

void OpenANN::Net::save ( std::ostream & stream)

Save network.

Parameters

stream output stream

Net& OpenANN::Net::setErrorFunction ( ErrorFunction errorFunction)

Set the error function.

Parameters

errorFunction error function

Returns: this for chaining

virtual void OpenANN::Net::setParameters ( const Eigen::VectorXd & parameters)

virtual

Set new parameters.

Parameters

parameters new parameters

Implements OpenANN::Optimizable.

Net& OpenANN::Net::setRegularization	(	double	l1Penalty = `0.0`,
		double	l2Penalty = `0.0`,
		double	maxSquaredWeightNorm = `0.0`
	)

Net& OpenANN::Net::sparseAutoEncoderLayer	(	int	H,
		double	beta,
		double	rho,
		ActivationFunction	act
	)

Add a sparse auto-encoder.

Parameters

H	number of outputs
beta	weight of sparsity
rho	desired mean activation of hidden neurons
act	activation function of the hidden layer

Net& OpenANN::Net::subsamplingLayer	(	int	kernelRows,
		int	kernelCols,
		ActivationFunction	act,
		double	stdDev = `0.05`,
		bool	bias = `true`
	)

Add a subsampling layer.

Parameters

kernelRows	number of kernel rows
kernelCols	number of kernel columns
act	activation function
stdDev	standard deviation of the Gaussian distributed initial weights
bias	add bias term

Returns: this for chaining

Net& OpenANN::Net::useDropout ( bool activate = true)

Toggle dropout.

Parameters

activate turn dropout on or off

Returns: this for chaining

Member Data Documentation

std::stringstream OpenANN::Net::architecture

protected

std::vector<double*> OpenANN::Net::derivatives

protected

bool OpenANN::Net::dropout

protected

ErrorFunction OpenANN::Net::errorFunction

protected

std::vector<OutputInfo> OpenANN::Net::infos

protected

bool OpenANN::Net::initialized

protected

int OpenANN::Net::L

protected

std::vector<Layer*> OpenANN::Net::layers

protected

int OpenANN::Net::P

protected

std::vector<double*> OpenANN::Net::parameters

protected

Eigen::VectorXd OpenANN::Net::parameterVector

protected

Regularization OpenANN::Net::regularization

protected

Eigen::MatrixXd OpenANN::Net::tempError

protected

Eigen::VectorXd OpenANN::Net::tempGradient

protected

Eigen::MatrixXd OpenANN::Net::tempInput

protected

Eigen::MatrixXd OpenANN::Net::tempOutput

protected

The documentation for this class was generated from the following file:

OpenANN/Net.h

Public Member Functions

Protected Member Functions

Protected Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation