CscNeuralNetwork.h

Go to the documentation of this file.

/*
 * CscNeuralNetwork.h
 *
 *  Created on: 24 novembre 2023
 *      Author: Max Lhermitte
 */
 
#ifndef CSCNEURALNETWORK_H_
#define CSCNEURALNETWORK_H_
 
#include "CscCommon.h"
 
#include <random>
 
using std::mt19937;
 
namespace conscience_core::ai::nn {
 
// Do not modify the enum values! This could prevent saved neural networks from loading correctly
enum CscNeuralNetworkType
{
    DenseNeuralNetwork = 0,
    RecurrentNeuralNetwork = 1
};
 
// Do not modify the enum values! This could prevent saved neural networks from loading correctly
enum CscActivationFunction
{
    Linear = 0,
    Sigmoid = 1, Tanh = 2,
    Elu = 3, Relu = 4, Relu6, Selu, LeakyRelu,
    Acosh, Asin, Asinh, Atan, Atanh, Sinh,
    Ceil,
    Erf,
    Exp, Expm1,
    Floor,
    Log, Log1p,
    Rint,
    Sign,
    Softsign,
    Softplus,
    Sqrt
};
 
// Do not modify the enum values! This could prevent saved neural networks from loading correctly
enum CscLossFunction
{
    MeanSquaredError = 0,
    MeanAbsoluteError = 1,
    HingeLoss = 2
};
 
enum  CscInitializationWeightsStrategy {
    WeightsLoadFromModel,
    He, // Used with Relu activation function or its variantes
    GlorotUniform, // Ideal for linear or piecewise linear activation functions (e.g., ReLU and its variants in certain contexts)
    GlorotNormal, // Ideal for symmetric activation functions and/or those whose output is centered around zero, like the hyperbolic tangent (tanh) or the sigmoid function.
    LeCunNormal // Suggested for piecewise linear activation functions like the symmetric sigmoid activation function
};
 
enum  CscInitializationBiasStrategy {
    BiasesLoadFromModel,
    Zero,
    LittleValue
};
 
enum CscNormalizationType {
    RangeMinus1To1,
    Range0To1
};
 
enum CscOptimizer {
    GradientDescent,
    Adagrad,
    Adadelta,
    Adam
};
 
class CSC_DLL_IMPORTEXPORT CscNeuralNetwork {
public:
    struct CSC_DLL_IMPORTEXPORT NeuralNetworkShape {
    private:
        const int nbInputs;
        int nbHiddenLayers;
        vector<int> nbNeuronsPerHiddenLayer;
        const int nbOutputs;
        
    public:
        NeuralNetworkShape(int nbInputsNN, int nbHiddenLayersNN, vector<int> nbNeuronsPerHiddenLayerNN, int nbOutputsNN);
        NeuralNetworkShape(int nbInputsNN, int nbHiddenLayersNN, int nbNeuronsPerHiddenLayerNN, int nbOutputsNN);
        const int getNbInputs() const;
        const int getNbHiddenLayers() const;
        vector<int> getNbNeuronsPerHiddenLayerVector() const;
        const int getNbOutputs() const;
        NeuralNetworkShape* getDuplicata();
        
        void setNbHiddenLayers(int newNbHiddenLayers);
        void setNbNeuronsPerHiddenLayer(int newNbNeuronsPerHiddenLayer);
    };
    
    struct CSC_DLL_IMPORTEXPORT DataSet {
    private:
        const int dataSizeDataSet;
        int nbDataPerMiniBatchSet;
        float*** inputsDataSet;
        float*** targetsDataSet;
    public:
        DataSet(const int dataSize, int nbDataPerMiniBatch, float*** inputs, float*** targets);
        const int getDataSize() const;
        const int getNbDataPerMiniBatchSize() const;
        float*** getInputs() const;
        float*** getTargets() const;
    };
    
    struct CSC_DLL_IMPORTEXPORT ResultTraining {
    private :
        const int nbEpochPerformed;
        const int bestEpochOnValidationData;
        const float meanOnTrainingData;
        const float meanOnValidationData;
        
    public:
        ResultTraining(const int nbEpochResult, const int bestEpochOnValidationDataResult, const float meanOnTrainingDataResult, const float meanOnValidationDataResult);
        const int getNbEpochPerformed() const;
        const int getBestEpochOnValidationData() const;
        const float getMeanOnTrainingData() const;
        const float getMeanOnValidationData() const;
        ResultTraining* getDuplicata();
    };
    
    struct CSC_DLL_IMPORTEXPORT TrainingHyperparameters {
    private :
        CscInitializationWeightsStrategy initWeightStrategy;
        CscInitializationBiasStrategy initBiasStrategy;
        CscLossFunction lossFunction;
        CscOptimizer optimizer; 
        float learningRate; 
        float rho = 0.95f; 
        float beta1 = 0.9f; 
        float beta2 = 0.999f; 
        float epsilon = 0.00000001f; 
        int nbDataPerMiniBatch;
        bool earlyStoppingActivation;
        int patience; 
    public :
        TrainingHyperparameters(CscInitializationWeightsStrategy initWeightStrategyInit, CscInitializationBiasStrategy initBiasStrategyInit, CscLossFunction lossFunctionInit, CscOptimizer optimizers, float learningRateInit, int nbDataPerMiniBatchInit, bool earlyStoppingInit, int patienceInit);
        void setInitializationWeightsStrategy(CscInitializationWeightsStrategy initWeightStrategyUpdate);
        void setInitializationBiasStrategy(CscInitializationBiasStrategy initBiasStrategyUpdate);
        void setLossFunction(CscLossFunction lossFunctionUpdate);
        void setOptimizer(CscOptimizer optimizerUpdate);
        void setLearningRate(float learningRateUpdate);
        void setRho(float rho);
        void setBeta1(float beta1Update);
        void setBeta2(float beta2Update);
        void setEpsilon(float epsilonUpdate);
        void setNbDataPerMiniBatch(int nbDataPerMiniBatchUpdate);
        void setEarlyStoppingActivation(bool earlyStoppingUpdate);
        void setPatience(int patienceUpdate);
        
        const CscInitializationWeightsStrategy getInitializationWeightsStrategy() const;
        const CscInitializationBiasStrategy getInitializationBiasStrategy() const;
        const CscLossFunction getLossFunction() const;
        const CscOptimizer getOptimizer() const;
        const float getLearningRate() const;
        const float getRho() const;
        const float getBeta1() const;
        const float getBeta2() const;
        const float getEpsilon() const;
        const int getNbDataPerMiniBatch() const;
        const bool getEarlyStoppingActivation() const;
        const int getPatience() const;
        TrainingHyperparameters* getDuplicata();
    };
    
    struct CSC_DLL_IMPORTEXPORT NeuralNetworkHyperparameters {
    private :
        NeuralNetworkShape* neuralNetworkShape;
        CscActivationFunction hiddenLayersFunctionActivation;
        CscActivationFunction outputLayerFunctionActivation;
        
    public :
        NeuralNetworkHyperparameters(NeuralNetworkShape* nnShape, CscActivationFunction hiddenLayersFunctActivation, CscActivationFunction outputLayerFunctActivation);
        
        ~NeuralNetworkHyperparameters();
        
        NeuralNetworkShape* getNeuralNetworkShape() const;
        CscActivationFunction getHiddenLayersFunctionActivation() const;
        CscActivationFunction getOutputLayerFunctionActivation() const;
        
        NeuralNetworkHyperparameters* getDuplicata();
        
        void setHiddenLayersFunctionActivation(CscActivationFunction newHiddenLayersFunctionActivation);
        void setOutputLayerFunctionActivation(CscActivationFunction newOutputLayerFunctionActivation);
        
    };
    
public:
    CscNeuralNetwork(CscNeuralNetworkType neuralNetworkType, NeuralNetworkHyperparameters* neuralNetworkHyperparametersInit, TrainingHyperparameters* trainingHyperparametersInit);
    CscNeuralNetwork(CscNeuralNetworkType neuralNetworkType,  NeuralNetworkHyperparameters* neuralNetworkHyperparametersInit, TrainingHyperparameters* trainingHyperparametersInit, float** weightsLoadedFromModel, float** biasesLoadedFromModel);
    virtual ~CscNeuralNetwork();
    
    CscNeuralNetworkType getNeuralNetworkType() const;
    CscNeuralNetwork::NeuralNetworkHyperparameters* getNeuralNetworkHyperparameters() const;
    CscNeuralNetwork::TrainingHyperparameters* getTrainingHyperparameters() const;
    
    virtual ResultTraining trainNeuralNetwork(const int nbEpochs, const int nbMiniBatchs, DataSet* trainingDataSet, DataSet* validationDataSet, const bool debugMode = false) = 0;
    static float*** adaptDataForMiniBatches(float** dataToTransform, const int minibatchSize, const int dataSetSize);
    
    // normalization methods
    static void normalizeDataAutoRange(float** data, size_t size, int nbInputsOrOutputs, CscNormalizationType normalizationType);
    static void normalizeDataCustomRange0to1(float** data, size_t size, int nbInputsOrOutputs, float* minValue, float* maxValue);
    static void normalizeDataCustomRangeMinus1to1(float** data, size_t size, int nbInputsOrOutputs, float* minValue, float* maxValue);
    static void normalizeData(float** data, size_t size, int rowData, float newMin, float newMax, float minValue, float maxValue);
 
    static void deleteAllDataMiniBatches(float ***dataMiniBatches, const int nbMiniBatches, const int nbDataPerMinibatch);
    
    // Methods to initialate weights
    static void initializeWeights(float* weightsToInitialize, int size,  CscInitializationWeightsStrategy strategy, int fanIn, int fanOut = 0, float* weightsLoaded = nullptr);
    
    static void initializeBias(float* biasToInitialize, int size,  CscInitializationBiasStrategy strategy, float* biasLoaded = nullptr);
    
protected:
    // Attributes
    CscNeuralNetworkType neuralNetworkType;
    TrainingHyperparameters* trainingHyperparameters = nullptr;
    NeuralNetworkHyperparameters* neuralNetworkHyperparameters = nullptr;
    unique_ptr<CscLogger> logger;
    
    // Methods
    virtual void createNeuralNetwork(CscActivationFunction hiddenLayersFunctionActivation, CscActivationFunction outputLayerFunctionActivation, CscLossFunction lossFunction,  CscInitializationWeightsStrategy initilizationStrategy,  CscInitializationBiasStrategy initializationBiasStrategy, bool withCostFunction) = 0;
    
    static float calculateStddevHe(int fanIn);
    static float calculateLimitGlorot(int fanIn, int fanOut);
    static float calculateStddevGlorot(int fanIn, int fanOut);
    static float calculateStddevLeCun(int fanIn);
    static void initNormal(float* weights, int size, mt19937& gen, double stddev);
    static void initUniform(float* weights, int size, mt19937& gen, float limit);
    
    void shuffleMinibatchData(float*** minibatchDataInputs, float*** minibatchDataTargets, const int numberOfBatches, const int minibatchSize);
};
}
 
#endif /* CSCNEURALNETWORK_H_ */