From 284d09a44617e6664b966dd4ec52db2395459daf Mon Sep 17 00:00:00 2001 From: Syntriax Date: Thu, 6 Jan 2022 14:43:03 +0300 Subject: [PATCH] Initial Commit --- .gitignore | 398 ++++++++++++++++++ .vscode/launch.json | 26 ++ .vscode/tasks.json | 42 ++ NeuralNetwork2021.csproj | 8 + Program.cs | 48 +++ Syntriax/NeuralSystem/InputNeuron.cs | 9 + Syntriax/NeuralSystem/Layer/HiddenLayer.cs | 18 + Syntriax/NeuralSystem/Layer/InputLayer.cs | 15 + Syntriax/NeuralSystem/Layer/Layer.cs | 25 ++ Syntriax/NeuralSystem/Layer/OutputLayer.cs | 15 + Syntriax/NeuralSystem/Misc/Data.cs | 92 ++++ Syntriax/NeuralSystem/Misc/DataNew.cs | 83 ++++ Syntriax/NeuralSystem/Misc/DataTest.cs | 163 +++++++ Syntriax/NeuralSystem/NeuralNetwork.cs | 39 ++ .../NeuralSystem/NeuralNetworkExtensions.cs | 184 ++++++++ .../Neuron/DropoutNeuronDecorator.cs | 23 + Syntriax/NeuralSystem/Neuron/INeuron.cs | 14 + Syntriax/NeuralSystem/Neuron/Neuron.cs | 49 +++ .../NeuralSystem/Neuron/NeuronDecorator.cs | 36 ++ .../NeuralSystem/NeuronActivations/Default.cs | 9 + .../NeuronActivations/INeuronActivation.cs | 8 + .../NeuralSystem/NeuronActivations/Relu.cs | 19 + .../NeuralSystem/NeuronActivations/Sigmoid.cs | 11 + .../NeuronActivations/StaticActivation.cs | 11 + .../NeuralSystem/NeuronActivations/TanH.cs | 11 + .../Synapse/DropoutSynapseDecorator.cs | 27 ++ Syntriax/NeuralSystem/Synapse/ISynapse.cs | 9 + .../Synapse/MomentumSynapseDecorator.cs | 27 ++ Syntriax/NeuralSystem/Synapse/Synapse.cs | 25 ++ .../NeuralSystem/Synapse/SynapseDecorator.cs | 16 + 30 files changed, 1460 insertions(+) create mode 100644 .gitignore create mode 100644 .vscode/launch.json create mode 100644 .vscode/tasks.json create mode 100644 NeuralNetwork2021.csproj create mode 100644 Program.cs create mode 100644 Syntriax/NeuralSystem/InputNeuron.cs create mode 100644 Syntriax/NeuralSystem/Layer/HiddenLayer.cs create mode 100644 Syntriax/NeuralSystem/Layer/InputLayer.cs create mode 100644 Syntriax/NeuralSystem/Layer/Layer.cs create mode 100644 Syntriax/NeuralSystem/Layer/OutputLayer.cs create mode 100644 Syntriax/NeuralSystem/Misc/Data.cs create mode 100644 Syntriax/NeuralSystem/Misc/DataNew.cs create mode 100644 Syntriax/NeuralSystem/Misc/DataTest.cs create mode 100644 Syntriax/NeuralSystem/NeuralNetwork.cs create mode 100644 Syntriax/NeuralSystem/NeuralNetworkExtensions.cs create mode 100644 Syntriax/NeuralSystem/Neuron/DropoutNeuronDecorator.cs create mode 100644 Syntriax/NeuralSystem/Neuron/INeuron.cs create mode 100644 Syntriax/NeuralSystem/Neuron/Neuron.cs create mode 100644 Syntriax/NeuralSystem/Neuron/NeuronDecorator.cs create mode 100644 Syntriax/NeuralSystem/NeuronActivations/Default.cs create mode 100644 Syntriax/NeuralSystem/NeuronActivations/INeuronActivation.cs create mode 100644 Syntriax/NeuralSystem/NeuronActivations/Relu.cs create mode 100644 Syntriax/NeuralSystem/NeuronActivations/Sigmoid.cs create mode 100644 Syntriax/NeuralSystem/NeuronActivations/StaticActivation.cs create mode 100644 Syntriax/NeuralSystem/NeuronActivations/TanH.cs create mode 100644 Syntriax/NeuralSystem/Synapse/DropoutSynapseDecorator.cs create mode 100644 Syntriax/NeuralSystem/Synapse/ISynapse.cs create mode 100644 Syntriax/NeuralSystem/Synapse/MomentumSynapseDecorator.cs create mode 100644 Syntriax/NeuralSystem/Synapse/Synapse.cs create mode 100644 Syntriax/NeuralSystem/Synapse/SynapseDecorator.cs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..426d76d --- /dev/null +++ b/.gitignore @@ -0,0 +1,398 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.tlog +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files +*.ncb +*.aps + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp + +# JetBrains Rider +*.sln.iml diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..9ee3acd --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + "version": "0.2.0", + "configurations": [ + { + // Use IntelliSense to find out which attributes exist for C# debugging + // Use hover for the description of the existing attributes + // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md + "name": ".NET Core Launch (console)", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + // If you have changed target frameworks, make sure to update the program path. + "program": "${workspaceFolder}/bin/Debug/net5.0/NeuralNetwork2021.dll", + "args": [], + "cwd": "${workspaceFolder}", + // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console + "console": "internalConsole", + "stopAtEntry": false + }, + { + "name": ".NET Core Attach", + "type": "coreclr", + "request": "attach" + } + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..5936938 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,42 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "build", + "command": "dotnet", + "type": "process", + "args": [ + "build", + "${workspaceFolder}/NeuralNetwork2021.csproj", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "publish", + "command": "dotnet", + "type": "process", + "args": [ + "publish", + "${workspaceFolder}/NeuralNetwork2021.csproj", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary" + ], + "problemMatcher": "$msCompile" + }, + { + "label": "watch", + "command": "dotnet", + "type": "process", + "args": [ + "watch", + "run", + "${workspaceFolder}/NeuralNetwork2021.csproj", + "/property:GenerateFullPaths=true", + "/consoleloggerparameters:NoSummary" + ], + "problemMatcher": "$msCompile" + } + ] +} \ No newline at end of file diff --git a/NeuralNetwork2021.csproj b/NeuralNetwork2021.csproj new file mode 100644 index 0000000..2082704 --- /dev/null +++ b/NeuralNetwork2021.csproj @@ -0,0 +1,8 @@ + + + + Exe + net5.0 + + + diff --git a/Program.cs b/Program.cs new file mode 100644 index 0000000..d8b7f43 --- /dev/null +++ b/Program.cs @@ -0,0 +1,48 @@ +using System; +using System.Linq; +using Syntriax.NeuralNetwork; +using Syntriax.NeuralNetwork.Misc; +using Syntriax.NeuralNetwork.NeuronActivations; + +namespace NeuralNetwork2021 +{ + class Program + { + static void Main(string[] args) + { + const int epochCount = 5000; + const int epochPrintInterval = 1; + + const int dataSeed = 10; + const int weightSeed = 0; + const int dropoutSeed = 0; + DropoutNeuronDecorator.Random = new Random(dropoutSeed); + + double learningRate = 0.001; + + Data data = new Data(DataTest.LoadData().ToList(), 4, seed: dataSeed); + + NeuralNetwork neuralNetwork = new NeuralNetwork + ( + data.InputCount, + new int[] { 10 }, + data.OutputCount + ); + + foreach (LayerBase layer in neuralNetwork.GetLayerList()) + layer.SetActivation(StaticActivation.Instance); + // neuralNetwork.outputLayer.SetActivation(StaticActivation.Instance); + + neuralNetwork.Randomize(weightSeed); + + for (int k = 0; k < epochCount; k++) + { + if (k % epochPrintInterval == 0) + Console.WriteLine($"Epoch: {k}\tHata: { neuralNetwork.GetTotalError(data) }"); + + neuralNetwork.Train(data, learningRate); + } + Console.WriteLine($"Hata: { neuralNetwork.GetTotalError(data) }"); + } + } +} diff --git a/Syntriax/NeuralSystem/InputNeuron.cs b/Syntriax/NeuralSystem/InputNeuron.cs new file mode 100644 index 0000000..f09a266 --- /dev/null +++ b/Syntriax/NeuralSystem/InputNeuron.cs @@ -0,0 +1,9 @@ +namespace Syntriax.NeuralNetwork +{ + public class InputNeuron : Neuron + { + public double Value = 0.0; + + public override double Output => Value; + } +} diff --git a/Syntriax/NeuralSystem/Layer/HiddenLayer.cs b/Syntriax/NeuralSystem/Layer/HiddenLayer.cs new file mode 100644 index 0000000..9b16a2f --- /dev/null +++ b/Syntriax/NeuralSystem/Layer/HiddenLayer.cs @@ -0,0 +1,18 @@ +namespace Syntriax.NeuralNetwork +{ + public class HiddenLayer : LayerBase + { + public HiddenLayer(int neuronCount, LayerBase from = null) : base(neuronCount, from) { } + + protected override void SetLayer(int neuronCount, LayerBase from) + { + neurons = new INeuron[neuronCount]; + + for (int i = 0; i < neuronCount; i++) + { + neurons[i] = new Neuron(from == null ? null : from.neurons); + // neurons[i] = new DropoutNeuronDecorator(neurons[i], 0.2); // TODO + } + } + } +} diff --git a/Syntriax/NeuralSystem/Layer/InputLayer.cs b/Syntriax/NeuralSystem/Layer/InputLayer.cs new file mode 100644 index 0000000..e93e9a8 --- /dev/null +++ b/Syntriax/NeuralSystem/Layer/InputLayer.cs @@ -0,0 +1,15 @@ +namespace Syntriax.NeuralNetwork +{ + public class InputLayer : LayerBase + { + public InputLayer(int neuronCount, LayerBase from = null) : base(neuronCount, from) { } + + protected override void SetLayer(int neuronCount, LayerBase from) + { + neurons = new Neuron[neuronCount]; + + for (int i = 0; i < neuronCount; i++) + neurons[i] = new InputNeuron(); + } + } +} diff --git a/Syntriax/NeuralSystem/Layer/Layer.cs b/Syntriax/NeuralSystem/Layer/Layer.cs new file mode 100644 index 0000000..a8de58a --- /dev/null +++ b/Syntriax/NeuralSystem/Layer/Layer.cs @@ -0,0 +1,25 @@ +using Syntriax.NeuralNetwork.NeuronActivations; + +namespace Syntriax.NeuralNetwork +{ + public abstract class LayerBase + { + public INeuron[] neurons { get; protected set; } = null; + + public LayerBase(int neuronCount, LayerBase from = null) => SetLayer(neuronCount, from); + + protected abstract void SetLayer(int neuronCount, LayerBase from); + + public void FireLayer() + { + foreach (INeuron neuron in neurons) + neuron.Calculate(); + } + + public void SetActivation(INeuronActivation neuronActivation) + { + foreach (INeuron neuron in neurons) + neuron.NeuronActivation = neuronActivation; + } + } +} diff --git a/Syntriax/NeuralSystem/Layer/OutputLayer.cs b/Syntriax/NeuralSystem/Layer/OutputLayer.cs new file mode 100644 index 0000000..ccf5fb0 --- /dev/null +++ b/Syntriax/NeuralSystem/Layer/OutputLayer.cs @@ -0,0 +1,15 @@ +namespace Syntriax.NeuralNetwork +{ + public class OutputLayer : HiddenLayer + { + public OutputLayer(int neuronCount, LayerBase from = null) : base(neuronCount, from) { } + + protected override void SetLayer(int neuronCount, LayerBase from) + { + neurons = new INeuron[neuronCount]; + + for (int i = 0; i < neuronCount; i++) + neurons[i] = new Neuron(from.neurons); + } + } +} diff --git a/Syntriax/NeuralSystem/Misc/Data.cs b/Syntriax/NeuralSystem/Misc/Data.cs new file mode 100644 index 0000000..521f8e4 --- /dev/null +++ b/Syntriax/NeuralSystem/Misc/Data.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; + +namespace Syntriax.NeuralNetwork.Misc +{ + public class Data + { + public int InputCount => trainInput[0].Length; + public int OutputCount => trainOutput[0].Length; + public List trainInput { get; private set; } = null; + public List trainOutput { get; private set; } = null; + public List testInput { get; private set; } = null; + public List testOutput { get; private set; } = null; + + public Data(List data, int inputCount, double trainRatio = 0.2, int? seed = null) + { + int indexToSwap = 0; + int count = data.Count; + int testCount = (int)(count * trainRatio); + Random random = new Random(seed ?? 0); + + double[] inputArray = null; + double[] outputArray = null; + + trainInput = new List(count - testCount); + trainOutput = new List(count - testCount); + testInput = new List(testCount); + testOutput = new List(testCount); + + for (int i = 0; i < count; i++) + { + indexToSwap = random.Next(i, count); + (data[i], data[indexToSwap]) = (data[indexToSwap], data[i]); + } + + for (int i = 0; i < count; i++) + { + (inputArray, outputArray) = SplitData(data[i], inputCount); + if (i < testCount) + { + testInput.Add(inputArray); + testOutput.Add(outputArray); + } + else + { + trainInput.Add(inputArray); + trainOutput.Add(outputArray); + } + } + } + public Data(List train, List test, int inputCount, double trainRatio = 0.2) + { + double[] inputArray = null; + double[] outputArray = null; + + trainInput = new List(train.Count); + trainOutput = new List(train.Count); + testInput = new List(test.Count); + testOutput = new List(test.Count); + + for (int i = 0; i < train.Count; i++) + { + (inputArray, outputArray) = SplitData(train[i], inputCount); + trainInput.Add(inputArray); + trainOutput.Add(outputArray); + } + + for (int i = 0; i < test.Count; i++) + { + (inputArray, outputArray) = SplitData(test[i], inputCount); + testInput.Add(inputArray); + testOutput.Add(outputArray); + } + } + + private (double[] inputArray, double[] outputArray) SplitData(double[] array, int inputCount) + { + int outputCount = array.Length - inputCount + 1; + int i = 0; + double[] input = new double[inputCount]; + double[] output = new double[outputCount]; + + for (i = 0; i < array.Length; i++) + if (i >= inputCount) + output[i - inputCount] = array[i]; + else + input[i] = array[i]; + + return (input, output); + } + } +} diff --git a/Syntriax/NeuralSystem/Misc/DataNew.cs b/Syntriax/NeuralSystem/Misc/DataNew.cs new file mode 100644 index 0000000..afb7470 --- /dev/null +++ b/Syntriax/NeuralSystem/Misc/DataNew.cs @@ -0,0 +1,83 @@ +// using System; + +// namespace Syntriax.NeuralNetwork.Misc +// { +// public class DataNew +// { +// public int InputCount => trainInput.GetLength(0); +// public int OutputCount => trainOutput.GetLength(0); +// public double[,] trainInput { get; private set; } = null; +// public double[,] trainOutput { get; private set; } = null; +// public double[,] testInput { get; private set; } = null; +// public double[,] testOutput { get; private set; } = null; + +// public DataNew(double[,] data, int inputCount, double testRatio = 0.2, int? seed = null) +// { +// int attributeCount = data.GetLength(1); +// int dataCount = data.GetLength(0); +// int testCount = (int)(dataCount * testRatio); +// int trainCount = dataCount - testCount; + +// double[,] trainData = new double[trainCount, attributeCount]; +// double[,] testData = new double[testCount, attributeCount]; + +// if (seed != null) +// Shuffle(data, attributeCount, new Random(seed.Value)); + +// int i = 0; +// int a = 0; +// for (i = 0; i < dataCount; i++) +// for (a = 0; a < attributeCount; a++) + + +// } + +// private void Shuffle(double[,] data, int attributeCount, Random random) +// { +// int indexToSwap = 0; +// int length = data.Length; +// int i = 0; +// int a = 0; + +// for (i = 0; i < length; i++) +// { +// indexToSwap = random.Next(i, length); +// for (a = 0; a < attributeCount; a++) +// (data[i, a], data[indexToSwap, a]) = (data[indexToSwap, a], data[i, a]); +// } +// } + +// public DataNew(double[,] train, double[,] test, int inputCount) +// { +// int attributeCount = train.GetLength(1); +// int attributeInputCount = inputCount; +// int attributeOutputCount = attributeCount - attributeInputCount; + +// int trainCount = train.GetLength(0); +// int testCount = test.GetLength(0); + +// trainInput = new double[train.GetLength(0), attributeInputCount]; +// trainOutput = new double[train.GetLength(0), attributeOutputCount]; +// testInput = new double[test.GetLength(0), attributeInputCount]; +// testOutput = new double[test.GetLength(0), attributeOutputCount]; + +// SplitIntoAttributeArrays(train, trainInput, trainOutput, trainCount, attributeInputCount, attributeOutputCount); +// SplitIntoAttributeArrays(test, testInput, testOutput, testCount, attributeInputCount, attributeOutputCount); +// } + +// private void SplitIntoAttributeArrays(double[,] source, double[,] inputDestination, double[,] outputDestination, +// int count, int inputCount, int outputCount) +// { +// int i = 0; +// int j = 0; + +// for (i = 0; i < count; i++) +// { +// for (j = 0; j < inputCount; j++) +// inputDestination[i, j] = source[i, j]; +// for (j = 0; j < outputCount; j++) +// outputDestination[i, j] = source[i, j + inputCount]; +// } +// } +// } +// } diff --git a/Syntriax/NeuralSystem/Misc/DataTest.cs b/Syntriax/NeuralSystem/Misc/DataTest.cs new file mode 100644 index 0000000..a4552ed --- /dev/null +++ b/Syntriax/NeuralSystem/Misc/DataTest.cs @@ -0,0 +1,163 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace Syntriax.NeuralNetwork.Misc +{ + public class DataTest + { + public static double[][] LoadData() => new double[][] + { + new double[] {5.1,3.5,1.4,0.2,0,0,1}, + new double[] {4.9,3.0,1.4,0.2,0,0,1}, + new double[] {4.7,3.2,1.3,0.2,0,0,1}, + new double[] {4.6,3.1,1.5,0.2,0,0,1}, + new double[] {5.0,3.6,1.4,0.2,0,0,1}, + new double[] {5.4,3.9,1.7,0.4,0,0,1}, + new double[] {4.6,3.4,1.4,0.3,0,0,1}, + new double[] {5.0,3.4,1.5,0.2,0,0,1}, + new double[] {4.4,2.9,1.4,0.2,0,0,1}, + new double[] {4.9,3.1,1.5,0.1,0,0,1}, + new double[] {5.4,3.7,1.5,0.2,0,0,1}, + new double[] {4.8,3.4,1.6,0.2,0,0,1}, + new double[] {4.8,3.0,1.4,0.1,0,0,1}, + new double[] {4.3,3.0,1.1,0.1,0,0,1}, + new double[] {5.8,4.0,1.2,0.2,0,0,1}, + new double[] {5.7,4.4,1.5,0.4,0,0,1}, + new double[] {5.4,3.9,1.3,0.4,0,0,1}, + new double[] {5.1,3.5,1.4,0.3,0,0,1}, + new double[] {5.7,3.8,1.7,0.3,0,0,1}, + new double[] {5.1,3.8,1.5,0.3,0,0,1}, + new double[] {5.4,3.4,1.7,0.2,0,0,1}, + new double[] {5.1,3.7,1.5,0.4,0,0,1}, + new double[] {4.6,3.6,1.0,0.2,0,0,1}, + new double[] {5.1,3.3,1.7,0.5,0,0,1}, + new double[] {4.8,3.4,1.9,0.2,0,0,1}, + new double[] {5.0,3.0,1.6,0.2,0,0,1}, + new double[] {5.0,3.4,1.6,0.4,0,0,1}, + new double[] {5.2,3.5,1.5,0.2,0,0,1}, + new double[] {5.2,3.4,1.4,0.2,0,0,1}, + new double[] {4.7,3.2,1.6,0.2,0,0,1}, + new double[] {4.8,3.1,1.6,0.2,0,0,1}, + new double[] {5.4,3.4,1.5,0.4,0,0,1}, + new double[] {5.2,4.1,1.5,0.1,0,0,1}, + new double[] {5.5,4.2,1.4,0.2,0,0,1}, + new double[] {4.9,3.1,1.5,0.1,0,0,1}, + new double[] {5.0,3.2,1.2,0.2,0,0,1}, + new double[] {5.5,3.5,1.3,0.2,0,0,1}, + new double[] {4.9,3.1,1.5,0.1,0,0,1}, + new double[] {4.4,3.0,1.3,0.2,0,0,1}, + new double[] {5.1,3.4,1.5,0.2,0,0,1}, + new double[] {5.0,3.5,1.3,0.3,0,0,1}, + new double[] {4.5,2.3,1.3,0.3,0,0,1}, + new double[] {4.4,3.2,1.3,0.2,0,0,1}, + new double[] {5.0,3.5,1.6,0.6,0,0,1}, + new double[] {5.1,3.8,1.9,0.4,0,0,1}, + new double[] {4.8,3.0,1.4,0.3,0,0,1}, + new double[] {5.1,3.8,1.6,0.2,0,0,1}, + new double[] {4.6,3.2,1.4,0.2,0,0,1}, + new double[] {5.3,3.7,1.5,0.2,0,0,1}, + new double[] {5.0,3.3,1.4,0.2,0,0,1}, + new double[] {7.0,3.2,4.7,1.4,0,1,0}, + new double[] {6.4,3.2,4.5,1.5,0,1,0}, + new double[] {6.9,3.1,4.9,1.5,0,1,0}, + new double[] {5.5,2.3,4.0,1.3,0,1,0}, + new double[] {6.5,2.8,4.6,1.5,0,1,0}, + new double[] {5.7,2.8,4.5,1.3,0,1,0}, + new double[] {6.3,3.3,4.7,1.6,0,1,0}, + new double[] {4.9,2.4,3.3,1.0,0,1,0}, + new double[] {6.6,2.9,4.6,1.3,0,1,0}, + new double[] {5.2,2.7,3.9,1.4,0,1,0}, + new double[] {5.0,2.0,3.5,1.0,0,1,0}, + new double[] {5.9,3.0,4.2,1.5,0,1,0}, + new double[] {6.0,2.2,4.0,1.0,0,1,0}, + new double[] {6.1,2.9,4.7,1.4,0,1,0}, + new double[] {5.6,2.9,3.6,1.3,0,1,0}, + new double[] {6.7,3.1,4.4,1.4,0,1,0}, + new double[] {5.6,3.0,4.5,1.5,0,1,0}, + new double[] {5.8,2.7,4.1,1.0,0,1,0}, + new double[] {6.2,2.2,4.5,1.5,0,1,0}, + new double[] {5.6,2.5,3.9,1.1,0,1,0}, + new double[] {5.9,3.2,4.8,1.8,0,1,0}, + new double[] {6.1,2.8,4.0,1.3,0,1,0}, + new double[] {6.3,2.5,4.9,1.5,0,1,0}, + new double[] {6.1,2.8,4.7,1.2,0,1,0}, + new double[] {6.4,2.9,4.3,1.3,0,1,0}, + new double[] {6.6,3.0,4.4,1.4,0,1,0}, + new double[] {6.8,2.8,4.8,1.4,0,1,0}, + new double[] {6.7,3.0,5.0,1.7,0,1,0}, + new double[] {6.0,2.9,4.5,1.5,0,1,0}, + new double[] {5.7,2.6,3.5,1.0,0,1,0}, + new double[] {5.5,2.4,3.8,1.1,0,1,0}, + new double[] {5.5,2.4,3.7,1.0,0,1,0}, + new double[] {5.8,2.7,3.9,1.2,0,1,0}, + new double[] {6.0,2.7,5.1,1.6,0,1,0}, + new double[] {5.4,3.0,4.5,1.5,0,1,0}, + new double[] {6.0,3.4,4.5,1.6,0,1,0}, + new double[] {6.7,3.1,4.7,1.5,0,1,0}, + new double[] {6.3,2.3,4.4,1.3,0,1,0}, + new double[] {5.6,3.0,4.1,1.3,0,1,0}, + new double[] {5.5,2.5,4.0,1.3,0,1,0}, + new double[] {5.5,2.6,4.4,1.2,0,1,0}, + new double[] {6.1,3.0,4.6,1.4,0,1,0}, + new double[] {5.8,2.6,4.0,1.2,0,1,0}, + new double[] {5.0,2.3,3.3,1.0,0,1,0}, + new double[] {5.6,2.7,4.2,1.3,0,1,0}, + new double[] {5.7,3.0,4.2,1.2,0,1,0}, + new double[] {5.7,2.9,4.2,1.3,0,1,0}, + new double[] {6.2,2.9,4.3,1.3,0,1,0}, + new double[] {5.1,2.5,3.0,1.1,0,1,0}, + new double[] {5.7,2.8,4.1,1.3,0,1,0}, + new double[] {6.3,3.3,6.0,2.5,1,0,0}, + new double[] {5.8,2.7,5.1,1.9,1,0,0}, + new double[] {7.1,3.0,5.9,2.1,1,0,0}, + new double[] {6.3,2.9,5.6,1.8,1,0,0}, + new double[] {6.5,3.0,5.8,2.2,1,0,0}, + new double[] {7.6,3.0,6.6,2.1,1,0,0}, + new double[] {4.9,2.5,4.5,1.7,1,0,0}, + new double[] {7.3,2.9,6.3,1.8,1,0,0}, + new double[] {6.7,2.5,5.8,1.8,1,0,0}, + new double[] {7.2,3.6,6.1,2.5,1,0,0}, + new double[] {6.5,3.2,5.1,2.0,1,0,0}, + new double[] {6.4,2.7,5.3,1.9,1,0,0}, + new double[] {6.8,3.0,5.5,2.1,1,0,0}, + new double[] {5.7,2.5,5.0,2.0,1,0,0}, + new double[] {5.8,2.8,5.1,2.4,1,0,0}, + new double[] {6.4,3.2,5.3,2.3,1,0,0}, + new double[] {6.5,3.0,5.5,1.8,1,0,0}, + new double[] {7.7,3.8,6.7,2.2,1,0,0}, + new double[] {7.7,2.6,6.9,2.3,1,0,0}, + new double[] {6.0,2.2,5.0,1.5,1,0,0}, + new double[] {6.9,3.2,5.7,2.3,1,0,0}, + new double[] {5.6,2.8,4.9,2.0,1,0,0}, + new double[] {7.7,2.8,6.7,2.0,1,0,0}, + new double[] {6.3,2.7,4.9,1.8,1,0,0}, + new double[] {6.7,3.3,5.7,2.1,1,0,0}, + new double[] {7.2,3.2,6.0,1.8,1,0,0}, + new double[] {6.2,2.8,4.8,1.8,1,0,0}, + new double[] {6.1,3.0,4.9,1.8,1,0,0}, + new double[] {6.4,2.8,5.6,2.1,1,0,0}, + new double[] {7.2,3.0,5.8,1.6,1,0,0}, + new double[] {7.4,2.8,6.1,1.9,1,0,0}, + new double[] {7.9,3.8,6.4,2.0,1,0,0}, + new double[] {6.4,2.8,5.6,2.2,1,0,0}, + new double[] {6.3,2.8,5.1,1.5,1,0,0}, + new double[] {6.1,2.6,5.6,1.4,1,0,0}, + new double[] {7.7,3.0,6.1,2.3,1,0,0}, + new double[] {6.3,3.4,5.6,2.4,1,0,0}, + new double[] {6.4,3.1,5.5,1.8,1,0,0}, + new double[] {6.0,3.0,4.8,1.8,1,0,0}, + new double[] {6.9,3.1,5.4,2.1,1,0,0}, + new double[] {6.7,3.1,5.6,2.4,1,0,0}, + new double[] {6.9,3.1,5.1,2.3,1,0,0}, + new double[] {5.8,2.7,5.1,1.9,1,0,0}, + new double[] {6.8,3.2,5.9,2.3,1,0,0}, + new double[] {6.7,3.3,5.7,2.5,1,0,0}, + new double[] {6.7,3.0,5.2,2.3,1,0,0}, + new double[] {6.3,2.5,5.0,1.9,1,0,0}, + new double[] {6.5,3.0,5.2,2.0,1,0,0}, + new double[] {6.2,3.4,5.4,2.3,1,0,0}, + new double[] {5.9,3.0,5.1,1.8,1,0,0} + }; + } +} diff --git a/Syntriax/NeuralSystem/NeuralNetwork.cs b/Syntriax/NeuralSystem/NeuralNetwork.cs new file mode 100644 index 0000000..1cb7c7c --- /dev/null +++ b/Syntriax/NeuralSystem/NeuralNetwork.cs @@ -0,0 +1,39 @@ +namespace Syntriax.NeuralNetwork +{ + public class NeuralNetwork + { + public LayerBase inputLayer { get; private set; } = null; + public LayerBase[] hiddenLayers { get; private set; } = null; + public LayerBase outputLayer { get; private set; } = null; + + public NeuralNetwork(int inputCount, int[] hiddenCounts = null, int outputCount = 1) + { + inputLayer = new InputLayer(inputCount); + + if (hiddenCounts != null && hiddenCounts.Length > 0) + { + int hiddenCount = hiddenCounts.Length; + + hiddenLayers = new HiddenLayer[hiddenCount]; + hiddenLayers[0] = new HiddenLayer(hiddenCounts[0], inputLayer); + + for (int i = 1; i < hiddenCount; i++) + hiddenLayers[i] = new HiddenLayer(hiddenCounts[i], hiddenLayers[i - 1]); + + outputLayer = new HiddenLayer(outputCount, hiddenLayers[hiddenLayers.Length - 1]); + return; + } + + outputLayer = new OutputLayer(outputCount, inputLayer); + } + + public void FireNetwork() + { + if (hiddenLayers != null) + foreach (LayerBase layer in hiddenLayers) + layer.FireLayer(); + + outputLayer.FireLayer(); + } + } +} diff --git a/Syntriax/NeuralSystem/NeuralNetworkExtensions.cs b/Syntriax/NeuralSystem/NeuralNetworkExtensions.cs new file mode 100644 index 0000000..d527d2f --- /dev/null +++ b/Syntriax/NeuralSystem/NeuralNetworkExtensions.cs @@ -0,0 +1,184 @@ +using System; +using System.Collections.Generic; +using Syntriax.NeuralNetwork.Misc; + +namespace Syntriax.NeuralNetwork +{ + public static class NeuralNetworkExtensions + { + public static List GetLayerList(this NeuralNetwork neuralNetwork) + { + List layers = new List(2 + neuralNetwork.hiddenLayers.Length); + + layers.Add(neuralNetwork.inputLayer); + layers.AddRange(neuralNetwork.hiddenLayers); + layers.Add(neuralNetwork.outputLayer); + + return layers; + } + + public static List GetNeuronList(this NeuralNetwork neuralNetwork) + { + int neuronCount = 0; + List layers = neuralNetwork.GetLayerList(); + + foreach (LayerBase layer in layers) + neuronCount += layer.neurons.Length; + + List neurons = new List(neuronCount); + + foreach (LayerBase layer in layers) + neurons.AddRange(layer.neurons); + + return neurons; + } + + public static List GetSynapseList(this NeuralNetwork neuralNetwork) + { + int synapseCount = 0; + List neurons = neuralNetwork.GetNeuronList(); + + foreach (INeuron neuron in neurons) + synapseCount += neuron.Synapses.Length + 1; // + 1 for the bias + + List synapses = new List(synapseCount); + + foreach (INeuron neuron in neurons) + { + synapses.AddRange(neuron.Synapses); + synapses.Add(neuron.Bias); + } + + return synapses; + } + + /* ------------------------------------------------------------- */ + + public static void Randomize(this NeuralNetwork neuralNetwork, int? seed = null) + { + Random random = new Random(seed ?? 0); + + foreach (ISynapse synapse in neuralNetwork.GetSynapseList()) + synapse.Weight = random.NextDouble(); + } + + /* ------------------------------------------------------------- */ + + public static void SetInput(this NeuralNetwork neuralNetwork, int neuronIndex, double value) + { + InputNeuron inputNeuron = (InputNeuron)neuralNetwork.inputLayer.neurons[neuronIndex]; + inputNeuron.Value = value; + } + public static void SetInputs(this NeuralNetwork neuralNetwork, double[] inputs) + { + for (int i = 0; i < inputs.Length; i++) + neuralNetwork.SetInput(i, inputs[i]); + } + + public static double GetOutput(this NeuralNetwork neuralNetwork, int neuronIndex) => + neuralNetwork.outputLayer.neurons[neuronIndex].Output; + + + /* ------------------------------------------------------------- */ + + public static double GetErrorOfNeuron(NeuralNetwork neuralNetwork, double[] output, int i) => + output[i] - neuralNetwork.GetOutput(i); + + public static double[] GetErrors(this NeuralNetwork neuralNetwork, double[] outputs) + { + int length = neuralNetwork.outputLayer.neurons.Length; + double[] result = new double[length]; + for (int i = 0; i < length; i++) + result[i] = GetErrorOfNeuron(neuralNetwork, outputs, i); + return result; + } + + public static double GetTotalError(this NeuralNetwork neuralNetwork, Data data) + { + double totalError = 0.0f; + double[] errors = null; + + for (int i = 0; i < data.testInput.Count; i++) + { + neuralNetwork.SetInputs(data.testInput[i]); + neuralNetwork.FireNetwork(); + + errors = neuralNetwork.GetErrors(data.testOutput[i]); + + foreach (double error in errors) + totalError += error * error; + } + + return totalError; + } + + /* ------------------------------------------------------------- */ + + public static double[] GetSoftMax(this NeuralNetwork neuralNetwork) + { + // TODO + return null; + } + + public static int GetMaxIndex(this NeuralNetwork neuralNetwork) + { + double maxValue = double.MinValue; + int maxIndex = 0; + + int count = neuralNetwork.outputLayer.neurons.Length; + for (int i = 0; i < count; i++) + { + double output = neuralNetwork.GetOutput(i); + if (output > maxValue) + { + maxValue = output; + maxIndex = i; + } + } + + return maxIndex; + } + + /* ------------------------------------------------------------- */ + + public static void Train(this NeuralNetwork neuralNetwork, Data data, double learningRate) + { + double[] errors = null; + for (int i = 0; i < data.trainInput.Count; i++) + { + neuralNetwork.SetInputs(data.trainInput[i]); + neuralNetwork.FireNetwork(); + + errors = neuralNetwork.GetErrors(data.trainOutput[i]); + + neuralNetwork.BackPropagate(errors, learningRate); + } + } + + public static void BackPropagate(this NeuralNetwork neuralNetwork, double[] errors, double learningRate) + { + for (int i = 0; i < errors.Length; i++) + Correct( + neuralNetwork.outputLayer.neurons[i], + errors[i], + learningRate + ); + } + + public static void Correct(this INeuron neuron, double error, double learningRate) + { + if (neuron.Synapses == null) + return; + + double incoming = neuron.NeuronActivation.Derivative(neuron.Output) * error; + foreach (ISynapse synapse in neuron.Synapses) + Correct(synapse.From, incoming * synapse.Weight, learningRate); + + double delta = incoming * learningRate; + foreach (ISynapse synapse in neuron.Synapses) + synapse.Weight += delta * synapse.From.Output; + + neuron.Bias.Weight += delta; + } + } +} diff --git a/Syntriax/NeuralSystem/Neuron/DropoutNeuronDecorator.cs b/Syntriax/NeuralSystem/Neuron/DropoutNeuronDecorator.cs new file mode 100644 index 0000000..ecfbe88 --- /dev/null +++ b/Syntriax/NeuralSystem/Neuron/DropoutNeuronDecorator.cs @@ -0,0 +1,23 @@ +using System; + +namespace Syntriax.NeuralNetwork +{ + public class DropoutNeuronDecorator : NeuronDecorator + { + private double dropoutRate = 0; + public bool IsActive = false; + public static Random Random = new Random(0); + + public DropoutNeuronDecorator(INeuron neuron, double dropoutRate) : base(neuron) => this.dropoutRate = dropoutRate; + + public override double Calculate() + { + base.Calculate(); + + if (Random.NextDouble() < dropoutRate) + Output = 0.0; + + return Output; + } + } +} diff --git a/Syntriax/NeuralSystem/Neuron/INeuron.cs b/Syntriax/NeuralSystem/Neuron/INeuron.cs new file mode 100644 index 0000000..4c95384 --- /dev/null +++ b/Syntriax/NeuralSystem/Neuron/INeuron.cs @@ -0,0 +1,14 @@ +using Syntriax.NeuralNetwork.NeuronActivations; + +namespace Syntriax.NeuralNetwork +{ + public interface INeuron + { + ISynapse[] Synapses { get; } + ISynapse Bias { get; } + INeuronActivation NeuronActivation { get; set; } + double Output { get; } + + double Calculate(); + } +} diff --git a/Syntriax/NeuralSystem/Neuron/Neuron.cs b/Syntriax/NeuralSystem/Neuron/Neuron.cs new file mode 100644 index 0000000..1eae391 --- /dev/null +++ b/Syntriax/NeuralSystem/Neuron/Neuron.cs @@ -0,0 +1,49 @@ +using Syntriax.NeuralNetwork.NeuronActivations; + +namespace Syntriax.NeuralNetwork +{ + public class Neuron : INeuron + { + public ISynapse[] Synapses { get; private set; } = null; + public ISynapse Bias { get; private set; } = null; + + public INeuronActivation NeuronActivation { get; set; } = new Default(); + + public virtual double Output { get; private set; } = 0.0; + + public Neuron(INeuron[] neurons = null) + { + if (neurons == null) + { + Synapses = new Synapse[0]; + Bias = new Synapse(); + return; + } + + Synapses = new ISynapse[neurons.Length]; + Bias = new Synapse(); + + int length = neurons.Length; + for (int i = 0; i < length; i++) + { + Synapses[i] = new Synapse(); + Synapses[i].From = neurons[i]; + // Synapses[i] = new MomentumSynapseDecorator(Synapses[i]); // TODO + } + } + + public double Calculate() + { + Output = 0.0; + + foreach (ISynapse synapse in Synapses) + Output += synapse.Output; + + Output += Bias.Output; + + Output = NeuronActivation.Activation(Output); + + return Output; + } + } +} diff --git a/Syntriax/NeuralSystem/Neuron/NeuronDecorator.cs b/Syntriax/NeuralSystem/Neuron/NeuronDecorator.cs new file mode 100644 index 0000000..9c6e1b6 --- /dev/null +++ b/Syntriax/NeuralSystem/Neuron/NeuronDecorator.cs @@ -0,0 +1,36 @@ +using Syntriax.NeuralNetwork.NeuronActivations; + +namespace Syntriax.NeuralNetwork +{ + public abstract class NeuronDecorator : INeuron + { + protected INeuron _neuron = null; + + public ISynapse[] Synapses => _neuron.Synapses; + public ISynapse Bias => _neuron.Bias; + + public INeuronActivation NeuronActivation + { + get => _neuron.NeuronActivation; + set => _neuron.NeuronActivation = value; + } + + public virtual double Output { get; protected set; } = 0.0; + + protected NeuronDecorator(INeuron neuron) => _neuron = neuron; + + public virtual double Calculate() + { + Output = 0.0; + + foreach (ISynapse synapse in Synapses) + Output += synapse.Output; + + Output += Bias.Output; + + Output = NeuronActivation.Activation(Output); + + return Output; + } + } +} diff --git a/Syntriax/NeuralSystem/NeuronActivations/Default.cs b/Syntriax/NeuralSystem/NeuronActivations/Default.cs new file mode 100644 index 0000000..3995fcc --- /dev/null +++ b/Syntriax/NeuralSystem/NeuronActivations/Default.cs @@ -0,0 +1,9 @@ +namespace Syntriax.NeuralNetwork.NeuronActivations +{ + public class Default : INeuronActivation + { + public double Activation(double value) => value; + + public double Derivative(double value) => value; + } +} diff --git a/Syntriax/NeuralSystem/NeuronActivations/INeuronActivation.cs b/Syntriax/NeuralSystem/NeuronActivations/INeuronActivation.cs new file mode 100644 index 0000000..82798d2 --- /dev/null +++ b/Syntriax/NeuralSystem/NeuronActivations/INeuronActivation.cs @@ -0,0 +1,8 @@ +namespace Syntriax.NeuralNetwork.NeuronActivations +{ + public interface INeuronActivation + { + double Activation(double value); + double Derivative(double value); + } +} diff --git a/Syntriax/NeuralSystem/NeuronActivations/Relu.cs b/Syntriax/NeuralSystem/NeuronActivations/Relu.cs new file mode 100644 index 0000000..2b6ae2d --- /dev/null +++ b/Syntriax/NeuralSystem/NeuronActivations/Relu.cs @@ -0,0 +1,19 @@ +namespace Syntriax.NeuralNetwork.NeuronActivations +{ + public class Relu : INeuronActivation + { + public double Activation(double value) + { + if (value >= 0.0) + return value; + return 0.0; + } + + public double Derivative(double value) + { + if (value >= 0.0) + return 1.0; + return 0.001; + } + } +} diff --git a/Syntriax/NeuralSystem/NeuronActivations/Sigmoid.cs b/Syntriax/NeuralSystem/NeuronActivations/Sigmoid.cs new file mode 100644 index 0000000..e7c8e9f --- /dev/null +++ b/Syntriax/NeuralSystem/NeuronActivations/Sigmoid.cs @@ -0,0 +1,11 @@ +using System; + +namespace Syntriax.NeuralNetwork.NeuronActivations +{ + public class Sigmoid : INeuronActivation + { + public double Activation(double value) => 1.0 / (1.0 + Math.Exp(-value)); + + public double Derivative(double value) => value * (1.0 - value); + } +} diff --git a/Syntriax/NeuralSystem/NeuronActivations/StaticActivation.cs b/Syntriax/NeuralSystem/NeuronActivations/StaticActivation.cs new file mode 100644 index 0000000..49c42d2 --- /dev/null +++ b/Syntriax/NeuralSystem/NeuronActivations/StaticActivation.cs @@ -0,0 +1,11 @@ +using System; + +namespace Syntriax.NeuralNetwork.NeuronActivations +{ + public class StaticActivation where T : INeuronActivation, new() + { + private static readonly Lazy instance = new Lazy(() => new T()); + + public static T Instance => instance.Value; + } +} diff --git a/Syntriax/NeuralSystem/NeuronActivations/TanH.cs b/Syntriax/NeuralSystem/NeuronActivations/TanH.cs new file mode 100644 index 0000000..08a2be9 --- /dev/null +++ b/Syntriax/NeuralSystem/NeuronActivations/TanH.cs @@ -0,0 +1,11 @@ +using System; + +namespace Syntriax.NeuralNetwork.NeuronActivations +{ + public class TanH : INeuronActivation + { + public double Activation(double value) => (Math.Exp(value) - Math.Exp(-value)) / (Math.Exp(value) + Math.Exp(-value)); + + public double Derivative(double value) => 1.0 - value * value; + } +} diff --git a/Syntriax/NeuralSystem/Synapse/DropoutSynapseDecorator.cs b/Syntriax/NeuralSystem/Synapse/DropoutSynapseDecorator.cs new file mode 100644 index 0000000..714a11e --- /dev/null +++ b/Syntriax/NeuralSystem/Synapse/DropoutSynapseDecorator.cs @@ -0,0 +1,27 @@ +using System; + +namespace Syntriax.NeuralNetwork +{ + public class DropoutSynapseDecorator : SynapseDecorator + { + public bool IsActive = false; + public Random random = null; + + public override double Weight + { + get => base.Weight; + set + { + if (random.Next() % 2 == 0) + base.Weight = 0; + else + base.Weight = value; + } + } + + public DropoutSynapseDecorator(ISynapse synapse, Random random = null) : base(synapse) + => SetRandom(random); + + public void SetRandom(Random random) => this.random = random; + } +} diff --git a/Syntriax/NeuralSystem/Synapse/ISynapse.cs b/Syntriax/NeuralSystem/Synapse/ISynapse.cs new file mode 100644 index 0000000..6327cae --- /dev/null +++ b/Syntriax/NeuralSystem/Synapse/ISynapse.cs @@ -0,0 +1,9 @@ +namespace Syntriax.NeuralNetwork +{ + public interface ISynapse + { + INeuron From { get; set; } + double Output { get; } + double Weight { get; set; } + } +} diff --git a/Syntriax/NeuralSystem/Synapse/MomentumSynapseDecorator.cs b/Syntriax/NeuralSystem/Synapse/MomentumSynapseDecorator.cs new file mode 100644 index 0000000..2f42491 --- /dev/null +++ b/Syntriax/NeuralSystem/Synapse/MomentumSynapseDecorator.cs @@ -0,0 +1,27 @@ +namespace Syntriax.NeuralNetwork +{ + public class MomentumSynapseDecorator : SynapseDecorator + { + private double _momentum = 0.0; + private const double Beta = 0.9; + + public override double Weight + { + get => base.Weight; + set + { + double difference = value - base.Weight; + + // TODO Might be an error here + if (difference * _momentum >= 0.0 == _momentum >= 0.0) + base.Weight = value + _momentum * Beta; + else + base.Weight = value; + + _momentum = difference; + } + } + + public MomentumSynapseDecorator(ISynapse synapse) : base(synapse) { } + } +} diff --git a/Syntriax/NeuralSystem/Synapse/Synapse.cs b/Syntriax/NeuralSystem/Synapse/Synapse.cs new file mode 100644 index 0000000..535b1fb --- /dev/null +++ b/Syntriax/NeuralSystem/Synapse/Synapse.cs @@ -0,0 +1,25 @@ +namespace Syntriax.NeuralNetwork +{ + + public class Synapse : ISynapse + { + private INeuron from = null; + public INeuron From + { + get => from; + set + { + if (from == null) + from = value; + } + } + + public double Output => Weight * (from == null ? 1.0 : from.Output); + + + public double Weight { get; set; } = 0.0; + + public Synapse() { } + public Synapse(INeuron from) => this.from = from; + } +} diff --git a/Syntriax/NeuralSystem/Synapse/SynapseDecorator.cs b/Syntriax/NeuralSystem/Synapse/SynapseDecorator.cs new file mode 100644 index 0000000..56f580d --- /dev/null +++ b/Syntriax/NeuralSystem/Synapse/SynapseDecorator.cs @@ -0,0 +1,16 @@ +namespace Syntriax.NeuralNetwork +{ + public abstract class SynapseDecorator : ISynapse + { + protected ISynapse _synapse = null; + + public INeuron From { get => _synapse.From; set => _synapse.From = value; } + + public virtual double Output => _synapse.Output; + + public virtual double Weight { get => _synapse.Weight; set => _synapse.Weight = value; } + + protected SynapseDecorator(ISynapse synapse) => SetSynapse(synapse); + public virtual void SetSynapse(ISynapse synapse) => _synapse = synapse; + } +}