diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..426d76d
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,398 @@
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+##
+## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore
+
+# User-specific files
+*.rsuser
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Mono auto generated files
+mono_crash.*
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+[Ww][Ii][Nn]32/
+[Aa][Rr][Mm]/
+[Aa][Rr][Mm]64/
+bld/
+[Bb]in/
+[Oo]bj/
+[Ll]og/
+[Ll]ogs/
+
+# Visual Studio 2015/2017 cache/options directory
+.vs/
+# Uncomment if you have tasks that create the project's static files in wwwroot
+#wwwroot/
+
+# Visual Studio 2017 auto generated files
+Generated\ Files/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUnit
+*.VisualState.xml
+TestResult.xml
+nunit-*.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# Benchmark Results
+BenchmarkDotNet.Artifacts/
+
+# .NET Core
+project.lock.json
+project.fragment.lock.json
+artifacts/
+
+# ASP.NET Scaffolding
+ScaffoldingReadMe.txt
+
+# StyleCop
+StyleCopReport.xml
+
+# Files built by Visual Studio
+*_i.c
+*_p.c
+*_h.h
+*.ilk
+*.meta
+*.obj
+*.iobj
+*.pch
+*.pdb
+*.ipdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*_wpftmp.csproj
+*.log
+*.tlog
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
+*.VC.db
+*.VC.VC.opendb
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+*.sap
+
+# Visual Studio Trace Files
+*.e2e
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# AxoCover is a Code Coverage Tool
+.axoCover/*
+!.axoCover/settings.json
+
+# Coverlet is a free, cross platform Code Coverage Tool
+coverage*.json
+coverage*.xml
+coverage*.info
+
+# Visual Studio code coverage results
+*.coverage
+*.coveragexml
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+nCrunchTemp_*
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# Note: Comment the next line if you want to checkin your web deploy settings,
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# Microsoft Azure Web App publish settings. Comment the next line if you want to
+# checkin your Azure Web App publish settings, but sensitive information contained
+# in these scripts will be unencrypted
+PublishScripts/
+
+# NuGet Packages
+*.nupkg
+# NuGet Symbol Packages
+*.snupkg
+# The packages folder can be ignored because of Package Restore
+**/[Pp]ackages/*
+# except build/, which is used as an MSBuild target.
+!**/[Pp]ackages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/[Pp]ackages/repositories.config
+# NuGet v3's project.json files produces more ignorable files
+*.nuget.props
+*.nuget.targets
+
+# Microsoft Azure Build Output
+csx/
+*.build.csdef
+
+# Microsoft Azure Emulator
+ecf/
+rcf/
+
+# Windows Store app package directories and files
+AppPackages/
+BundleArtifacts/
+Package.StoreAssociation.xml
+_pkginfo.txt
+*.appx
+*.appxbundle
+*.appxupload
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!?*.[Cc]ache/
+
+# Others
+ClientBin/
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.jfm
+*.pfx
+*.publishsettings
+orleans.codegen.cs
+
+# Including strong name files can present a security risk
+# (https://github.com/github/gitignore/pull/2483#issue-259490424)
+#*.snk
+
+# Since there are multiple workflows, uncomment next line to ignore bower_components
+# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
+#bower_components/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+ServiceFabricBackup/
+*.rptproj.bak
+
+# SQL Server files
+*.mdf
+*.ldf
+*.ndf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+*.rptproj.rsuser
+*- [Bb]ackup.rdl
+*- [Bb]ackup ([0-9]).rdl
+*- [Bb]ackup ([0-9][0-9]).rdl
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# GhostDoc plugin setting file
+*.GhostDoc.xml
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+node_modules/
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
+*.vbw
+
+# Visual Studio 6 auto-generated project file (contains which files were open etc.)
+*.vbp
+
+# Visual Studio 6 workspace and project file (working project files containing files to include in project)
+*.dsw
+*.dsp
+
+# Visual Studio 6 technical files
+*.ncb
+*.aps
+
+# Visual Studio LightSwitch build output
+**/*.HTMLClient/GeneratedArtifacts
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+_Pvt_Extensions
+
+# Paket dependency manager
+.paket/paket.exe
+paket-files/
+
+# FAKE - F# Make
+.fake/
+
+# CodeRush personal settings
+.cr/personal
+
+# Python Tools for Visual Studio (PTVS)
+__pycache__/
+*.pyc
+
+# Cake - Uncomment if you are using it
+# tools/**
+# !tools/packages.config
+
+# Tabs Studio
+*.tss
+
+# Telerik's JustMock configuration file
+*.jmconfig
+
+# BizTalk build output
+*.btp.cs
+*.btm.cs
+*.odx.cs
+*.xsd.cs
+
+# OpenCover UI analysis results
+OpenCover/
+
+# Azure Stream Analytics local run output
+ASALocalRun/
+
+# MSBuild Binary and Structured Log
+*.binlog
+
+# NVidia Nsight GPU debugger configuration file
+*.nvuser
+
+# MFractors (Xamarin productivity tool) working folder
+.mfractor/
+
+# Local History for Visual Studio
+.localhistory/
+
+# Visual Studio History (VSHistory) files
+.vshistory/
+
+# BeatPulse healthcheck temp database
+healthchecksdb
+
+# Backup folder for Package Reference Convert tool in Visual Studio 2017
+MigrationBackup/
+
+# Ionide (cross platform F# VS Code tools) working folder
+.ionide/
+
+# Fody - auto-generated XML schema
+FodyWeavers.xsd
+
+# VS Code files for those working on multiple tools
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/
+
+# Windows Installer files from build outputs
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# JetBrains Rider
+*.sln.iml
diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 0000000..9ee3acd
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,26 @@
+{
+ "version": "0.2.0",
+ "configurations": [
+ {
+ // Use IntelliSense to find out which attributes exist for C# debugging
+ // Use hover for the description of the existing attributes
+ // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
+ "name": ".NET Core Launch (console)",
+ "type": "coreclr",
+ "request": "launch",
+ "preLaunchTask": "build",
+ // If you have changed target frameworks, make sure to update the program path.
+ "program": "${workspaceFolder}/bin/Debug/net5.0/NeuralNetwork2021.dll",
+ "args": [],
+ "cwd": "${workspaceFolder}",
+ // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
+ "console": "internalConsole",
+ "stopAtEntry": false
+ },
+ {
+ "name": ".NET Core Attach",
+ "type": "coreclr",
+ "request": "attach"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
new file mode 100644
index 0000000..5936938
--- /dev/null
+++ b/.vscode/tasks.json
@@ -0,0 +1,42 @@
+{
+ "version": "2.0.0",
+ "tasks": [
+ {
+ "label": "build",
+ "command": "dotnet",
+ "type": "process",
+ "args": [
+ "build",
+ "${workspaceFolder}/NeuralNetwork2021.csproj",
+ "/property:GenerateFullPaths=true",
+ "/consoleloggerparameters:NoSummary"
+ ],
+ "problemMatcher": "$msCompile"
+ },
+ {
+ "label": "publish",
+ "command": "dotnet",
+ "type": "process",
+ "args": [
+ "publish",
+ "${workspaceFolder}/NeuralNetwork2021.csproj",
+ "/property:GenerateFullPaths=true",
+ "/consoleloggerparameters:NoSummary"
+ ],
+ "problemMatcher": "$msCompile"
+ },
+ {
+ "label": "watch",
+ "command": "dotnet",
+ "type": "process",
+ "args": [
+ "watch",
+ "run",
+ "${workspaceFolder}/NeuralNetwork2021.csproj",
+ "/property:GenerateFullPaths=true",
+ "/consoleloggerparameters:NoSummary"
+ ],
+ "problemMatcher": "$msCompile"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/NeuralNetwork2021.csproj b/NeuralNetwork2021.csproj
new file mode 100644
index 0000000..2082704
--- /dev/null
+++ b/NeuralNetwork2021.csproj
@@ -0,0 +1,8 @@
+
+
+
+ Exe
+ net5.0
+
+
+
diff --git a/Program.cs b/Program.cs
new file mode 100644
index 0000000..d8b7f43
--- /dev/null
+++ b/Program.cs
@@ -0,0 +1,48 @@
+using System;
+using System.Linq;
+using Syntriax.NeuralNetwork;
+using Syntriax.NeuralNetwork.Misc;
+using Syntriax.NeuralNetwork.NeuronActivations;
+
+namespace NeuralNetwork2021
+{
+ class Program
+ {
+ static void Main(string[] args)
+ {
+ const int epochCount = 5000;
+ const int epochPrintInterval = 1;
+
+ const int dataSeed = 10;
+ const int weightSeed = 0;
+ const int dropoutSeed = 0;
+ DropoutNeuronDecorator.Random = new Random(dropoutSeed);
+
+ double learningRate = 0.001;
+
+ Data data = new Data(DataTest.LoadData().ToList(), 4, seed: dataSeed);
+
+ NeuralNetwork neuralNetwork = new NeuralNetwork
+ (
+ data.InputCount,
+ new int[] { 10 },
+ data.OutputCount
+ );
+
+ foreach (LayerBase layer in neuralNetwork.GetLayerList())
+ layer.SetActivation(StaticActivation.Instance);
+ // neuralNetwork.outputLayer.SetActivation(StaticActivation.Instance);
+
+ neuralNetwork.Randomize(weightSeed);
+
+ for (int k = 0; k < epochCount; k++)
+ {
+ if (k % epochPrintInterval == 0)
+ Console.WriteLine($"Epoch: {k}\tHata: { neuralNetwork.GetTotalError(data) }");
+
+ neuralNetwork.Train(data, learningRate);
+ }
+ Console.WriteLine($"Hata: { neuralNetwork.GetTotalError(data) }");
+ }
+ }
+}
diff --git a/Syntriax/NeuralSystem/InputNeuron.cs b/Syntriax/NeuralSystem/InputNeuron.cs
new file mode 100644
index 0000000..f09a266
--- /dev/null
+++ b/Syntriax/NeuralSystem/InputNeuron.cs
@@ -0,0 +1,9 @@
+namespace Syntriax.NeuralNetwork
+{
+ public class InputNeuron : Neuron
+ {
+ public double Value = 0.0;
+
+ public override double Output => Value;
+ }
+}
diff --git a/Syntriax/NeuralSystem/Layer/HiddenLayer.cs b/Syntriax/NeuralSystem/Layer/HiddenLayer.cs
new file mode 100644
index 0000000..9b16a2f
--- /dev/null
+++ b/Syntriax/NeuralSystem/Layer/HiddenLayer.cs
@@ -0,0 +1,18 @@
+namespace Syntriax.NeuralNetwork
+{
+ public class HiddenLayer : LayerBase
+ {
+ public HiddenLayer(int neuronCount, LayerBase from = null) : base(neuronCount, from) { }
+
+ protected override void SetLayer(int neuronCount, LayerBase from)
+ {
+ neurons = new INeuron[neuronCount];
+
+ for (int i = 0; i < neuronCount; i++)
+ {
+ neurons[i] = new Neuron(from == null ? null : from.neurons);
+ // neurons[i] = new DropoutNeuronDecorator(neurons[i], 0.2); // TODO
+ }
+ }
+ }
+}
diff --git a/Syntriax/NeuralSystem/Layer/InputLayer.cs b/Syntriax/NeuralSystem/Layer/InputLayer.cs
new file mode 100644
index 0000000..e93e9a8
--- /dev/null
+++ b/Syntriax/NeuralSystem/Layer/InputLayer.cs
@@ -0,0 +1,15 @@
+namespace Syntriax.NeuralNetwork
+{
+ public class InputLayer : LayerBase
+ {
+ public InputLayer(int neuronCount, LayerBase from = null) : base(neuronCount, from) { }
+
+ protected override void SetLayer(int neuronCount, LayerBase from)
+ {
+ neurons = new Neuron[neuronCount];
+
+ for (int i = 0; i < neuronCount; i++)
+ neurons[i] = new InputNeuron();
+ }
+ }
+}
diff --git a/Syntriax/NeuralSystem/Layer/Layer.cs b/Syntriax/NeuralSystem/Layer/Layer.cs
new file mode 100644
index 0000000..a8de58a
--- /dev/null
+++ b/Syntriax/NeuralSystem/Layer/Layer.cs
@@ -0,0 +1,25 @@
+using Syntriax.NeuralNetwork.NeuronActivations;
+
+namespace Syntriax.NeuralNetwork
+{
+ public abstract class LayerBase
+ {
+ public INeuron[] neurons { get; protected set; } = null;
+
+ public LayerBase(int neuronCount, LayerBase from = null) => SetLayer(neuronCount, from);
+
+ protected abstract void SetLayer(int neuronCount, LayerBase from);
+
+ public void FireLayer()
+ {
+ foreach (INeuron neuron in neurons)
+ neuron.Calculate();
+ }
+
+ public void SetActivation(INeuronActivation neuronActivation)
+ {
+ foreach (INeuron neuron in neurons)
+ neuron.NeuronActivation = neuronActivation;
+ }
+ }
+}
diff --git a/Syntriax/NeuralSystem/Layer/OutputLayer.cs b/Syntriax/NeuralSystem/Layer/OutputLayer.cs
new file mode 100644
index 0000000..ccf5fb0
--- /dev/null
+++ b/Syntriax/NeuralSystem/Layer/OutputLayer.cs
@@ -0,0 +1,15 @@
+namespace Syntriax.NeuralNetwork
+{
+ public class OutputLayer : HiddenLayer
+ {
+ public OutputLayer(int neuronCount, LayerBase from = null) : base(neuronCount, from) { }
+
+ protected override void SetLayer(int neuronCount, LayerBase from)
+ {
+ neurons = new INeuron[neuronCount];
+
+ for (int i = 0; i < neuronCount; i++)
+ neurons[i] = new Neuron(from.neurons);
+ }
+ }
+}
diff --git a/Syntriax/NeuralSystem/Misc/Data.cs b/Syntriax/NeuralSystem/Misc/Data.cs
new file mode 100644
index 0000000..521f8e4
--- /dev/null
+++ b/Syntriax/NeuralSystem/Misc/Data.cs
@@ -0,0 +1,92 @@
+using System;
+using System.Collections.Generic;
+
+namespace Syntriax.NeuralNetwork.Misc
+{
+ public class Data
+ {
+ public int InputCount => trainInput[0].Length;
+ public int OutputCount => trainOutput[0].Length;
+ public List trainInput { get; private set; } = null;
+ public List trainOutput { get; private set; } = null;
+ public List testInput { get; private set; } = null;
+ public List testOutput { get; private set; } = null;
+
+ public Data(List data, int inputCount, double trainRatio = 0.2, int? seed = null)
+ {
+ int indexToSwap = 0;
+ int count = data.Count;
+ int testCount = (int)(count * trainRatio);
+ Random random = new Random(seed ?? 0);
+
+ double[] inputArray = null;
+ double[] outputArray = null;
+
+ trainInput = new List(count - testCount);
+ trainOutput = new List(count - testCount);
+ testInput = new List(testCount);
+ testOutput = new List(testCount);
+
+ for (int i = 0; i < count; i++)
+ {
+ indexToSwap = random.Next(i, count);
+ (data[i], data[indexToSwap]) = (data[indexToSwap], data[i]);
+ }
+
+ for (int i = 0; i < count; i++)
+ {
+ (inputArray, outputArray) = SplitData(data[i], inputCount);
+ if (i < testCount)
+ {
+ testInput.Add(inputArray);
+ testOutput.Add(outputArray);
+ }
+ else
+ {
+ trainInput.Add(inputArray);
+ trainOutput.Add(outputArray);
+ }
+ }
+ }
+ public Data(List train, List test, int inputCount, double trainRatio = 0.2)
+ {
+ double[] inputArray = null;
+ double[] outputArray = null;
+
+ trainInput = new List(train.Count);
+ trainOutput = new List(train.Count);
+ testInput = new List(test.Count);
+ testOutput = new List(test.Count);
+
+ for (int i = 0; i < train.Count; i++)
+ {
+ (inputArray, outputArray) = SplitData(train[i], inputCount);
+ trainInput.Add(inputArray);
+ trainOutput.Add(outputArray);
+ }
+
+ for (int i = 0; i < test.Count; i++)
+ {
+ (inputArray, outputArray) = SplitData(test[i], inputCount);
+ testInput.Add(inputArray);
+ testOutput.Add(outputArray);
+ }
+ }
+
+ private (double[] inputArray, double[] outputArray) SplitData(double[] array, int inputCount)
+ {
+ int outputCount = array.Length - inputCount + 1;
+ int i = 0;
+ double[] input = new double[inputCount];
+ double[] output = new double[outputCount];
+
+ for (i = 0; i < array.Length; i++)
+ if (i >= inputCount)
+ output[i - inputCount] = array[i];
+ else
+ input[i] = array[i];
+
+ return (input, output);
+ }
+ }
+}
diff --git a/Syntriax/NeuralSystem/Misc/DataNew.cs b/Syntriax/NeuralSystem/Misc/DataNew.cs
new file mode 100644
index 0000000..afb7470
--- /dev/null
+++ b/Syntriax/NeuralSystem/Misc/DataNew.cs
@@ -0,0 +1,83 @@
+// using System;
+
+// namespace Syntriax.NeuralNetwork.Misc
+// {
+// public class DataNew
+// {
+// public int InputCount => trainInput.GetLength(0);
+// public int OutputCount => trainOutput.GetLength(0);
+// public double[,] trainInput { get; private set; } = null;
+// public double[,] trainOutput { get; private set; } = null;
+// public double[,] testInput { get; private set; } = null;
+// public double[,] testOutput { get; private set; } = null;
+
+// public DataNew(double[,] data, int inputCount, double testRatio = 0.2, int? seed = null)
+// {
+// int attributeCount = data.GetLength(1);
+// int dataCount = data.GetLength(0);
+// int testCount = (int)(dataCount * testRatio);
+// int trainCount = dataCount - testCount;
+
+// double[,] trainData = new double[trainCount, attributeCount];
+// double[,] testData = new double[testCount, attributeCount];
+
+// if (seed != null)
+// Shuffle(data, attributeCount, new Random(seed.Value));
+
+// int i = 0;
+// int a = 0;
+// for (i = 0; i < dataCount; i++)
+// for (a = 0; a < attributeCount; a++)
+
+
+// }
+
+// private void Shuffle(double[,] data, int attributeCount, Random random)
+// {
+// int indexToSwap = 0;
+// int length = data.Length;
+// int i = 0;
+// int a = 0;
+
+// for (i = 0; i < length; i++)
+// {
+// indexToSwap = random.Next(i, length);
+// for (a = 0; a < attributeCount; a++)
+// (data[i, a], data[indexToSwap, a]) = (data[indexToSwap, a], data[i, a]);
+// }
+// }
+
+// public DataNew(double[,] train, double[,] test, int inputCount)
+// {
+// int attributeCount = train.GetLength(1);
+// int attributeInputCount = inputCount;
+// int attributeOutputCount = attributeCount - attributeInputCount;
+
+// int trainCount = train.GetLength(0);
+// int testCount = test.GetLength(0);
+
+// trainInput = new double[train.GetLength(0), attributeInputCount];
+// trainOutput = new double[train.GetLength(0), attributeOutputCount];
+// testInput = new double[test.GetLength(0), attributeInputCount];
+// testOutput = new double[test.GetLength(0), attributeOutputCount];
+
+// SplitIntoAttributeArrays(train, trainInput, trainOutput, trainCount, attributeInputCount, attributeOutputCount);
+// SplitIntoAttributeArrays(test, testInput, testOutput, testCount, attributeInputCount, attributeOutputCount);
+// }
+
+// private void SplitIntoAttributeArrays(double[,] source, double[,] inputDestination, double[,] outputDestination,
+// int count, int inputCount, int outputCount)
+// {
+// int i = 0;
+// int j = 0;
+
+// for (i = 0; i < count; i++)
+// {
+// for (j = 0; j < inputCount; j++)
+// inputDestination[i, j] = source[i, j];
+// for (j = 0; j < outputCount; j++)
+// outputDestination[i, j] = source[i, j + inputCount];
+// }
+// }
+// }
+// }
diff --git a/Syntriax/NeuralSystem/Misc/DataTest.cs b/Syntriax/NeuralSystem/Misc/DataTest.cs
new file mode 100644
index 0000000..a4552ed
--- /dev/null
+++ b/Syntriax/NeuralSystem/Misc/DataTest.cs
@@ -0,0 +1,163 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace Syntriax.NeuralNetwork.Misc
+{
+ public class DataTest
+ {
+ public static double[][] LoadData() => new double[][]
+ {
+ new double[] {5.1,3.5,1.4,0.2,0,0,1},
+ new double[] {4.9,3.0,1.4,0.2,0,0,1},
+ new double[] {4.7,3.2,1.3,0.2,0,0,1},
+ new double[] {4.6,3.1,1.5,0.2,0,0,1},
+ new double[] {5.0,3.6,1.4,0.2,0,0,1},
+ new double[] {5.4,3.9,1.7,0.4,0,0,1},
+ new double[] {4.6,3.4,1.4,0.3,0,0,1},
+ new double[] {5.0,3.4,1.5,0.2,0,0,1},
+ new double[] {4.4,2.9,1.4,0.2,0,0,1},
+ new double[] {4.9,3.1,1.5,0.1,0,0,1},
+ new double[] {5.4,3.7,1.5,0.2,0,0,1},
+ new double[] {4.8,3.4,1.6,0.2,0,0,1},
+ new double[] {4.8,3.0,1.4,0.1,0,0,1},
+ new double[] {4.3,3.0,1.1,0.1,0,0,1},
+ new double[] {5.8,4.0,1.2,0.2,0,0,1},
+ new double[] {5.7,4.4,1.5,0.4,0,0,1},
+ new double[] {5.4,3.9,1.3,0.4,0,0,1},
+ new double[] {5.1,3.5,1.4,0.3,0,0,1},
+ new double[] {5.7,3.8,1.7,0.3,0,0,1},
+ new double[] {5.1,3.8,1.5,0.3,0,0,1},
+ new double[] {5.4,3.4,1.7,0.2,0,0,1},
+ new double[] {5.1,3.7,1.5,0.4,0,0,1},
+ new double[] {4.6,3.6,1.0,0.2,0,0,1},
+ new double[] {5.1,3.3,1.7,0.5,0,0,1},
+ new double[] {4.8,3.4,1.9,0.2,0,0,1},
+ new double[] {5.0,3.0,1.6,0.2,0,0,1},
+ new double[] {5.0,3.4,1.6,0.4,0,0,1},
+ new double[] {5.2,3.5,1.5,0.2,0,0,1},
+ new double[] {5.2,3.4,1.4,0.2,0,0,1},
+ new double[] {4.7,3.2,1.6,0.2,0,0,1},
+ new double[] {4.8,3.1,1.6,0.2,0,0,1},
+ new double[] {5.4,3.4,1.5,0.4,0,0,1},
+ new double[] {5.2,4.1,1.5,0.1,0,0,1},
+ new double[] {5.5,4.2,1.4,0.2,0,0,1},
+ new double[] {4.9,3.1,1.5,0.1,0,0,1},
+ new double[] {5.0,3.2,1.2,0.2,0,0,1},
+ new double[] {5.5,3.5,1.3,0.2,0,0,1},
+ new double[] {4.9,3.1,1.5,0.1,0,0,1},
+ new double[] {4.4,3.0,1.3,0.2,0,0,1},
+ new double[] {5.1,3.4,1.5,0.2,0,0,1},
+ new double[] {5.0,3.5,1.3,0.3,0,0,1},
+ new double[] {4.5,2.3,1.3,0.3,0,0,1},
+ new double[] {4.4,3.2,1.3,0.2,0,0,1},
+ new double[] {5.0,3.5,1.6,0.6,0,0,1},
+ new double[] {5.1,3.8,1.9,0.4,0,0,1},
+ new double[] {4.8,3.0,1.4,0.3,0,0,1},
+ new double[] {5.1,3.8,1.6,0.2,0,0,1},
+ new double[] {4.6,3.2,1.4,0.2,0,0,1},
+ new double[] {5.3,3.7,1.5,0.2,0,0,1},
+ new double[] {5.0,3.3,1.4,0.2,0,0,1},
+ new double[] {7.0,3.2,4.7,1.4,0,1,0},
+ new double[] {6.4,3.2,4.5,1.5,0,1,0},
+ new double[] {6.9,3.1,4.9,1.5,0,1,0},
+ new double[] {5.5,2.3,4.0,1.3,0,1,0},
+ new double[] {6.5,2.8,4.6,1.5,0,1,0},
+ new double[] {5.7,2.8,4.5,1.3,0,1,0},
+ new double[] {6.3,3.3,4.7,1.6,0,1,0},
+ new double[] {4.9,2.4,3.3,1.0,0,1,0},
+ new double[] {6.6,2.9,4.6,1.3,0,1,0},
+ new double[] {5.2,2.7,3.9,1.4,0,1,0},
+ new double[] {5.0,2.0,3.5,1.0,0,1,0},
+ new double[] {5.9,3.0,4.2,1.5,0,1,0},
+ new double[] {6.0,2.2,4.0,1.0,0,1,0},
+ new double[] {6.1,2.9,4.7,1.4,0,1,0},
+ new double[] {5.6,2.9,3.6,1.3,0,1,0},
+ new double[] {6.7,3.1,4.4,1.4,0,1,0},
+ new double[] {5.6,3.0,4.5,1.5,0,1,0},
+ new double[] {5.8,2.7,4.1,1.0,0,1,0},
+ new double[] {6.2,2.2,4.5,1.5,0,1,0},
+ new double[] {5.6,2.5,3.9,1.1,0,1,0},
+ new double[] {5.9,3.2,4.8,1.8,0,1,0},
+ new double[] {6.1,2.8,4.0,1.3,0,1,0},
+ new double[] {6.3,2.5,4.9,1.5,0,1,0},
+ new double[] {6.1,2.8,4.7,1.2,0,1,0},
+ new double[] {6.4,2.9,4.3,1.3,0,1,0},
+ new double[] {6.6,3.0,4.4,1.4,0,1,0},
+ new double[] {6.8,2.8,4.8,1.4,0,1,0},
+ new double[] {6.7,3.0,5.0,1.7,0,1,0},
+ new double[] {6.0,2.9,4.5,1.5,0,1,0},
+ new double[] {5.7,2.6,3.5,1.0,0,1,0},
+ new double[] {5.5,2.4,3.8,1.1,0,1,0},
+ new double[] {5.5,2.4,3.7,1.0,0,1,0},
+ new double[] {5.8,2.7,3.9,1.2,0,1,0},
+ new double[] {6.0,2.7,5.1,1.6,0,1,0},
+ new double[] {5.4,3.0,4.5,1.5,0,1,0},
+ new double[] {6.0,3.4,4.5,1.6,0,1,0},
+ new double[] {6.7,3.1,4.7,1.5,0,1,0},
+ new double[] {6.3,2.3,4.4,1.3,0,1,0},
+ new double[] {5.6,3.0,4.1,1.3,0,1,0},
+ new double[] {5.5,2.5,4.0,1.3,0,1,0},
+ new double[] {5.5,2.6,4.4,1.2,0,1,0},
+ new double[] {6.1,3.0,4.6,1.4,0,1,0},
+ new double[] {5.8,2.6,4.0,1.2,0,1,0},
+ new double[] {5.0,2.3,3.3,1.0,0,1,0},
+ new double[] {5.6,2.7,4.2,1.3,0,1,0},
+ new double[] {5.7,3.0,4.2,1.2,0,1,0},
+ new double[] {5.7,2.9,4.2,1.3,0,1,0},
+ new double[] {6.2,2.9,4.3,1.3,0,1,0},
+ new double[] {5.1,2.5,3.0,1.1,0,1,0},
+ new double[] {5.7,2.8,4.1,1.3,0,1,0},
+ new double[] {6.3,3.3,6.0,2.5,1,0,0},
+ new double[] {5.8,2.7,5.1,1.9,1,0,0},
+ new double[] {7.1,3.0,5.9,2.1,1,0,0},
+ new double[] {6.3,2.9,5.6,1.8,1,0,0},
+ new double[] {6.5,3.0,5.8,2.2,1,0,0},
+ new double[] {7.6,3.0,6.6,2.1,1,0,0},
+ new double[] {4.9,2.5,4.5,1.7,1,0,0},
+ new double[] {7.3,2.9,6.3,1.8,1,0,0},
+ new double[] {6.7,2.5,5.8,1.8,1,0,0},
+ new double[] {7.2,3.6,6.1,2.5,1,0,0},
+ new double[] {6.5,3.2,5.1,2.0,1,0,0},
+ new double[] {6.4,2.7,5.3,1.9,1,0,0},
+ new double[] {6.8,3.0,5.5,2.1,1,0,0},
+ new double[] {5.7,2.5,5.0,2.0,1,0,0},
+ new double[] {5.8,2.8,5.1,2.4,1,0,0},
+ new double[] {6.4,3.2,5.3,2.3,1,0,0},
+ new double[] {6.5,3.0,5.5,1.8,1,0,0},
+ new double[] {7.7,3.8,6.7,2.2,1,0,0},
+ new double[] {7.7,2.6,6.9,2.3,1,0,0},
+ new double[] {6.0,2.2,5.0,1.5,1,0,0},
+ new double[] {6.9,3.2,5.7,2.3,1,0,0},
+ new double[] {5.6,2.8,4.9,2.0,1,0,0},
+ new double[] {7.7,2.8,6.7,2.0,1,0,0},
+ new double[] {6.3,2.7,4.9,1.8,1,0,0},
+ new double[] {6.7,3.3,5.7,2.1,1,0,0},
+ new double[] {7.2,3.2,6.0,1.8,1,0,0},
+ new double[] {6.2,2.8,4.8,1.8,1,0,0},
+ new double[] {6.1,3.0,4.9,1.8,1,0,0},
+ new double[] {6.4,2.8,5.6,2.1,1,0,0},
+ new double[] {7.2,3.0,5.8,1.6,1,0,0},
+ new double[] {7.4,2.8,6.1,1.9,1,0,0},
+ new double[] {7.9,3.8,6.4,2.0,1,0,0},
+ new double[] {6.4,2.8,5.6,2.2,1,0,0},
+ new double[] {6.3,2.8,5.1,1.5,1,0,0},
+ new double[] {6.1,2.6,5.6,1.4,1,0,0},
+ new double[] {7.7,3.0,6.1,2.3,1,0,0},
+ new double[] {6.3,3.4,5.6,2.4,1,0,0},
+ new double[] {6.4,3.1,5.5,1.8,1,0,0},
+ new double[] {6.0,3.0,4.8,1.8,1,0,0},
+ new double[] {6.9,3.1,5.4,2.1,1,0,0},
+ new double[] {6.7,3.1,5.6,2.4,1,0,0},
+ new double[] {6.9,3.1,5.1,2.3,1,0,0},
+ new double[] {5.8,2.7,5.1,1.9,1,0,0},
+ new double[] {6.8,3.2,5.9,2.3,1,0,0},
+ new double[] {6.7,3.3,5.7,2.5,1,0,0},
+ new double[] {6.7,3.0,5.2,2.3,1,0,0},
+ new double[] {6.3,2.5,5.0,1.9,1,0,0},
+ new double[] {6.5,3.0,5.2,2.0,1,0,0},
+ new double[] {6.2,3.4,5.4,2.3,1,0,0},
+ new double[] {5.9,3.0,5.1,1.8,1,0,0}
+ };
+ }
+}
diff --git a/Syntriax/NeuralSystem/NeuralNetwork.cs b/Syntriax/NeuralSystem/NeuralNetwork.cs
new file mode 100644
index 0000000..1cb7c7c
--- /dev/null
+++ b/Syntriax/NeuralSystem/NeuralNetwork.cs
@@ -0,0 +1,39 @@
+namespace Syntriax.NeuralNetwork
+{
+ public class NeuralNetwork
+ {
+ public LayerBase inputLayer { get; private set; } = null;
+ public LayerBase[] hiddenLayers { get; private set; } = null;
+ public LayerBase outputLayer { get; private set; } = null;
+
+ public NeuralNetwork(int inputCount, int[] hiddenCounts = null, int outputCount = 1)
+ {
+ inputLayer = new InputLayer(inputCount);
+
+ if (hiddenCounts != null && hiddenCounts.Length > 0)
+ {
+ int hiddenCount = hiddenCounts.Length;
+
+ hiddenLayers = new HiddenLayer[hiddenCount];
+ hiddenLayers[0] = new HiddenLayer(hiddenCounts[0], inputLayer);
+
+ for (int i = 1; i < hiddenCount; i++)
+ hiddenLayers[i] = new HiddenLayer(hiddenCounts[i], hiddenLayers[i - 1]);
+
+ outputLayer = new HiddenLayer(outputCount, hiddenLayers[hiddenLayers.Length - 1]);
+ return;
+ }
+
+ outputLayer = new OutputLayer(outputCount, inputLayer);
+ }
+
+ public void FireNetwork()
+ {
+ if (hiddenLayers != null)
+ foreach (LayerBase layer in hiddenLayers)
+ layer.FireLayer();
+
+ outputLayer.FireLayer();
+ }
+ }
+}
diff --git a/Syntriax/NeuralSystem/NeuralNetworkExtensions.cs b/Syntriax/NeuralSystem/NeuralNetworkExtensions.cs
new file mode 100644
index 0000000..d527d2f
--- /dev/null
+++ b/Syntriax/NeuralSystem/NeuralNetworkExtensions.cs
@@ -0,0 +1,184 @@
+using System;
+using System.Collections.Generic;
+using Syntriax.NeuralNetwork.Misc;
+
+namespace Syntriax.NeuralNetwork
+{
+ public static class NeuralNetworkExtensions
+ {
+ public static List GetLayerList(this NeuralNetwork neuralNetwork)
+ {
+ List layers = new List(2 + neuralNetwork.hiddenLayers.Length);
+
+ layers.Add(neuralNetwork.inputLayer);
+ layers.AddRange(neuralNetwork.hiddenLayers);
+ layers.Add(neuralNetwork.outputLayer);
+
+ return layers;
+ }
+
+ public static List GetNeuronList(this NeuralNetwork neuralNetwork)
+ {
+ int neuronCount = 0;
+ List layers = neuralNetwork.GetLayerList();
+
+ foreach (LayerBase layer in layers)
+ neuronCount += layer.neurons.Length;
+
+ List neurons = new List(neuronCount);
+
+ foreach (LayerBase layer in layers)
+ neurons.AddRange(layer.neurons);
+
+ return neurons;
+ }
+
+ public static List GetSynapseList(this NeuralNetwork neuralNetwork)
+ {
+ int synapseCount = 0;
+ List neurons = neuralNetwork.GetNeuronList();
+
+ foreach (INeuron neuron in neurons)
+ synapseCount += neuron.Synapses.Length + 1; // + 1 for the bias
+
+ List synapses = new List(synapseCount);
+
+ foreach (INeuron neuron in neurons)
+ {
+ synapses.AddRange(neuron.Synapses);
+ synapses.Add(neuron.Bias);
+ }
+
+ return synapses;
+ }
+
+ /* ------------------------------------------------------------- */
+
+ public static void Randomize(this NeuralNetwork neuralNetwork, int? seed = null)
+ {
+ Random random = new Random(seed ?? 0);
+
+ foreach (ISynapse synapse in neuralNetwork.GetSynapseList())
+ synapse.Weight = random.NextDouble();
+ }
+
+ /* ------------------------------------------------------------- */
+
+ public static void SetInput(this NeuralNetwork neuralNetwork, int neuronIndex, double value)
+ {
+ InputNeuron inputNeuron = (InputNeuron)neuralNetwork.inputLayer.neurons[neuronIndex];
+ inputNeuron.Value = value;
+ }
+ public static void SetInputs(this NeuralNetwork neuralNetwork, double[] inputs)
+ {
+ for (int i = 0; i < inputs.Length; i++)
+ neuralNetwork.SetInput(i, inputs[i]);
+ }
+
+ public static double GetOutput(this NeuralNetwork neuralNetwork, int neuronIndex) =>
+ neuralNetwork.outputLayer.neurons[neuronIndex].Output;
+
+
+ /* ------------------------------------------------------------- */
+
+ public static double GetErrorOfNeuron(NeuralNetwork neuralNetwork, double[] output, int i) =>
+ output[i] - neuralNetwork.GetOutput(i);
+
+ public static double[] GetErrors(this NeuralNetwork neuralNetwork, double[] outputs)
+ {
+ int length = neuralNetwork.outputLayer.neurons.Length;
+ double[] result = new double[length];
+ for (int i = 0; i < length; i++)
+ result[i] = GetErrorOfNeuron(neuralNetwork, outputs, i);
+ return result;
+ }
+
+ public static double GetTotalError(this NeuralNetwork neuralNetwork, Data data)
+ {
+ double totalError = 0.0f;
+ double[] errors = null;
+
+ for (int i = 0; i < data.testInput.Count; i++)
+ {
+ neuralNetwork.SetInputs(data.testInput[i]);
+ neuralNetwork.FireNetwork();
+
+ errors = neuralNetwork.GetErrors(data.testOutput[i]);
+
+ foreach (double error in errors)
+ totalError += error * error;
+ }
+
+ return totalError;
+ }
+
+ /* ------------------------------------------------------------- */
+
+ public static double[] GetSoftMax(this NeuralNetwork neuralNetwork)
+ {
+ // TODO
+ return null;
+ }
+
+ public static int GetMaxIndex(this NeuralNetwork neuralNetwork)
+ {
+ double maxValue = double.MinValue;
+ int maxIndex = 0;
+
+ int count = neuralNetwork.outputLayer.neurons.Length;
+ for (int i = 0; i < count; i++)
+ {
+ double output = neuralNetwork.GetOutput(i);
+ if (output > maxValue)
+ {
+ maxValue = output;
+ maxIndex = i;
+ }
+ }
+
+ return maxIndex;
+ }
+
+ /* ------------------------------------------------------------- */
+
+ public static void Train(this NeuralNetwork neuralNetwork, Data data, double learningRate)
+ {
+ double[] errors = null;
+ for (int i = 0; i < data.trainInput.Count; i++)
+ {
+ neuralNetwork.SetInputs(data.trainInput[i]);
+ neuralNetwork.FireNetwork();
+
+ errors = neuralNetwork.GetErrors(data.trainOutput[i]);
+
+ neuralNetwork.BackPropagate(errors, learningRate);
+ }
+ }
+
+ public static void BackPropagate(this NeuralNetwork neuralNetwork, double[] errors, double learningRate)
+ {
+ for (int i = 0; i < errors.Length; i++)
+ Correct(
+ neuralNetwork.outputLayer.neurons[i],
+ errors[i],
+ learningRate
+ );
+ }
+
+ public static void Correct(this INeuron neuron, double error, double learningRate)
+ {
+ if (neuron.Synapses == null)
+ return;
+
+ double incoming = neuron.NeuronActivation.Derivative(neuron.Output) * error;
+ foreach (ISynapse synapse in neuron.Synapses)
+ Correct(synapse.From, incoming * synapse.Weight, learningRate);
+
+ double delta = incoming * learningRate;
+ foreach (ISynapse synapse in neuron.Synapses)
+ synapse.Weight += delta * synapse.From.Output;
+
+ neuron.Bias.Weight += delta;
+ }
+ }
+}
diff --git a/Syntriax/NeuralSystem/Neuron/DropoutNeuronDecorator.cs b/Syntriax/NeuralSystem/Neuron/DropoutNeuronDecorator.cs
new file mode 100644
index 0000000..ecfbe88
--- /dev/null
+++ b/Syntriax/NeuralSystem/Neuron/DropoutNeuronDecorator.cs
@@ -0,0 +1,23 @@
+using System;
+
+namespace Syntriax.NeuralNetwork
+{
+ public class DropoutNeuronDecorator : NeuronDecorator
+ {
+ private double dropoutRate = 0;
+ public bool IsActive = false;
+ public static Random Random = new Random(0);
+
+ public DropoutNeuronDecorator(INeuron neuron, double dropoutRate) : base(neuron) => this.dropoutRate = dropoutRate;
+
+ public override double Calculate()
+ {
+ base.Calculate();
+
+ if (Random.NextDouble() < dropoutRate)
+ Output = 0.0;
+
+ return Output;
+ }
+ }
+}
diff --git a/Syntriax/NeuralSystem/Neuron/INeuron.cs b/Syntriax/NeuralSystem/Neuron/INeuron.cs
new file mode 100644
index 0000000..4c95384
--- /dev/null
+++ b/Syntriax/NeuralSystem/Neuron/INeuron.cs
@@ -0,0 +1,14 @@
+using Syntriax.NeuralNetwork.NeuronActivations;
+
+namespace Syntriax.NeuralNetwork
+{
+ public interface INeuron
+ {
+ ISynapse[] Synapses { get; }
+ ISynapse Bias { get; }
+ INeuronActivation NeuronActivation { get; set; }
+ double Output { get; }
+
+ double Calculate();
+ }
+}
diff --git a/Syntriax/NeuralSystem/Neuron/Neuron.cs b/Syntriax/NeuralSystem/Neuron/Neuron.cs
new file mode 100644
index 0000000..1eae391
--- /dev/null
+++ b/Syntriax/NeuralSystem/Neuron/Neuron.cs
@@ -0,0 +1,49 @@
+using Syntriax.NeuralNetwork.NeuronActivations;
+
+namespace Syntriax.NeuralNetwork
+{
+ public class Neuron : INeuron
+ {
+ public ISynapse[] Synapses { get; private set; } = null;
+ public ISynapse Bias { get; private set; } = null;
+
+ public INeuronActivation NeuronActivation { get; set; } = new Default();
+
+ public virtual double Output { get; private set; } = 0.0;
+
+ public Neuron(INeuron[] neurons = null)
+ {
+ if (neurons == null)
+ {
+ Synapses = new Synapse[0];
+ Bias = new Synapse();
+ return;
+ }
+
+ Synapses = new ISynapse[neurons.Length];
+ Bias = new Synapse();
+
+ int length = neurons.Length;
+ for (int i = 0; i < length; i++)
+ {
+ Synapses[i] = new Synapse();
+ Synapses[i].From = neurons[i];
+ // Synapses[i] = new MomentumSynapseDecorator(Synapses[i]); // TODO
+ }
+ }
+
+ public double Calculate()
+ {
+ Output = 0.0;
+
+ foreach (ISynapse synapse in Synapses)
+ Output += synapse.Output;
+
+ Output += Bias.Output;
+
+ Output = NeuronActivation.Activation(Output);
+
+ return Output;
+ }
+ }
+}
diff --git a/Syntriax/NeuralSystem/Neuron/NeuronDecorator.cs b/Syntriax/NeuralSystem/Neuron/NeuronDecorator.cs
new file mode 100644
index 0000000..9c6e1b6
--- /dev/null
+++ b/Syntriax/NeuralSystem/Neuron/NeuronDecorator.cs
@@ -0,0 +1,36 @@
+using Syntriax.NeuralNetwork.NeuronActivations;
+
+namespace Syntriax.NeuralNetwork
+{
+ public abstract class NeuronDecorator : INeuron
+ {
+ protected INeuron _neuron = null;
+
+ public ISynapse[] Synapses => _neuron.Synapses;
+ public ISynapse Bias => _neuron.Bias;
+
+ public INeuronActivation NeuronActivation
+ {
+ get => _neuron.NeuronActivation;
+ set => _neuron.NeuronActivation = value;
+ }
+
+ public virtual double Output { get; protected set; } = 0.0;
+
+ protected NeuronDecorator(INeuron neuron) => _neuron = neuron;
+
+ public virtual double Calculate()
+ {
+ Output = 0.0;
+
+ foreach (ISynapse synapse in Synapses)
+ Output += synapse.Output;
+
+ Output += Bias.Output;
+
+ Output = NeuronActivation.Activation(Output);
+
+ return Output;
+ }
+ }
+}
diff --git a/Syntriax/NeuralSystem/NeuronActivations/Default.cs b/Syntriax/NeuralSystem/NeuronActivations/Default.cs
new file mode 100644
index 0000000..3995fcc
--- /dev/null
+++ b/Syntriax/NeuralSystem/NeuronActivations/Default.cs
@@ -0,0 +1,9 @@
+namespace Syntriax.NeuralNetwork.NeuronActivations
+{
+ public class Default : INeuronActivation
+ {
+ public double Activation(double value) => value;
+
+ public double Derivative(double value) => value;
+ }
+}
diff --git a/Syntriax/NeuralSystem/NeuronActivations/INeuronActivation.cs b/Syntriax/NeuralSystem/NeuronActivations/INeuronActivation.cs
new file mode 100644
index 0000000..82798d2
--- /dev/null
+++ b/Syntriax/NeuralSystem/NeuronActivations/INeuronActivation.cs
@@ -0,0 +1,8 @@
+namespace Syntriax.NeuralNetwork.NeuronActivations
+{
+ public interface INeuronActivation
+ {
+ double Activation(double value);
+ double Derivative(double value);
+ }
+}
diff --git a/Syntriax/NeuralSystem/NeuronActivations/Relu.cs b/Syntriax/NeuralSystem/NeuronActivations/Relu.cs
new file mode 100644
index 0000000..2b6ae2d
--- /dev/null
+++ b/Syntriax/NeuralSystem/NeuronActivations/Relu.cs
@@ -0,0 +1,19 @@
+namespace Syntriax.NeuralNetwork.NeuronActivations
+{
+ public class Relu : INeuronActivation
+ {
+ public double Activation(double value)
+ {
+ if (value >= 0.0)
+ return value;
+ return 0.0;
+ }
+
+ public double Derivative(double value)
+ {
+ if (value >= 0.0)
+ return 1.0;
+ return 0.001;
+ }
+ }
+}
diff --git a/Syntriax/NeuralSystem/NeuronActivations/Sigmoid.cs b/Syntriax/NeuralSystem/NeuronActivations/Sigmoid.cs
new file mode 100644
index 0000000..e7c8e9f
--- /dev/null
+++ b/Syntriax/NeuralSystem/NeuronActivations/Sigmoid.cs
@@ -0,0 +1,11 @@
+using System;
+
+namespace Syntriax.NeuralNetwork.NeuronActivations
+{
+ public class Sigmoid : INeuronActivation
+ {
+ public double Activation(double value) => 1.0 / (1.0 + Math.Exp(-value));
+
+ public double Derivative(double value) => value * (1.0 - value);
+ }
+}
diff --git a/Syntriax/NeuralSystem/NeuronActivations/StaticActivation.cs b/Syntriax/NeuralSystem/NeuronActivations/StaticActivation.cs
new file mode 100644
index 0000000..49c42d2
--- /dev/null
+++ b/Syntriax/NeuralSystem/NeuronActivations/StaticActivation.cs
@@ -0,0 +1,11 @@
+using System;
+
+namespace Syntriax.NeuralNetwork.NeuronActivations
+{
+ public class StaticActivation where T : INeuronActivation, new()
+ {
+ private static readonly Lazy instance = new Lazy(() => new T());
+
+ public static T Instance => instance.Value;
+ }
+}
diff --git a/Syntriax/NeuralSystem/NeuronActivations/TanH.cs b/Syntriax/NeuralSystem/NeuronActivations/TanH.cs
new file mode 100644
index 0000000..08a2be9
--- /dev/null
+++ b/Syntriax/NeuralSystem/NeuronActivations/TanH.cs
@@ -0,0 +1,11 @@
+using System;
+
+namespace Syntriax.NeuralNetwork.NeuronActivations
+{
+ public class TanH : INeuronActivation
+ {
+ public double Activation(double value) => (Math.Exp(value) - Math.Exp(-value)) / (Math.Exp(value) + Math.Exp(-value));
+
+ public double Derivative(double value) => 1.0 - value * value;
+ }
+}
diff --git a/Syntriax/NeuralSystem/Synapse/DropoutSynapseDecorator.cs b/Syntriax/NeuralSystem/Synapse/DropoutSynapseDecorator.cs
new file mode 100644
index 0000000..714a11e
--- /dev/null
+++ b/Syntriax/NeuralSystem/Synapse/DropoutSynapseDecorator.cs
@@ -0,0 +1,27 @@
+using System;
+
+namespace Syntriax.NeuralNetwork
+{
+ public class DropoutSynapseDecorator : SynapseDecorator
+ {
+ public bool IsActive = false;
+ public Random random = null;
+
+ public override double Weight
+ {
+ get => base.Weight;
+ set
+ {
+ if (random.Next() % 2 == 0)
+ base.Weight = 0;
+ else
+ base.Weight = value;
+ }
+ }
+
+ public DropoutSynapseDecorator(ISynapse synapse, Random random = null) : base(synapse)
+ => SetRandom(random);
+
+ public void SetRandom(Random random) => this.random = random;
+ }
+}
diff --git a/Syntriax/NeuralSystem/Synapse/ISynapse.cs b/Syntriax/NeuralSystem/Synapse/ISynapse.cs
new file mode 100644
index 0000000..6327cae
--- /dev/null
+++ b/Syntriax/NeuralSystem/Synapse/ISynapse.cs
@@ -0,0 +1,9 @@
+namespace Syntriax.NeuralNetwork
+{
+ public interface ISynapse
+ {
+ INeuron From { get; set; }
+ double Output { get; }
+ double Weight { get; set; }
+ }
+}
diff --git a/Syntriax/NeuralSystem/Synapse/MomentumSynapseDecorator.cs b/Syntriax/NeuralSystem/Synapse/MomentumSynapseDecorator.cs
new file mode 100644
index 0000000..2f42491
--- /dev/null
+++ b/Syntriax/NeuralSystem/Synapse/MomentumSynapseDecorator.cs
@@ -0,0 +1,27 @@
+namespace Syntriax.NeuralNetwork
+{
+ public class MomentumSynapseDecorator : SynapseDecorator
+ {
+ private double _momentum = 0.0;
+ private const double Beta = 0.9;
+
+ public override double Weight
+ {
+ get => base.Weight;
+ set
+ {
+ double difference = value - base.Weight;
+
+ // TODO Might be an error here
+ if (difference * _momentum >= 0.0 == _momentum >= 0.0)
+ base.Weight = value + _momentum * Beta;
+ else
+ base.Weight = value;
+
+ _momentum = difference;
+ }
+ }
+
+ public MomentumSynapseDecorator(ISynapse synapse) : base(synapse) { }
+ }
+}
diff --git a/Syntriax/NeuralSystem/Synapse/Synapse.cs b/Syntriax/NeuralSystem/Synapse/Synapse.cs
new file mode 100644
index 0000000..535b1fb
--- /dev/null
+++ b/Syntriax/NeuralSystem/Synapse/Synapse.cs
@@ -0,0 +1,25 @@
+namespace Syntriax.NeuralNetwork
+{
+
+ public class Synapse : ISynapse
+ {
+ private INeuron from = null;
+ public INeuron From
+ {
+ get => from;
+ set
+ {
+ if (from == null)
+ from = value;
+ }
+ }
+
+ public double Output => Weight * (from == null ? 1.0 : from.Output);
+
+
+ public double Weight { get; set; } = 0.0;
+
+ public Synapse() { }
+ public Synapse(INeuron from) => this.from = from;
+ }
+}
diff --git a/Syntriax/NeuralSystem/Synapse/SynapseDecorator.cs b/Syntriax/NeuralSystem/Synapse/SynapseDecorator.cs
new file mode 100644
index 0000000..56f580d
--- /dev/null
+++ b/Syntriax/NeuralSystem/Synapse/SynapseDecorator.cs
@@ -0,0 +1,16 @@
+namespace Syntriax.NeuralNetwork
+{
+ public abstract class SynapseDecorator : ISynapse
+ {
+ protected ISynapse _synapse = null;
+
+ public INeuron From { get => _synapse.From; set => _synapse.From = value; }
+
+ public virtual double Output => _synapse.Output;
+
+ public virtual double Weight { get => _synapse.Weight; set => _synapse.Weight = value; }
+
+ protected SynapseDecorator(ISynapse synapse) => SetSynapse(synapse);
+ public virtual void SetSynapse(ISynapse synapse) => _synapse = synapse;
+ }
+}