This example trains a 3-layer network using 48 training patterns from four nominal input attributes. The first two nominal attributes have two classifications. The third and fourth nominal attributes have three and four classifications respectively. All four attributes are encoded using binary encoding. This results in eleven binary network input columns. The output class is 1 if the first two nominal attributes sum to 1, and 0 otherwise.
The structure of the network consists of eleven input nodes and three layers, with three perceptrons in the first hidden layer, two perceptrons in the second hidden layer, and one perceptron in the output layer.
There are a total of 47 weights in this network, including the six bias weights. The linear activation function is used for both hidden layers. Since the target output is binary classification the logistic activation function is used in the output layer. Training is conducted using the quasi-newton trainer with the binary-entropy error function provided by the BinaryClassification
class.
using System;
using Imsl.DataMining.Neural;
using Random = Imsl.Stat.Random;
using System.Runtime.Serialization;
using System.Runtime.Serialization.Formatters.Binary;
using PrintMatrix = Imsl.Math.PrintMatrix;
using PrintMatrixFormat = Imsl.Math.PrintMatrixFormat;
//*****************************************************************************
// Two Layer Feed-Forward Network with 11 inputs: 4 nominal with 2,2,3,4
// categories, encoded using binary encoding, and 1 output target (class).
//
// new classification training_ex1.c
//*****************************************************************************
[Serializable]
public class BinaryClassificationEx1
{
// Network Settings
private static int nObs = 48; // number of training patterns
private static int nInputs = 11; // four nominal with 2,2,3,4 categories
private static int nCategorical = 11; // three categorical attributes
private static int nOutputs = 1; // one continuous output (nClasses=2)
private static int nPerceptrons1 = 3; // perceptrons in 1st hidden layer
private static int nPerceptrons2 = 2; // perceptrons in 2nd hidden layer
private static IActivation hiddenLayerActivation =
Imsl.DataMining.Neural.Activation.Linear;
private static IActivation outputLayerActivation =
Imsl.DataMining.Neural.Activation.Logistic;
/* 2 classifications */
private static int[] x1 = new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2};
/* 2 classifications */
private static int[] x2 = new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2};
/* 3 classifications */
private static int[] x3 = new int[]{1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 1,
1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 1,
1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3};
/* 4 classifications */
private static int[] x4 = new int[]{1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1,
2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1,
2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4};
// **********************************************************************
// MAIN
// **********************************************************************
public static void Main(System.String[] args)
{
double[,] xData; // Input Attributes for Trainer
int[] yData; // Output Attributes for Trainer
int i, j; // array indicies
// ******************************************************************
// Binary encode 4 categorical variables.
// Var x1 contains 2 classes
// Var x2 contains 2 classes
// Var x3 contains 3 classes
// Var x4 contains 4 classes
// *******************************************************************
int[,] z1;
int[,] z2;
int[,] z3;
int[,] z4;
UnsupervisedNominalFilter filter = new UnsupervisedNominalFilter(2);
z1 = filter.Encode(x1);
z2 = filter.Encode(x2);
filter = new UnsupervisedNominalFilter(3);
z3 = filter.Encode(x3);
filter = new UnsupervisedNominalFilter(4);
z4 = filter.Encode(x4);
/* Concatenate binary encoded z's */
xData = new double[nObs,nInputs];
yData = new int[nObs];
for (i = 0; i < (nObs); i++)
{
for (j = 0; j < nCategorical; j++)
{
xData[i,j] = 0;
if (j < 2)
xData[i,j] = (double) z1[i,j];
if (j > 1 && j < 4)
xData[i,j] = (double) z2[i,j - 2];
if (j > 3 && j < 7)
xData[i,j] = (double) z3[i,j - 4];
if (j > 6)
xData[i,j] = (double) z4[i,j - 7];
}
yData[i] = ((x1[i] + x2[i] == 2)?1:0);
}
// **********************************************************************
// CREATE FEEDFORWARD NETWORK
// **********************************************************************
long t0 = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
FeedForwardNetwork network = new FeedForwardNetwork();
network.InputLayer.CreateInputs(nInputs);
network.CreateHiddenLayer().CreatePerceptrons(nPerceptrons1);
network.CreateHiddenLayer().CreatePerceptrons(nPerceptrons2);
network.OutputLayer.CreatePerceptrons(nOutputs);
BinaryClassification classification = new BinaryClassification(network);
network.LinkAll();
System.Random r = new System.Random(123457);
network.SetRandomWeights(xData, r);
Perceptron[] perceptrons = network.Perceptrons;
for (i = 0; i < perceptrons.Length - 1; i++)
{
perceptrons[i].Activation = hiddenLayerActivation;
}
perceptrons[perceptrons.Length - 1].Activation = outputLayerActivation;
// **********************************************************************
// TRAIN NETWORK USING QUASI-NEWTON TRAINER
// **********************************************************************
QuasiNewtonTrainer trainer = new QuasiNewtonTrainer();
trainer.Error = classification.Error;
trainer.MaximumTrainingIterations = 1000;
trainer.MaximumStepsize = 3.0;
trainer.GradientTolerance = 1.0e-20;
trainer.StepTolerance = 1.0e-20;
classification.Train(trainer, xData, yData);
// **********************************************************************
// DISPLAY TRAINING STATISTICS
// **********************************************************************
double[] stats = classification.ComputeStatistics(xData, yData);
System.Console.Out.WriteLine(
"***********************************************");
System.Console.Out.WriteLine("--> Cross-entropy error: " +
(float)stats[0]);
System.Console.Out.WriteLine("--> Classification error rate: " +
(float)stats[1]);
System.Console.Out.WriteLine(
"***********************************************");
System.Console.Out.WriteLine("");
// **********************************************************************
// OBTAIN AND DISPLAY NETWORK WEIGHTS AND GRADIENTS
// **********************************************************************
double[] weight = network.Weights;
double[] gradient = trainer.ErrorGradient;
double[,] wg = new double[weight.Length,2];
for (i = 0; i < weight.Length; i++)
{
wg[i,0] = weight[i];
wg[i,1] = gradient[i];
}
PrintMatrixFormat pmf = new PrintMatrixFormat();
pmf.SetColumnLabels(new System.String[]{"Weights", "Gradients"});
new PrintMatrix().Print(pmf, wg);
// ****************************
// forecast the network
// ****************************
double[,] report = new double[nObs,6];
for (i = 0; i < nObs; i++)
{
report[i,0] = x1[i];
report[i,1] = x2[i];
report[i,2] = x3[i];
report[i,3] = x4[i];
report[i,4] = yData[i];
double[] tmp = new double[xData.GetLength(1)];
for ( j=0; j<xData.GetLength(1); j++)
tmp[j] = xData[i,j];
report[i,5] = classification.PredictedClass(tmp);
}
pmf = new PrintMatrixFormat();
pmf.SetColumnLabels( new System.String[]{"X1", "X2", "X3", "X4",
"Expected", "Predicted"});
new PrintMatrix("Forecast").Print(pmf, report);
// **********************************************************************
// DISPLAY CLASSIFICATION STATISTICS
// **********************************************************************
double[] statsClass = classification.ComputeStatistics(xData, yData);
// Display Network Errors
System.Console.Out.WriteLine(
"***********************************************");
System.Console.Out.WriteLine("--> Cross-Entropy Error: " +
(float)statsClass[0]);
System.Console.Out.WriteLine("--> Classification Error: " +
(float)statsClass[1]);
System.Console.Out.WriteLine(
"***********************************************");
System.Console.Out.WriteLine("");
long t1 = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
double time = t1 - t0;
time = time / 1000;
System.Console.Out.WriteLine("****************Time: " + time);
System.Console.Out.WriteLine("trainer.getErrorValue = " +
trainer.ErrorValue);
}
}
***********************************************
--> Cross-entropy error: 4.720552E-10
--> Classification error rate: 0
***********************************************
Weights Gradients
0 1.21627824426795 -1.82357413516947E-12
1 -7.10104582036496 4.00527512119816E-13
2 -4.48633964224764 9.39571675393506E-13
3 -2.60725959847449 4.68033472563643E-09
4 4.29051046748516 -1.0279827879731E-09
5 4.48156618132245 -2.41147856550398E-09
6 2.08243325149059 4.67851114994447E-09
7 -6.86920995138462 -1.02758226011905E-09
8 -4.71797133796275 -2.41053899302647E-09
9 -3.15604455817482 1.55679543755327E-18
10 6.63883077984471 -3.41932577051355E-19
11 4.87196888567896 -8.02117593848719E-19
12 -2.30032598691625 -1.82357569033727E-12
13 -1.68298220558949 4.00527853694902E-13
14 1.57459851986335 9.39572476672484E-13
15 -0.264451161585171 5.95159047420712E-10
16 -0.649412990130757 -1.30719978958719E-10
17 -0.124557044208244 -3.06647573315529E-10
18 0.125744106229942 4.08517567977089E-09
19 -0.550795793591294 -8.97262809355956E-10
20 0.213785518241144 -2.10483099298973E-09
21 -0.256460169762842 1.46860577618452E-15
22 0.719269734081326 -3.22562711586887E-16
23 0.181431096608458 -7.56679074905849E-16
24 0.708887793361652 4.98153020814716E-10
25 -0.13780872548447 -1.09413698206104E-10
26 -0.0714270451586269 -2.56666542556397E-10
27 -1.69080392381724 -1.82357569087194E-12
28 -1.05894442754157 4.00527853812335E-13
29 0.916792419032194 9.39572476947963E-13
30 0.8484018617882 4.18218023777164E-09
31 0.809968676185521 -9.18568767545977E-10
32 -0.621014362842385 -2.15481126707006E-09
33 3.92683360366486 -1.53010895369397E-09
34 3.92683360375881 -1.53010895423418E-09
35 -0.862484756532178 6.25095768828527E-10
36 -0.862484756327281 6.2509576904922E-10
37 -2.02324740024013 -1.05620718555491E-09
38 -2.02324740013514 -1.05620718592781E-09
39 1.26293423975155 -5.17748567942779E-09
40 1.26293424019744 -5.17748567954339E-09
41 -2.95381709009598 4.67851115150126E-09
42 3.4767065441968 -1.02758226046098E-09
43 -1.14723052699379 -2.41053899382858E-09
44 -3.57249695785368 5.9571038944113E-10
45 -3.57249695788631 5.95710389651447E-10
46 5.5108803426636 4.71687575402437E-10
Forecast
X1 X2 X3 X4 Expected Predicted
0 1 1 1 1 1 1
1 1 1 1 2 1 1
2 1 1 1 3 1 1
3 1 1 1 4 1 1
4 1 1 2 1 1 1
5 1 1 2 2 1 1
6 1 1 2 3 1 1
7 1 1 2 4 1 1
8 1 1 3 1 1 1
9 1 1 3 2 1 1
10 1 1 3 3 1 1
11 1 1 3 4 1 1
12 1 2 1 1 0 0
13 1 2 1 2 0 0
14 1 2 1 3 0 0
15 1 2 1 4 0 0
16 1 2 2 1 0 0
17 1 2 2 2 0 0
18 1 2 2 3 0 0
19 1 2 2 4 0 0
20 1 2 3 1 0 0
21 1 2 3 2 0 0
22 1 2 3 3 0 0
23 1 2 3 4 0 0
24 2 1 1 1 0 0
25 2 1 1 2 0 0
26 2 1 1 3 0 0
27 2 1 1 4 0 0
28 2 1 2 1 0 0
29 2 1 2 2 0 0
30 2 1 2 3 0 0
31 2 1 2 4 0 0
32 2 1 3 1 0 0
33 2 1 3 2 0 0
34 2 1 3 3 0 0
35 2 1 3 4 0 0
36 2 2 1 1 0 0
37 2 2 1 2 0 0
38 2 2 1 3 0 0
39 2 2 1 4 0 0
40 2 2 2 1 0 0
41 2 2 2 2 0 0
42 2 2 2 3 0 0
43 2 2 2 4 0 0
44 2 2 3 1 0 0
45 2 2 3 2 0 0
46 2 2 3 3 0 0
47 2 2 3 4 0 0
***********************************************
--> Cross-Entropy Error: 4.720552E-10
--> Classification Error: 0
***********************************************
****************Time: 0.359
trainer.getErrorValue = 4.72055172799E-10
Link to C# source.