Neuronal Network revisited

[Theo Gottwald]

This code simulates a learning Neuronal Network. To understand what Neural Networks can be  used for think like this:
1. You need a program that converts Binary Numbers (e.g. "0101" to dezimal number e.g. "5").
2. You are too lazy to find an appropriate Algorhytm, or the Algo is too complex (like for example in Face recognition).

So what you do is, you set up an Neuronal Network. See it like a "Self configuring Algorhytm".
Set up properly it will search itself the right "Formula" to solve your problem.

Here its the "Binary to Decimal" Conversion. Which is just an example.


It has 4 Input Neurons, 8 Neurons in the Hidden Layer and 8 Output-Neurons.

0 I ---- H ---- O1
1 I ---- H ---- O2
0 I ---- H ---- O3
1 I ---- H ---- O4

The Input Neurons are given a Binary Number from 1 to 8 (The Input Neurons are set to 0011 for "3" for example).
Then The Output Neuron O3 must have the highest value of all Output Neurons.

When the Net starts the result-values are rather random. After some learning, the net recognizes all binary numbers from 1 to 8.

Note that this version does not have any visible Output. You need to add your "Debug Print" into X_AU () to see something.

Code Select Expand


'***********************************************************************************************
'
'
'***********************************************************************************************

#DEBUG ERROR on
#COMPILE EXE

'#INCLUDE ONCE "private\G_MainLib.inc"

MACRO DebS(P1)=STR$(INT(P1*100000)/100000)
MACRO INST = INSTANCE
MACRO G_REG()
REGISTER R01 AS LONG,R02 AS LONG
END MACRO
MACRO G_S01()
LOCAL S01 AS STRING
END MACRO
MACRO G_S02()
LOCAL S01,S02 AS STRING
END MACRO

MACRO G_S03()
LOCAL S01,S02,S03 AS STRING
END MACRO

' Your Debug Print comes in here
SUB X_AU(BYVAL P1 AS STRING)
  ' Print String P1
END SUB

'***********************************************************************************************
GLOBAL NA AS NeoNet_Interface

FUNCTION PBMAIN () AS LONG

NA=CLASS "NeoNet"
NA.TrainNet()

END FUNCTION


'***********************************************************************************************

'***********************************************************************************************
CLASS NeoNet
INST ACnt,BCnt,OCnt AS DWORD
INST LearnSteps AS LONG
INST LearnRate AS SINGLE
INST Inputs(), Outputs() AS SINGLE
INST Epsilon AS SINGLE
INST WeightsInput(),WeightsHidden() AS SINGLE
INST BiasHidden(),BiasOut(),HIDDEN() AS SINGLE
INST DeltaHidden(),DeltaOut(),Target() AS SINGLE
INST Net() AS BYTE
INST Total_Error AS SINGLE
' Minimaler Error bei X, Largest Output bei X
INST LAON,EMIN AS LONG
' Soll-Ausgabe
INST Soll AS LONG
'-------------------------------

CLASS METHOD CREATE()

END METHOD
'-------------------------------

'-------------------------------
INTERFACE NeoNet_Interface
INHERIT IUNKNOWN

'-------------------------------

'-------------------------------
METHOD TrainNet()
  G_REG
  G_S01
    ME.Init_Mem()
    ME.InitWeights()
FOR R02=1 TO LearnSteps
    FOR R01=1 TO OCnt
      ME.SetInputsAndTargets(R01)
      ME.FeedForward()
      ME.FeedBackward()
      ME.DrawNet(R02)
    NEXT
NEXT
END METHOD
'-------------------------------
PROPERTY GET Inputs(BYVAL A1 AS LONG) AS SINGLE
PROPERTY=Inputs(A1)
END PROPERTY

PROPERTY SET Inputs(BYVAL A1 AS LONG,BYREF A2 AS SINGLE)
Inputs(A1)=A2
END PROPERTY
'-------------------------------
END INTERFACE
'-------------------------------
CLASS METHOD Init_Mem()
  ME.InitVars()
  DIM Inputs(ACnt)
  DIM Outputs(OCnt),Target(OCnt),BiasOut(OCnt)
  DIM WeightsInput(BCnt,ACnt),WeightsHidden(OCnt,BCnt)
  DIM BiasHidden(BCnt),DeltaHidden(BCnt),HIDDEN(BCnt)
  DIM DeltaOut(OCnt)
END METHOD
'-------------------------------
' A1 - Pass-Nummer
CLASS METHOD DrawNet(BYVAL A1 AS DWORD)
REGISTER R01 AS DWORD,R02 AS LONG
G_S02
X_AU "Pass-Nr."+STR$(A1)+" Total Error: "+TRIM$(Total_Error)
S01="":S02=""
FOR R01=0 TO ACnt
  S01+="I("+TRIM$(R01)+")="+TRIM$(Inputs(R01))+" -- "
NEXT r01
X_AU S01

S01=" Soll="+TRIM$(Soll)+"   Ist="+TRIM$(LAON)+"  with: "+TRIM$(Outputs(LAON))
X_AU S01
X_AU "-------------------------------------------------------------------"
'Sleep 10
END METHOD
'-------------------------------
CLASS METHOD InitVars()
G_REG
  ACnt=4:BCnt=8:OCnt=8
  LearnSteps=130000
  LearnRate=0.03
  Epsilon=LearnRate
END METHOD
'-------------------------------
CLASS METHOD Activation(BYREF A1 AS SINGLE) AS SINGLE
LOCAL S1 AS SINGLE
S1=1/(1+EXP(-A1))
METHOD=S1
END METHOD
'-------------------------------
CLASS METHOD Abl(BYREF P1 AS SINGLE) AS SINGLE
LOCAL S1 AS SINGLE
S1=P1*(1-P1)
  METHOD=S1
END METHOD
'-------------------------------
CLASS METHOD InitWeights()
  REGISTER i AS LONG
  REGISTER j AS LONG
  FOR i=0 TO OCnt
    FOR j=0 TO BCnt
      WeightsHidden(i,j)=RND()*2 - 1
    NEXT
    BiasOut(i)=RND()*2 - 1
  NEXT

  FOR i=0 TO BCnt
    FOR j=0 TO ACnt
      WeightsInput(i,j)=RND()*2 - 1
    NEXT
    BiasHidden(i) =RND()*2 - 1
  NEXT
END METHOD
'-------------------------------
CLASS METHOD SetInputsAndTargets(BYVAL A1 AS LONG)
  G_REG
  G_S01
  S01=BIN$(A1,4)
  FOR R01=1 TO 4
    Inputs(R01)=VAL(MID$(S01,R01,1))
  NEXT r01

  FOR R01=1 TO OCnt
    Target(R01) = 0
  NEXT
Target(A1)=1
Soll=A1
END METHOD

'-------------------------------
CLASS METHOD FeedForward()
  REGISTER i AS LONG
  REGISTER j AS LONG
  LOCAL aH,aI AS SINGLE

  FOR i = 0 TO BCnt
    aI=0
    FOR j=0 TO ACnt
      aI+=Inputs(j) * WeightsInput (i,j)
    NEXT
    HIDDEN(i)  = ME.Activation(aI+BiasHidden(i))
  NEXT

  FOR i = 0 TO OCnt
    aH=0
    FOR j=0 TO BCnt
      aH+=HIDDEN(j) * WeightsHidden(i,j)
    NEXT
    Outputs(i) = ME.Activation(aH+BiasOut(i)   )
  NEXT

END METHOD

'-------------------------------
CLASS METHOD FeedBackward()
  REGISTER i AS LONG
  REGISTER j AS LONG
  LOCAL S1,S2 AS SINGLE
  ' Errormin und Largest Output
  LOCAL EMI,LAO AS SINGLE
  EMI=1:LAO=-1:EMIN=-1:LAON=-1
  FOR i=0 TO OCnt
    S1=Target(i)-Outputs(i)
    S2+=ABS(S1)
    DeltaOut(i) = S1 * ME.Abl(Outputs(i))
    IF S1<EMI THEN EMI=S1:EMIN=i
    IF Outputs(i)>LAO THEN LAO=Outputs(i):LAON=i
  NEXT
    Total_Error=S2

  FOR i=0 TO OCnt
    FOR j=0 TO BCnt
      WeightsHidden(i,j)+=(Epsilon*DeltaOut(i)*HIDDEN(j))
    NEXT
    BiasOut(i)+=(Epsilon*DeltaOut(i))
  NEXT

  FOR i=0 TO OCnt
    DeltaHidden(i)=0
    FOR j=0 TO BCnt
      DeltaHidden(i)+=(DeltaOut(j)*WeightsHidden(i,j))
    NEXT
    DeltaOut(i)*=ME.Abl(HIDDEN(i))
  NEXT

  FOR i=0 TO BCnt
    FOR j=0 TO ACnt
      WeightsInput(i,j)=WeightsInput(i,j) + (Epsilon * DeltaHidden(i) * Inputs(j))
    NEXT
    BiasHidden(i)+=(Epsilon * DeltaHidden(i))
  NEXT
END METHOD
END CLASS   



[Theo Gottwald]

Here is the next logical Step. This program gets Binary Numbers from 0 to 127 from Inputs (0 to 5).
It will then Activate the Output-Neuron with that number .... after long time of learning.

Let it run for an hour before you read what the Output is.
It also demonstrates why larger Neuronal Nets are often trained usingthe GPU.
Because it takes too much time using the CPU.

Code Select Expand

#COMPILE EXE

'#INCLUDE ONCE "private\G_MainLib.inc"

MACRO DebS(P1)=STR$(INT(P1*100000)/100000)
MACRO INST = INSTANCE
MACRO G_REG()
REGISTER R01 AS LONG,R02 AS LONG
END MACRO
MACRO G_S01()
LOCAL S01 AS STRING
END MACRO
MACRO G_S02()
LOCAL S01,S02 AS STRING
END MACRO

MACRO G_S03()
LOCAL S01,S02,S03 AS STRING
END MACRO

' Your Debug Print comes in here
SUB X_AU(BYVAL P1 AS STRING)
  ' Print String P1
END SUB

'***********************************************************************************************
GLOBAL NA AS NeoNet_Interface

FUNCTION PBMAIN () AS LONG

NA=CLASS "NeoNet"
NA.TrainNet()

END FUNCTION


'***********************************************************************************************

'***********************************************************************************************
Class NeoNet
INST ACnt,BCnt,OCnt as dword
INST LearnSteps as long
INST LearnRate as single
INST Inputs(), Outputs() as single
INST Epsilon as single
INST WeightsInput(),WeightsHidden() as single
INST BiasHidden(),BiasOut(),Hidden() as single
INST DeltaHidden(),DeltaOut(),Target() as single
INST TE() as SIngle
INST Epsilondec as Single
INST Net() as byte
INST Total_Error as single
INST Last_Error as single
INST LastChange as Dword
' Minimaler Error bei X, Largest Output bei X
INST LAON,EMIN as long
' Soll-Ausgabe
INST Soll as long
' Pass Nr.
Inst Pass as dword
' Letzter Totaler Fehler über alle Eingaben
INST Last_TE as SIngle
' Richtige Ergebnise
INST Ri as dword
Inst Ok() as Byte
'-------------------------------
'
CLASS METHOD CREATE()

end method
'-------------------------------

'-------------------------------
interface NeoNet_Interface
Inherit Iunknown

'-------------------------------

'-------------------------------
Method TrainNet()
  G_REG
  G_S01
X_AU "Hallo"
    ME.Init_Mem()
    ME.InitWeights()
  FOr R02=1 to LearnSteps
    Pass=R02
    for R01=0 to OCnt
      ME.SetInputsAndTargets(R01)
      ME.FeedForward()
      ME.FeedBackward()
      ME.DrawNet(R02)
    next
    ME.CheckEpsilon()
next
end method
'-------------------------------
Property Get Inputs(byval A1 as long) as Single
Property=Inputs(A1)
end property

Property Set Inputs(byval A1 as long,byref A2 as Single)
Inputs(A1)=A2
end property
'-------------------------------

end interface
'-------------------------------
class method InitVars()
G_REG
' Outp=1.23*Hidden
  ACnt=6:BCnt=360:OCnt=127
  LearnSteps=1300000
  LearnRate=0.9
  Epsilon=LearnRate
  EpsilonDec=0.999
end method
'-------------------------------
class method Init_Mem()
  ME.InitVars()
  dim Inputs(ACnt)
  dim Outputs(OCnt),Target(OCnt),BiasOut(OCnt)
  dim WeightsInput(BCnt,ACnt),WeightsHidden(OCnt,BCnt)
  dim BiasHidden(BCnt),DeltaHidden(BCnt),Hidden(BCnt)
  dim DeltaOut(OCnt),TE(OCnt),Ok(OCnt)
end method
'-------------------------------
' A1 - Pass-Nummer
class Method DrawNet(byval A1 as dword)
Register R01 as DWORD,R02 as Long
G_T02
G_S04

X_AU "Pass-Nr."+STR$(A1)+" Total Error: "+DebS(Total_Error)+" -- Epsilon="+DebS(Epsilon)+"  Last_TE="+DebS(Last_TE)
if Last_TE>1 or Total_Error>0.1 then exit method
#if 0
S01="":S02=""
For R01=0 to ACnt
  S01+="I("+trim$(R01)+")="+trim$(Inputs(R01))+" -- "
next r01
X_AU S01
#ENDIF

S01="":S02="":S03="":T01=0:T02=0
For R01=0 to OCnt
'   S01+="O("+trim$(R01)+")="+DebS(Outputs(R01))+" -- "
'   S02+="Target("+trim$(R01)+")="+DebS(Target(R01))+" -- "
   if Ok(R01)=1 then S04="OK":incr T02 else S04="ER"
   S03+=trim$(R01)+"="+S04+"**"   
   incr T01
   if T01>16 then S03+=$crlf:T01=0
next r01
S03+=$crlf+"Total: "+STR$(T02)+"/"+STR$(OCnt+1)
if T02=OCnt+1 then SLEEP 4000
Ri=T02
if Ri<(OCnt/2) then exit method
'X_AU S01
'X_AU S02
X_AU S03


S01=" Soll="+trim$(Soll)+"   Ist="+trim$(LAON)+"  with: "+trim$(Outputs(LAON))
X_AU S01
X_AU "-------------------------------------------------------------------"
'Sleep 10
end method

'-------------------------------
class method Activation(byref A1 as single) as single
local S1 as Single
S1=1/(1+exp(-A1))
method=S1
end method
'-------------------------------
clASs Method Abl(byref P1 as single) as SIngle
local S1 as Single
S1=P1*(1-P1)
  method=S1
end method
'-------------------------------
class method InitWeights()
  Register i as long
  Register j as long
  for i=0 to OCnt
    for j=0 to BCnt
      WeightsHidden(i,j)=rnd()*2 - 1
    next
    BiasOut(i)=rnd()*2 - 1
  next

  for i=0 to BCnt
    for j=0 to ACnt
      WeightsInput(i,j)=rnd()*2 - 1
    next
    BiasHidden(i) =rnd()*2 - 1
  next
end method
'-------------------------------
class Method SetInputsAndTargets(byval A1 as Long)
  G_REG
  G_S01
  S01=BIN$(A1,ACnt+1)
  FOR R01=0 TO ACnt
    Inputs(R01)=VAL(MID$(S01,R01+1,1))
  NEXT r01

  For R01=0 to OCnt
    Target(R01) = 0
  next
Target(A1)=1
Soll=A1
end method

'-------------------------------
class method FeedForward()
  Register i as long
  Register j as long
  local aH,aI as SIngle

  for i = 0 to BCnt
    aI=0
    for j=0 to ACnt
      aI+=Inputs(j) * WeightsInput (i,j)
    next
    Hidden(i)  = ME.Activation(aI+BiasHidden(i))
  next

  for i = 0 to OCnt
    aH=0
    for j=0 to BCnt
      aH+=Hidden(j) * WeightsHidden(i,j)
    next
    Outputs(i) = ME.Activation(aH+BiasOut(i))
  next

end method

'-------------------------------
class method FeedBackward()
  Register i as long
  Register j as long
  Local S1,S2 as single
  ' Errormin und Largest Output
  Local EMI,LAO as Single
  EMI=1:LAO=-1:EMIN=-1:LAON=-1
  for i=0 to OCnt
    S1=Target(i)-Outputs(i)
    S2+=ABS(S1)
    DeltaOut(i) = S1 * ME.Abl(Outputs(i))
    if S1<EMI then EMI=S1:EMIN=i
    if Outputs(i)>LAO  then LAO=Outputs(i):LAON=i
  next
    If Soll=LAON then OK(Soll)=1 else Ok(Soll)=0
    Total_Error=S2:TE(Soll)=S2
 
  for i=0 to OCnt
    for j=0 to BCnt
      WeightsHidden(i,j)+=(Epsilon*DeltaOut(i)*Hidden(j))
    next
    BiasOut(i)+=(Epsilon*DeltaOut(i))
  next

  for i=0 to OCnt
    DeltaHidden(i)=0
    for j=0 to BCnt
      DeltaHidden(i)+=(DeltaOut(j)*WeightsHidden(i,j))
    next
    DeltaOut(i)*=ME.Abl(Hidden(i))
  next

  for i=0 to BCnt
    for j=0 to ACnt
      WeightsInput(i,j)=WeightsInput(i,j) + (Epsilon * DeltaHidden(i) * Inputs(j))
    next
    BiasHidden(i)+=(Epsilon * DeltaHidden(i))
  next
end method
'-------------------------------
class Method CheckEpsilon()
G_REG
Local S1 as Single
if Epsilon<0.002 then exit method
For R01=0 to OCnt
   S1+=TE(R01)
next r01
S1=S1/OCnt
X_AU "Pass: "+trim$(Pass)+" Total Error="+STR$(S1)+" Epsilon="+DebS(Epsilon)
if Last_TE<S1 then Epsilon*=Epsilondec
Last_TE=S1
end method
END CLASS