# Which neural network can approximate the function $y = x^2 + b$?

I am new to ANN. I am trying out several 'simple' algorithms to see what ANN can (or cannot) be used for and how. I played around with Conv2d once and had it recognize images successfully. Now I am looking into trend line analyses. I have succeeded in training a network where it solved for linear equations. Now I am trying to see if it can be trained to solve for $$y$$ in the formula $$y = b + x^2$$.

No matter what parameters I change, or the number of dense layers, I get high values for loss and validation loss, and the predictions are incorrect.

Is it possible to solve this equation, and with what network? If it is not possible, why not? I am not looking to solve a practical problem, rather build up understanding and intuition about ANNs.

See the code I tried with below

#region Imports
from __future__ import absolute_import, division, print_function, unicode_literals
import math
import numpy as np
import tensorflow as tf
from tensorflow.keras import models, optimizers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Activation, Dropout, Flatten, Dense, Lambda
import tensorflow.keras.backend as K
#endregion

#region Constants
learningRate = 0.01
epochs: int = 1000
batch_size = None
trainingValidationFactor = 0.75
nrOfSamples = 100
activation = None
#endregion

#region Function definitions
def CreateNetwork(inputDimension):
model = Sequential()
# sgd = optimizers.SGD(lr=learningRate, decay=1e-6, momentum=0.9, nesterov=True)
return model

def SplitDataForValidation(factor, data, labels):
upperBoundary = int(len(data) * factor)

trainingData = data[:upperBoundary]
trainingLabels = labels[:upperBoundary]

validationData = data[upperBoundary:]
validationLabels = labels[upperBoundary:]
return ((trainingData, trainingLabels), (validationData, validationLabels))

def Train(network, training, validation):
trainingData, trainingLabels = training
history = network.fit(
trainingData
,trainingLabels
,validation_data=validation
,epochs=epochs
,batch_size=batch_size
)

return history

def subtractMean(data):
mean = np.mean(data)
data -= mean
return mean

def rescale(data):
max = np.amax(data)
factor = 1 / max
data *= factor
return factor

def Normalize(data, labels):
dataScaleFactor = rescale(data)
dataMean = subtractMean(data)

labels *= dataScaleFactor
labelsMean = np.mean(labels)
labels -= labelsMean

def Randomize(data, labels):
rng_state = np.random.get_state()
np.random.shuffle(data)
np.random.set_state(rng_state)
np.random.shuffle(labels)

def CreateTestData(nrOfSamples):
data = np.zeros(shape=(nrOfSamples,2))
labels = np.zeros(nrOfSamples)

for i in range(nrOfSamples):
for j in range(2):
randomInt = np.random.randint(1, 5)
data[i, j] = (randomInt * i) + 10
labels[i] = data[i, 0] + math.pow(data[i, 1], 2)

Randomize(data, labels)
return (data, labels)
#endregion

allData, allLabels = CreateTestData(nrOfSamples)
Normalize(allData, allLabels)
training, validation = SplitDataForValidation(trainingValidationFactor, allData, allLabels)

inputDimension = np.size(allData, 1)
network = CreateNetwork(inputDimension)

history = Train(network, training, validation)

prediction = network.predict([
[2, 2], # Should be 2 + 2 * 2 = 6
[4, 7], # Should be 4 + 7 * 7 = 53
[23, 56], # Should be 23 + 56 * 56 = 3159
[128,256] # Should be 128 + 256 * 256 = 65664
])
print(str(prediction))


$$f(x) = x^2 + b$$ is a polynomial (more precisely, a parabola) so it is continuous, thus, a neural network (with at least one hidden layer) should be able to approximate that function (given the universal approximation theorem).
After a very quick look at your code, I noticed you aren't using an activation function for your dense layers (i.e. your activation function is None). Try to use e.g. ReLU.