To practice what I learned about PyTorch, I gave myself the following problem:
Create a model that given a vector, predicts what the 2nd largest number in it is.
For example, model([ 0.3, 0.4. 0.9 0.7 ]) should return [ 0.0 0.0 0.0 1.0 ]. I am pretty new to neural networks so I tried a feed forward network. However, no matter how large I make the dataset or how many training iterations I do I don't see my loss function decreasing and my model at the end gives pretty much random results. I feel like this should be a pretty easy problem to solve with a neural network, so what am I doing wrong here? Is a feed forward design not appropriate here?
import torch as t
import torch.nn as nn
import torch.optim as optim
# Example: input [ 0.3, 0.4. 0.9 0.7 ] -> [ 0.0 0.0 0.0 1.0 ]
training_set = []
for i in range(10000):
data_point = t.rand(4)
target = sorted(data_point)[-2]
label = t.tensor([ 1. if v == target else 0. for v in data_point ])
training_set.append([data_point, label])
class Model(nn.Module):
def __init__(self, hidden=100):
super(Model, self).__init__()
self.lin1 = nn.Linear(4, hidden)
self.sig = nn.Sigmoid()
self.lin2 = nn.Linear(hidden, 4)
# Simple feed forward network
def forward(self, input):
return self.lin2(self.sig(self.lin1(input)))
model = Model()
criterion = nn.MSELoss()
optimizer = optim.SGD(model.parameters())
for epoch in range(3):
running_loss = 0.0
for i, (input, label) in enumerate(training_set):
optimizer.zero_grad()
pred = model(input) # Forward pass
loss = criterion(pred, label) # Calculate loss
loss.backward() # Backward pass
optimizer.step()
# Print statistics
running_loss += loss.item()
step = 500
if i % step == 0:
print(epoch, i, running_loss / step)
running_loss = 0.0
test = t.rand(4)
print('Input', test)
print('Output', model(test))
