I have a neural network that's trained on a sine wave. It uses a lookback of 20 to see what the last 20 predictions were and predict the next value. This network has only a single Linear layer (input size 20, output size 1) with no activation function and from just that, it is able to extrapolate a sine wave almost perfectly.
This is very confusing to me, as the $sin$ function is non-linear: $sin(a+b) \neq sin(a) + sin(b)$ so the network shouldn't be able to approximate it (very well).
The code to reproduce this is below:
import numpy as np import matplotlib.pyplot as plt import torch import torch.nn as nn from torch.utils.data import DataLoader, Dataset import torch.optim as optim X_train = np.arange(0,100,0.5) y_train = np.sin(X_train) X_test = np.arange(100,200,0.5) y_test = np.sin(X_test) n_features = 1 train_series = torch.from_numpy(y_train) test_series = torch.from_numpy(y_test) # Expects input of (batch, sequence, features) # So shape should be (1, 179, 20) and labels (1, 1, 179) look_back = 20 train_dataset =  train_labels =  for i in range(len(train_series)-look_back): train_dataset.append(train_series[i:i+20]) train_labels.append(train_series[i+20]) train_dataset = torch.stack(train_dataset).unsqueeze(0) train_labels = torch.stack(train_labels).unsqueeze(0).unsqueeze(2) class Net(nn.Module): def __init__(self, input_shape): super(Net, self).__init__() self.fc = nn.Linear(input_shape, 1) def forward(self, x): out = self.fc(x) return out model = Net(look_back).double() loss_function = nn.MSELoss() optimizer = optim.Adam(model.parameters(), lr=0.001) loss_curve =  for epoch in range(300): loss_total = 0 model.zero_grad() predictions = model(train_dataset) loss = loss_function(predictions, train_labels) loss_total += loss.item() loss.backward() optimizer.step() loss_curve.append(loss_total) extrapolation =  seed_batch = test_series[:20].reshape(1, 1, 20) current_batch = seed_batch with torch.no_grad(): for i in range(180): predicted_value = model(current_batch) extrapolation.append(predicted_value.item()) current_batch = torch.cat((current_batch[:,:,1:], predicted_value), axis=2) x = np.arange(110,200,0.5) fig, ax = plt.subplots(1, 1, figsize=(15, 5)) ax.plot(X_train,y_train, lw=2, label='train data') ax.plot(X_test,y_test, lw=3, c='y', label='test data') ax.plot(x,extrapolation, lw=3, c='r',linestyle = ':', label='extrapolation') ax.legend(loc="lower left") plt.show();