# Why is my SVM not reaching good accuracy when trained to perform binary classification of search results?

I am trying to perform binary classification of search results based on the relevance to the query. I followed this tutorial on how to make an SVM, and I got it to work with a small iris dataset. Now, I am attempting to use the LETOR 4.0 MQ2007 dataset by Microsoft to classify. The dataset has 21 input vectors as well as a score from 0 to 2. I classified 0 as -1 and 1, 2 as 1. My algorithm reaches 57.4% accuracy after 1000 epochs with 500 samples of each classification. My learning rate is 0.0001. here is my code.

from tqdm import tqdm
import numpy as np
from sklearn.metrics import accuracy_score

print("-------------------------------------")
choice = input("Train or Test: ")
print("-------------------------------------")

# HYPERPARAMETERS
feature_num = 21
epochs = 1000
sample_size = 500
learning_rate = 0.0001

if choice == "Train":

out_file = open('weights.txt', 'w')
out_file.close()

print("Serializing Train Data...")

# SERIALIZE DATA
file = open('train.txt')
positive = []
negative = []

# GRAB TRAINING SAMPLES
for i in train_set:
if (i[0] == '1' or i[0] == '2') and len(positive) < sample_size:
positive.append(i)
if (i[0] == '0') and len(negative) < sample_size:
negative.append(i)

train_set = positive+negative
file.close()

features = []
query = []

# CREATE TRAINING VECTORS
alpha = np.full(feature_num, learning_rate)
weights = np.zeros((len(train_set), feature_num))
output = np.zeros((len(train_set), feature_num))
score = np.zeros((len(train_set), feature_num))

for i in tqdm(range(len(train_set))):
elements = train_set[i].split(' ')
if int(elements[0]) == 0:
score[i] = [-1] * feature_num
else:
score[i] = [1] * feature_num

query.append(int(elements[1].split(':')[1]))
tmp = []
for feature in elements[2:2+feature_num]:
if feature.split(':')[1] == 'NULL':
tmp.append(0.0)
else:
tmp.append(float(feature.split(':')[1]))
features.append(tmp)

features = np.asarray(features)

print("-------------------------------------")
print("Training Initialized...")

# TRAIN MODEL
for i in tqdm(range(epochs)):

# FORWARD y = sum(wx)
for sample in range(len(train_set)):
output[sample] = weights[sample]*features[sample]
output[sample] = np.full((feature_num), np.sum(output[sample]))

# NORMALIZE NEGATIVE SIGNS
output = output*score
# UPDATE WEIGHTS
count = 0
for val in output:
if(val[0] >= 1):
cost = 0
weights = weights - alpha * (2 * 1/epochs * weights)
else:
cost = 1 - val[0]
# WEIGHTS = WEIGHTS + LEARNING RATE * [X] * [Y]
weights = weights + alpha * (features[count] * score[count] - 2 * 1/epochs * weights)

count += 1

# EXPORT WEIGHTS
out_file = open('weights.txt', 'a+')
for i in weights[0]:
out_file.write(str(i)+'\n')
out_file.close()

elif choice == "Test":

print("Serializing Test Data...")

# SERIALIZE DATA
file = open('train.txt')
positive = []
negative = []
for i in train_set:
if (i[0] == '1' or i[0] == '2') and len(positive) < sample_size:
positive.append(i)
if (i[0] == '0') and len(negative) < sample_size:
negative.append(i)

test_set = positive+negative

weights = np.zeros((len(test_set), feature_num))

# CREATE TEST SET
for i in range(len(weights)):
weights[i] = file
features = []
query = []
output = np.zeros((len(test_set), feature_num))
score = np.zeros((len(test_set)))

for i in tqdm(range(len(test_set))):
elements = test_set[i].split(' ')
if int(elements[0]) == 0:
score[i] = -1
else:
score[i] = 1

query.append(int(elements[1].split(':')[1]))
tmp = []
for feature in elements[2:2+feature_num]:
if feature.split(':')[1] == 'NULL':
tmp.append(0.0)
else:
tmp.append(float(feature.split(':')[1]))
features.append(tmp)

features = np.asarray(features)

for sample in range(len(test_set)):
output[sample] = weights[sample]*features[sample]
output[sample] = np.full((feature_num), np.sum(output[sample]))

predictions = []
for val in output:
if(val[0] > 1):
predictions.append(1)
else:
predictions.append(-1)

print("-------------------------------------")
print("Predicting...")
print("-------------------------------------")
print("Prediction finished with "+str(accuracy_score(score, predictions)*100)+"% accuracy.")



My training algorithm

if(val[0] >= 1):
cost = 0
weights = weights - alpha * (2 * 1/epochs * weights)
else:
cost = 1 - val[0]
# WEIGHTS = WEIGHTS + LEARNING RATE * [X] * [Y]
weights = weights + alpha * (features[count] * score[count] - 2 * 1/epochs * weights)



What could I do to help the model train? Am I not giving it enough time? Is the algorithm wrong? Are the hyperparameters ok? Thanks for all your help.

• Can you clarify this part "I classified 0 as -1 and 1, 2 as 1"? Does it mean that the original datasets has 3 classes and you're joining two classes so that you can perform binary classification? Also, you have only a total of 1k training examples? That's a relatively small dataset.
– nbro
Apr 7 '20 at 1:22
• Yes. The samples are scored 0 - 2, so I classified 0 as -1, and 1 and 2 as 1. Is this not a good way to do it? What would be an adequate sample size? Is my training algorithm ok? Apr 7 '20 at 3:21