2
$\begingroup$

Context

I was making a Transformer Model to convert English Sentences to German Sentences. But the loss stops reducing after some time.

Code

import string
import re
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Embedding, LSTM, RepeatVector, Dense, Dropout, BatchNormalization, TimeDistributed, AdditiveAttention, Input, Concatenate, Flatten
from tensorflow.keras.layers import Activation, LayerNormalization, GRU, GlobalAveragePooling1D, Attention
from tensorflow.keras.optimizers import Adam
from tensorflow.nn import tanh, softmax
import time
from tensorflow.keras.losses import SparseCategoricalCrossentropy, CategoricalCrossentropy
from numpy import array
from tensorflow.keras.utils import plot_model
from sklearn.utils import shuffle
import time
import tensorflow as tf
from numpy import array
import numpy as np
from tensorflow.keras.models import load_model
from tensorflow.keras.datasets.imdb import load_data

def load_data(filename):
    file = open(filename, 'r')
    text = file.read()
    file.close()
    return text

def to_lines(text):
    return text.split('\n')

def clean_data(pair):
    pair = 'start_seq_ ' + pair + ' end_seq_'

    re_print = re.compile('[^%s]' % re.escape(string.printable))
    table = str.maketrans('', '', string.punctuation)
    tokens = [token.translate(table) for token in pair.split()]
    tokens = [token.lower() for token in tokens]
    tokens = [re_print.sub('', token) for token in tokens]
    tokens = [token for token in tokens if token.isalpha()]
    return tokens

lines = to_lines(load_data('/content/drive/My Drive/spa.txt'))

english_pair = []
german_pair = []
language = []
for line in lines:
    if line != '':
        pairs = line.split('\t')
        english_pair.append(clean_data(pairs[0]))
        german_pair.append(clean_data(pairs[1]))

        language.append(clean_data(pairs[0]))
        language.append(clean_data(pairs[1]))

english_pair = array(english_pair)
german_pair = array(german_pair)
language = array(language)

def create_tokenizer(data):
    tokenizer = Tokenizer()
    tokenizer.fit_on_texts(data)
    return tokenizer

def max_len(lines):
    length = []
    for line in lines:
        length.append(len(line))
    return max(length)

tokenizer = create_tokenizer(language)

vocab_size = len(tokenizer.word_index) + 1

max_len = max_len(language)

def create_sequences(sequences, max_len):
    sequences = tokenizer.texts_to_sequences(sequences)
    sequences = pad_sequences(sequences, maxlen=max_len, padding='post')
    return sequences

X1 = create_sequences(english_pair, max_len)
X2 = create_sequences(german_pair, max_len)
Y = create_sequences(german_pair, max_len)


X1, X2, Y = shuffle(X1, X2, Y)

training_samples = int(X1.shape[0] * 1.0)

train_x1, train_x2, train_y = X1[:training_samples], X2[:training_samples], Y[:training_samples]
test_x1, test_x2, test_y = X1[training_samples:], X2[training_samples:], Y[training_samples:]

train_x2 = train_x2[:, :-1]
test_x2 = test_x2[:, :-1]
train_y = train_y[:, 1:].reshape(-1, max_len-1)
test_y = test_y[:, 1:].reshape(-1, max_len-1)

train_x2 = pad_sequences(train_x2, maxlen=max_len, padding='post')
test_x2 = pad_sequences(test_x2, maxlen=max_len, padding='post')

train_y = pad_sequences(train_y, maxlen=max_len, padding='post')
test_y = pad_sequences(test_y, maxlen=max_len, padding='post')

All code above just prepares the Data, so if you want you can skip that part. Code After this starts implementing the Transformer Model.

class EncoderBlock(tf.keras.layers.Layer):
    def __init__(self, mid_ffn_dim, embed_dim, num_heads, max_len, batch_size):
        super(EncoderBlock, self).__init__()
        # Variables
        self.batch_size = batch_size
        self.max_len = max_len
        self.mid_ffn_dim = mid_ffn_dim
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.attention_vector_len = self.embed_dim // self.num_heads
        if self.embed_dim % self.num_heads != 0:
            raise ValueError('I am Batman!')

        # Trainable Layers
        self.mid_ffn = Dense(self.mid_ffn_dim, activation='relu')
        self.final_ffn = Dense(self.embed_dim)

        self.layer_norm1 = LayerNormalization(epsilon=1e-6)
        self.layer_norm2 = LayerNormalization(epsilon=1e-6)

        self.combine_heads = Dense(self.embed_dim)

        self.query_dense = Dense(self.embed_dim)
        self.key_dense = Dense(self.embed_dim)
        self.value_dense = Dense(self.embed_dim)

    def separate_heads(self, x):
        x = tf.reshape(x, (-1, self.max_len, self.num_heads, self.attention_vector_len))
        return tf.transpose(x, perm=[0, 2, 1, 3])

    def compute_self_attention(self, query, key, value):
        score = tf.matmul(query, key, transpose_b=True)
        dim_key = tf.cast(tf.shape(key)[-1], tf.float32)
        scaled_score = score / tf.math.sqrt(dim_key)
        weights = tf.nn.softmax(scaled_score, axis=-1)
        output = tf.matmul(weights, value)
        return output

    def self_attention_layer(self, x):
        query = self.query_dense(x)
        key = self.key_dense(x)
        value = self.value_dense(x)

        query_heads = self.separate_heads(query)    
        key_heads = self.separate_heads(key)
        value_heads = self.separate_heads(value)

        attention = self.compute_self_attention(query_heads, key_heads, value_heads)

        attention = tf.transpose(attention, perm=[0, 2, 1, 3]) 
        attention = tf.reshape(attention, (-1, self.max_len, self.embed_dim))

        output = self.combine_heads(attention)
        return output
        
    def get_output(self, x):
        attn_output = self.self_attention_layer(x)
        out1 = self.layer_norm1(x + attn_output)

        ffn_output = self.final_ffn(self.mid_ffn(out1))

        encoder_output = self.layer_norm2(out1 + ffn_output)
        return encoder_output

class DecoderBlock(tf.keras.layers.Layer):
    def __init__(self, mid_ffn_dim, embed_dim, num_heads, max_len, batch_size):
        super(DecoderBlock, self).__init__()
        # Variables
        self.batch_size = batch_size
        self.max_len = max_len
        self.mid_ffn_dim = mid_ffn_dim
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.attention_vector_len = self.embed_dim // self.num_heads
        if self.embed_dim % self.num_heads != 0:
            raise ValueError('I am Batman!')

        # Trainable Layers

        self.query_dense1 = Dense(self.embed_dim, name='query_dense1')
        self.key_dense1 = Dense(self.embed_dim, name='key_dense1')
        self.value_dense1 = Dense(self.embed_dim, name='value_dense1')

        self.mid_ffn = Dense(self.mid_ffn_dim, activation='relu', name='dec_mid_ffn')
        self.final_ffn = Dense(self.embed_dim, name='dec_final_ffn')

        self.layer_norm1 = LayerNormalization(epsilon=1e-6)
        self.layer_norm2 = LayerNormalization(epsilon=1e-6)
        self.layer_norm3 = LayerNormalization(epsilon=1e-6)

        self.combine_heads = Dense(self.embed_dim, name='dec_combine_heads')

        self.query_dense2 = Dense(self.embed_dim, name='query_dense2')
        self.key_dense2 = Dense(self.embed_dim, name='key_dense2')
        self.value_dense2 = Dense(self.embed_dim, name='value_dense2')

    def separate_heads(self, x):
        x = tf.reshape(x, (-1, self.max_len, self.num_heads, self.attention_vector_len))
        return tf.transpose(x, perm=[0, 2, 1, 3])

    def compute_self_attention(self, query, key, value):
        score = tf.matmul(query, key, transpose_b=True)
        dim_key = tf.cast(tf.shape(key)[-1], tf.float32)
        scaled_score = score / tf.math.sqrt(dim_key)
        weights = tf.nn.softmax(scaled_score, axis=-1)
        output = tf.matmul(weights, value)
        return output

    def masking(self, x):
        b = []
        for batch in range(x.shape[0]):
            bat = []
            for head in range(x.shape[1]):
                headd = []
                for word in range(x.shape[2]):
                    current_word = []
                    for represented_in in range(x.shape[3]):
                        if represented_in > word:
                          current_word.append(np.NINF)
                        else:
                          current_word.append(0)
                    headd.append(current_word)
                bat.append(headd)
            b.append(bat)
        return b

    def compute_masked_self_attention(self, query, key, value):
        score = tf.matmul(query, key, transpose_b=True)
        score = score + self.masking(score)
        score = tf.convert_to_tensor(score)
                
        dim_key = tf.cast(tf.shape(key)[-1], tf.float32)
        scaled_score = score / tf.math.sqrt(dim_key)
        weights = tf.nn.softmax(scaled_score, axis=-1)
        output = tf.matmul(weights, value)
        return output

    def masked_self_attention_layer(self, x):
        query = self.query_dense1(x)
        key = self.key_dense1(x)
        value = self.value_dense1(x)

        query_heads = self.separate_heads(query)    
        key_heads = self.separate_heads(key)
        value_heads = self.separate_heads(value)

        attention = self.compute_masked_self_attention(query_heads, key_heads, value_heads)

        attention = tf.transpose(attention, perm=[0, 2, 1, 3]) 
        attention = tf.reshape(attention, (-1, self.max_len, self.embed_dim))

        output = self.combine_heads(attention)
        return output

    def second_attention_layer(self, x, encoder_output):
        query = self.query_dense2(x)
        key = self.key_dense2(encoder_output)
        value = self.value_dense2(encoder_output)

        query_heads = self.separate_heads(query)    
        key_heads = self.separate_heads(key)
        value_heads = self.separate_heads(value)

        attention = self.compute_self_attention(query_heads, key_heads, value_heads)

        attention = tf.transpose(attention, perm=[0, 2, 1, 3]) 
        attention = tf.reshape(attention, (-1, self.max_len, self.embed_dim))

        output = self.combine_heads(attention)
        return output
      
    def get_output(self, x, encoder_output):
        masked_attn_output = self.masked_self_attention_layer(x)
        out1 = self.layer_norm1(x + masked_attn_output)

        mutli_head_attn_output = self.second_attention_layer(out1, encoder_output)
        out2 = self.layer_norm2(out1 + mutli_head_attn_output)

        ffn_output = self.final_ffn(self.mid_ffn(out2))
        decoder_output = self.layer_norm3(out2 + ffn_output)
        return decoder_output

embed_dim = 512
mid_ffn_dim = 1024

num_heads = 8
max_len = max_len
batch_size = 32

encoder_block1 = EncoderBlock(mid_ffn_dim, embed_dim, num_heads, max_len, batch_size)
encoder_block2 = EncoderBlock(mid_ffn_dim, embed_dim, num_heads, max_len, batch_size)
encoder_block3 = EncoderBlock(mid_ffn_dim, embed_dim, num_heads, max_len, batch_size)

decoder_block1 = DecoderBlock(mid_ffn_dim, embed_dim, num_heads, max_len, batch_size)
decoder_block2 = DecoderBlock(mid_ffn_dim, embed_dim, num_heads, max_len, batch_size)
decoder_block3 = DecoderBlock(mid_ffn_dim, embed_dim, num_heads, max_len, batch_size)

# Define Loss and Optimizer
loss_object = SparseCategoricalCrossentropy()
optimizer = Adam()

embedding = Embedding(vocab_size, embed_dim, name='embedding')
position_embedding = Embedding(vocab_size, embed_dim)

final_transformer_layer = Dense(vocab_size, activation='softmax')

def positional_embedding(x):
    positions = tf.range(start=0, limit=max_len, delta=1)
    positions = position_embedding(positions)
    return x + positions

def train_step(english_sent, german_sent, german_trgt):
    with tf.GradientTape() as tape:
        english_embedded = embedding(english_sent)
        german_embedded = embedding(german_sent)

        english_positioned = positional_embedding(english_embedded)
        german_positioned = positional_embedding(german_embedded)

        # Encoders
        encoder_output = encoder_block1.get_output(english_positioned)
        encoder_output = encoder_block2.get_output(encoder_output)
        encoder_output = encoder_block3.get_output(encoder_output)

        # Decoders
        decoder_output = decoder_block1.get_output(german_positioned, encoder_output)
        decoder_output = decoder_block2.get_output(decoder_output, encoder_output)
        decoder_output = decoder_block3.get_output(decoder_output, encoder_output)

        # Final Output
        transformer_output = final_transformer_layer(decoder_output)

        # Compute Loss
        loss = loss_object(german_trgt, transformer_output)

    variables = embedding.trainable_variables + position_embedding.trainable_variables + encoder_block1.trainable_variables + encoder_block2.trainable_variables
    variables += encoder_block3.trainable_variables + decoder_block1.trainable_variables + decoder_block2.trainable_variables + decoder_block3.trainable_variables
    variables += final_transformer_layer.trainable_variables

    gradients = tape.gradient(loss, variables)
    optimizer.apply_gradients(zip(gradients, variables))

    return float(loss)

def train(epochs=10):
    batch_per_epoch = int(train_x1.shape[0] / batch_size)
    for epoch in range(epochs):
        for i in range(batch_per_epoch):
            english_sent_x = train_x1[i*batch_size : (i*batch_size)+batch_size].reshape(batch_size, max_len)
            german_sent_x = train_x2[i*batch_size : (i*batch_size)+batch_size].reshape(batch_size, max_len)
            german_sent_y = train_y[i*batch_size : (i*batch_size)+batch_size].reshape(batch_size, max_len, 1)

            loss = train_step(english_sent_x, german_sent_x, german_sent_y)

            print('Epoch ', epoch, 'Batch ', i, '/', batch_per_epoch, 'Loss ', loss)

train()

And the Code is done! But the loss stops reducing at around value of 1.2 after some time. Why is this happening?

Maybe Important

I tried debugging the model, by passing random input integers, and the model was still performing the same way it did when I gave real Sentences as input.

When I tried training the model with just 1 training sample, the loss stops reducing at around 0.2. When I train it with 2 training samples, the result was the approximately the same as when I trained it with 1 training sample.

When I stopped shuffling the dataset the loss gone till around 0.7 and again stopped learning.

I tried simplifying the model by removing some encoder and decoder blocks but the results were approximately the same. I even tried making the model more complex but the results were again approximately the same.

$\endgroup$
4
  • $\begingroup$ Is it supposed to translate English to German? Does it use dictionaries for both languages? How is success measured? $\endgroup$
    – mark mark
    Dec 10, 2020 at 7:05
  • $\begingroup$ @markmark yes it is supposed to translate English to German. No it does not uses dictionary, it uses array to store the data. I am measuring success in terms of loss. And also when saw the predicted sentence it is not that good $\endgroup$ Dec 10, 2020 at 7:17
  • $\begingroup$ I'm far from an expert, but here some ideas that I have. The problem is very complex for several reasons - the model needs to learn the relationship between the dictionaries (which are huge, and there's not always one to one match between words) and also it needs to learn the concept of grammar and semantics. That might need huge processing power. Maybe limiting number of allowed words in the training sets might help a bit? Not sure what architecture is proper for this task, but training many hidden layers is difficult to train without a strong GPU too. $\endgroup$
    – mark mark
    Dec 10, 2020 at 21:15
  • $\begingroup$ Another point is that it's hard to evaluate the success of translation numerically (there probably is, but I'm not sure SparseCategoricalCrossentropy is good enough for this), which makes it difficult to come up with an applicable loss function. For example, you might need some sort of custom loss function which will describe the distance in translations. Such function would assign different success values to different translations (e.g. translating "This is very good" as "Das ist sehr gut" vs "Das gut" vs a bunch of random words) $\endgroup$
    – mark mark
    Dec 10, 2020 at 21:26

0

You must log in to answer this question.