直观理解并使用Tensorflow实现Seq2Seq模型的注意机制（上）

import numpy as np
import pandas as pd
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import tensorflow as tf
from sklearn.model_selection import train_test_split
import time
from matplotlib import pyplot as plt
import os
import re
data = pd.read_csv("./Hindi_English_Truncated_Corpus.csv")
english_sentences = data["english_sentence"]
hindi_sentences = data["hindi_sentence"]

# Global variables and Hyperparameters
num_words = 10000
oov_token = '<UNK>'
english_vocab_size = num_words + 1
hindi_vocab_size = num_words + 1
MAX_WORDS_IN_A_SENTENCE = 16
test_ratio = 0.2
BATCH_SIZE = 512
embedding_dim = 64
hidden_units = 1024
learning_rate = 0.006
epochs = 100
def preprocess_sentence(sen, is_english):
if (type(sen) != str):
 return ''
sen = sen.strip('.')
# insert space between words and punctuations
sen = re.sub(r"([?.!,¿;।])", r" \1 ", sen)
sen = re.sub(r'[" "]+', " ", sen)
# For english, replacing everything with space except (a-z, A-Z, ".", "?", "!", ",", "'")
if(is_english == True):
 sen = re.sub(r"[^a-zA-Z?.!,¿']+", " ", sen)
 sen = sen.lower()
sen = sen.strip()
sen = 'sentencestart ' + sen + ' sentenceend'
sen = ' '.join(sen.split())
return sen

# Loop through each datapoint having english and hindi sentence
processed_e_sentences = []
processed_h_sentences = []
for (e_sen, h_sen) in zip(english_sentences, hindi_sentences):
processed_e_sen = preprocess_sentence(e_sen, True)
processed_h_sen = preprocess_sentence(h_sen, False)
if(processed_e_sen == '' or processed_h_sen == '' or processed_e_sen.count(' ') > (MAX_WORDS_IN_A_SENTENCE-1) or processed_h_sen.count(' ') > (MAX_WORDS_IN_A_SENTENCE-1)):
 continue
processed_e_sentences.append(processed_e_sen)
processed_h_sentences.append(processed_h_sen)
print("Sentence examples: ")
print(processed_e_sentences[0])
print(processed_h_sentences[0])
print("Length of English processed sentences: " + str(len(processed_e_sentences)))
print("Length of Hindi processed sentences: " + str(len(processed_h_sentences)))
def tokenize_sentences(processed_sentences, num_words, oov_token):
tokenizer = Tokenizer(num_words = num_words, oov_token = oov_token)
tokenizer.fit_on_texts(processed_sentences)
word_index = tokenizer.word_index
sequences = tokenizer.texts_to_sequences(processed_sentences)
sequences = pad_sequences(sequences, padding = 'post')
return word_index, sequences, tokenizer
english_word_index, english_sequences, english_tokenizer = tokenize_sentences(processed_e_sentences, num_words, oov_token)
hindi_word_index, hindi_sequences, hindi_tokenizer = tokenize_sentences(processed_h_sentences, num_words, oov_token)
# split into traning and validation set
english_train_sequences, english_val_sequences, hindi_train_sequences, hindi_val_sequences = train_test_split(english_sequences, hindi_sequences, test_size = test_ratio)
BUFFER_SIZE = len(english_train_sequences)
# Batching the training set
dataset = tf.data.Dataset.from_tensor_slices((english_train_sequences, hindi_train_sequences)).shuffle(BUFFER_SIZE)
dataset = dataset.batch(BATCH_SIZE, drop_remainder = True)
print("No. of batches: " + str(len(list(dataset.as_numpy_iterator()))))

class Encoder(tf.keras.Model):
def __init__(self, english_vocab_size, embedding_dim, hidden_units):
 super(Encoder, self).__init__()
 self.embedding = tf.keras.layers.Embedding(english_vocab_size, embedding_dim)
 self.gru = tf.keras.layers.GRU(hidden_units, return_sequences = True, return_state = True)
def call(self, input_sequence):
 x = self.embedding(input_sequence)
 encoder_sequence_output, final_encoder_state = self.gru(x)
 #Dimensions of encoder_sequence_output => (BATCH_SIZE, MAX_WORDS_IN_A_SENTENCE, hidden_units)
 #Dimensions of final_encoder_state => (BATCH_SIZE, hidden_units)
 return encoder_sequence_output, final_encoder_state
# initialize our encoder
encoder = Encoder(english_vocab_size, embedding_dim, hidden_units)