神经网络机器翻译seq2seq（nlp实践2）

本实践采用的是IWLST TED演讲en_zh数据集，基于tensorflow 实战google深度学习框架（第二版）一书所进行的实验，大部分代码直接是用的书上的，为了更好地巩固知识，所以整理成博客。

首先从相关网站上下载数据集，解压，代码如下所示：

wget https://wit3.fbk.eu/archive/2015-01//texts/en/zh/en-zh.tgz
tar xzvf en-zh.tgz
cd en-zh/

我们只关注train.tags.en-zh.en和train.tags.en-zh.zh两个文本数据文件，但是都带有html标记，需要进行预处理一下：

IWSLT15.TED.dev2010.en-zh.en.xml  IWSLT15.TED.tst2011.en-zh.en.xml  IWSLT15.TED.tst2013.en-zh.en.xml  train.tags.en-zh.zh
IWSLT15.TED.dev2010.en-zh.zh.xml  IWSLT15.TED.tst2011.en-zh.zh.xml  IWSLT15.TED.tst2013.en-zh.zh.xml  train.zh
IWSLT15.TED.tst2010.en-zh.en.xml  IWSLT15.TED.tst2012.en-zh.en.xml  README
IWSLT15.TED.tst2010.en-zh.zh.xml  IWSLT15.TED.tst2012.en-zh.zh.xml  train.tags.en-zh.en

在对原始数据进行处理时，需要进行分词、建库和数字化等操作，在将数据处理成可以输入的数据时还得进行padding。在这里，本人选取的中文和英文的分词工具都是stanfordcorenlp，相关知识请参考这篇博客。下面直接上代码：

#coding:utf-8
import collections
from operator import itemgetter
from stanfordcorenlp import StanfordCoreNLP
import tqdm

#第一步，把英文跟中文弄成一行一句的格式
def deletehtml(filename1,filename2):
    f1 = open(filename1,'r')
    f2 = open(filename2,'r') 

    data1 = f1.readlines()
    data2 = f2.readlines()
    assert len(data1)==len(data2)#用codecs会导致报错不知道为什么
    fw1 = open(filename1+".deletehtml",'w')
    fw2 = open(filename2+".deletehtml",'w')

    print("deletehtml...")

    for line1,line2 in tqdm.tqdm(zip(data1,data2)):
        line1 = line1.strip()
        line2 = line2.strip()
        if line1 and line2:
            if '<' not in line1 and '>' not in line1 and '<' not in line2 and '>' not in line2:
                fw1.write(line1+"\n")
                fw2.write(line2+"\n")
    fw1.close()
    f1.close()
    fw2.close()
    f2.close()

    return filename1+".deletehtml",filename2+".deletehtml"

#第二步，分词并建立词库
def segement_sentence(filename,vocab_size,lang='en'):
    nlp = StanfordCoreNLP("../stanford-corenlp-full-2018-10-05",lang=lang)
    with open(filename,'r') as f:
        data = f.readlines()
        counter = collections.Counter()
        f1 = open(filename+".segment",'w')
        print("segmenting...")
        for line in tqdm.tqdm(data):
            line = line.strip()
            word_list = nlp.word_tokenize(line.strip())
            sentence = ' '.join(word_list)
            f1.write(sentence+"\n")
            for word in word_list:
                counter[word] += 1
        f1.close()
    nlp.close()

    sorted_word_to_cnt = sorted(counter.items(),key=itemgetter(1),reverse=True)
    sorted_words =  ["<unk>","<sos>","<eos>"] + [x[0] for x in sorted_word_to_cnt] 

    if len(sorted_words)>vocab_size:
        sorted_words = sorted_words[:vocab_size]
    assert len(sorted_words)<=vocab_size
    with open(filename+".vocab",'w') as fw:
        for word in sorted_words:
            fw.write(word+"\n")
    return filename+".segment"

#第三步，将文本转换成数字编号
def convert_to_id(filename,vocab_file):
    with open(vocab_file,"r") as f:
        data = f.readlines()
        vocab = [w.strip() for w in data]
    word_to_id = {k:v for (k,v) in zip(vocab,range(len(vocab)))}

    with open(filename,"r") as f:
        data = f.readlines()
        f1 = open(filename+".id",'w')
        print("converting...")
        for line in tqdm.tqdm(data):
            words = line.strip().split()+["<eos>"]
            ids = ' '.join([str(word_to_id[word]) 
                if word in word_to_id else str(word_to_id["<unk>"]) 
                for word in words])
            f1.write(ids+"\n")
        f1.close()
    return filename+".id"

def main():
    src = "train.tags.en-zh.en"#有html标记
    trg = "train.tags.en-zh.zh"#同
    src_vocab_size = 10000
    trg_vocab_size = 4000

    src1,trg1 = deletehtml(src,trg)
    

    src2 = segement_sentence(src1,src_vocab_size,lang='en')
    trg2 = segement_sentence(trg1,trg_vocab_size,lang='zh')

    src3 = convert_to_id(src+".deletehtml.segment",src+".deletehtml.vocab")
    trg3 = convert_to_id(trg+".deletehtml.segment",trg+".deletehtml.vocab")

if __name__ == '__main__':
    main()

使用上述代码对数据文件进行处理后，我们就得到了每一行一句话，一句话都转成编号的形式，句子的某尾加<eos>标志。①但是由于每句话的长短不一，需要将一个batch中的较短句子的长度padding至最长句子的长度，这样才能使encoder正常工作；②在decoder阶段，由于输入需要以<sos>开头，那么还得将目标语言再进行处理一下，x y z <eos>变为<sos> x y z。直接上完整的训练代码：

#coding:utf-8
import tensorflow as tf

MAX_LEN = 50
SOS_ID = 1

SRC_TRAIN_DATA = "../train.tags.en-zh.en.deletehtml.segment.id"
TRG_TRAIN_DATA = "../train.tags.en-zh.zh.deletehtml.segment.id"
CHECKPOINT_PATH = "./seq2seq_ckpt"

HIDDEN_SIZE = 1024
NUM_LAYERS = 2
SRC_VOCAB_SIZE = 10000
TRG_VOCAB_SIZE = 4000
BATCH_SIZE = 100
NUM_EPOCH = 5
KEEP_PROB = 0.8
MAX_GRAD_NORM = 5
SHARE_EMB_AND_SOFTMAX = True

class NMTModel(object):
    def __init__(self):
        self.enc_cell = tf.nn.rnn_cell.MultiRNNCell([tf.nn.rnn_cell.BasicLSTMCell(HIDDEN_SIZE)\
         for _ in range(NUM_LAYERS)])
        self.dec_cell = tf.nn.rnn_cell.MultiRNNCell([tf.nn.rnn_cell.BasicLSTMCell(HIDDEN_SIZE)\
         for _ in range(NUM_LAYERS)])

        self.src_embedding = tf.get_variable(
            "src_emb",[SRC_VOCAB_SIZE,HIDDEN_SIZE])
        self.trg_embedding = tf.get_variable(
            "trg_emb",[TRG_VOCAB_SIZE,HIDDEN_SIZE])
        
        if SHARE_EMB_AND_SOFTMAX:
            self.softmax_weight = tf.transpose(self.trg_embedding)
        else:
            self.softmax_weight = tf.get_variable("weight",[HIDDEN_SIZE,TRG_VOCAB_SIZE])
        self.softmax_bias = tf.get_variable("softmax_bias",[TRG_VOCAB_SIZE])

    def forward(self,src_input,src_size,trg_input,trg_label,trg_size):
        batch_size = tf.shape(src_input)[0]
        src_emb = tf.nn.embedding_lookup(self.src_embedding,src_input)
        trg_emb = tf.nn.embedding_lookup(self.trg_embedding,trg_input)

        src_emb = tf.nn.dropout(src_emb,KEEP_PROB)
        trg_emb = tf.nn.dropout(trg_emb,KEEP_PROB)

        with tf.variable_scope("encoder"):
            enc_outputs,enc_state = tf.nn.dynamic_rnn(
                self.enc_cell,src_emb,src_size,dtype=tf.float32)

        with tf.variable_scope("decoder"):
            dec_outputs, _ = tf.nn.dynamic_rnn(
                self.dec_cell,trg_emb,trg_size,initial_state=enc_state)

        output = tf.reshape(dec_outputs,[-1,HIDDEN_SIZE])
        logits = tf.matmul(output,self.softmax_weight) + self.softmax_bias
        loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=tf.reshape(trg_label,[-1]),logits=logits)

        label_weights = tf.sequence_mask(trg_size,maxlen=tf.shape(trg_label)[1],dtype=tf.float32)
        label_weights = tf.reshape(label_weights,[-1])

        cost = tf.reduce_sum(loss*label_weights)
        cost_per_token = cost / tf.reduce_sum(label_weights)

        trainable_variables = tf.trainable_variables()

        grads = tf.gradients(cost / tf.to_float(batch_size), trainable_variables)
        grads,_ = tf.clip_by_global_norm(grads,MAX_GRAD_NORM)
        optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0)
        train_op = optimizer.apply_gradients(zip(grads,trainable_variables))

        return cost_per_token,train_op

def run_epoch(session,cost_op,train_op,saver,step):
    while True:
        try:
            cost,_ = session.run([cost_op,train_op])
            if step%10 == 0:
                print("steps %d, per token cost is %.3f"%(step,cost))
            if step%200 == 0:
                saver.save(session,CHECKPOINT_PATH,global_step=step)
            step += 1
        except tf.errors.OutOfRangeError:
            break
    return step


def MakeDataset(file_path):
    dataset = tf.data.TextLineDataset(file_path)
    dataset = dataset.map(lambda string: tf.string_split([string]).values)
    dataset = dataset.map(lambda string: tf.string_to_number(string,tf.int32))
    dataset = dataset.map(lambda x: (x,tf.size(x)))
    return dataset

def MakeSrcTrgDataset(src_path,trg_path,batch_size):
    src_data = MakeDataset(src_path)
    trg_data = MakeDataset(trg_path)

    dataset = tf.data.Dataset.zip((src_data,trg_data))

    def FilterLength(src_tuple,trg_tuple):
        ((src_input,src_len),(trg_label,trg_len)) = (src_tuple,trg_tuple)
        src_len_ok = tf.logical_and(tf.greater(src_len,1),tf.less_equal(src_len,MAX_LEN))
        trg_len_ok = tf.logical_and(tf.greater(trg_len,1),tf.less_equal(trg_len,MAX_LEN))
        return tf.logical_and(src_len_ok,trg_len_ok)
    dataset = dataset.filter(FilterLength)

    def MakeTrgInput(src_tuple,trg_tuple):
        ((src_input,src_len),(trg_label,trg_len)) = (src_tuple,trg_tuple)
        trg_input = tf.concat([[SOS_ID],trg_label[:-1]],axis=0)
        return ((src_input,src_len),(trg_input,trg_label,trg_len))
    dataset = dataset.map(MakeTrgInput)
    dataset = dataset.shuffle(10000)

    padded_shapes = (
        (tf.TensorShape([None]),
         tf.TensorShape([])),
        (tf.TensorShape([None]),
         tf.TensorShape([None]),
         tf.TensorShape([])))
    batched_dataset = dataset.padded_batch(batch_size,padded_shapes)
    return batched_dataset
                

def main():
    initializer = tf.random_uniform_initializer(-0.05,0.05)
    with tf.variable_scope("nmt_model",reuse=None,initializer=initializer):
        train_model = NMTModel()

    data = MakeSrcTrgDataset(SRC_TRAIN_DATA,TRG_TRAIN_DATA,BATCH_SIZE)
    iterator = data.make_initializable_iterator()
    (src,src_size),(trg_input,trg_label,trg_size) = iterator.get_next()

    cost_op,train_op = train_model.forward(src,src_size,trg_input,trg_label,trg_size)
    saver = tf.train.Saver()
    step = 0

    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7,allow_growth=True)
    session = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))

    with session as sess:
        tf.global_variables_initializer().run()
        for i in range(NUM_EPOCH):
            print("In iteration: %d"%(i+1))
            sess.run(iterator.initializer)
            step = run_epoch(sess,cost_op,train_op,saver,step)

if __name__ == '__main__':
    main()

在对新句子进行inference时，需要确定解码时的条件，因此要用到tf.while_loop函数，这里直接给出预测代码：

#coding:utf-8
import tensorflow as tf


CHECKPOINT_PATH = "./seq2seq_ckpt-9000"

HIDDEN_SIZE = 1024
NUM_LAYERS = 2
SRC_VOCAB_SIZE = 10000
TRG_VOCAB_SIZE = 4000
BATCH_SIZE = 100
SHARE_EMB_AND_SOFTMAX = True
SOS_ID = 1
EOS_ID = 2

class NMTModel(object):
    def __init__(self):
        self.enc_cell = tf.nn.rnn_cell.MultiRNNCell([tf.nn.rnn_cell.BasicLSTMCell(HIDDEN_SIZE)\
         for _ in range(NUM_LAYERS)])
        self.dec_cell = tf.nn.rnn_cell.MultiRNNCell([tf.nn.rnn_cell.BasicLSTMCell(HIDDEN_SIZE)\
         for _ in range(NUM_LAYERS)])

        self.src_embedding = tf.get_variable(
            "src_emb",[SRC_VOCAB_SIZE,HIDDEN_SIZE])
        self.trg_embedding = tf.get_variable(
            "trg_emb",[TRG_VOCAB_SIZE,HIDDEN_SIZE])
        
        if SHARE_EMB_AND_SOFTMAX:
            self.softmax_weight = tf.transpose(self.trg_embedding)
        else:
            self.softmax_weight = tf.get_variable("weight",[HIDDEN_SIZE,TRG_VOCAB_SIZE])
        self.softmax_bias = tf.get_variable("softmax_bias",[TRG_VOCAB_SIZE])

    def inference(self,src_input):
        src_size = tf.convert_to_tensor([len(src_input)],dtype=tf.int32)
        src_input = tf.convert_to_tensor([src_input],dtype=tf.int32)
        src_emb = tf.nn.embedding_lookup(self.src_embedding,src_input)

        with tf.variable_scope("encoder"):
            enc_outputs,enc_state = tf.nn.dynamic_rnn(
                self.enc_cell,src_emb,src_size,dtype=tf.float32)
        MAX_DEC_LEN = 100

        with tf.variable_scope("decoder/rnn/multi_rnn_cell"):
            init_array = tf.TensorArray(dtype=tf.int32,size=0,dynamic_size=True,clear_after_read=False)
            init_array = init_array.write(0,SOS_ID)

            init_loop_var = (enc_state,init_array,0)

            def continue_loop_condition(state,trg_ids,step):
                return tf.reduce_all(tf.logical_and(tf.not_equal(trg_ids.read(step),EOS_ID),tf.less(step,MAX_DEC_LEN-1)))

            def loop_body(state,trg_ids,step):
                trg_input = [trg_ids.read(step)]
                trg_emb = tf.nn.embedding_lookup(self.trg_embedding,trg_input)

                dec_outputs,next_state = self.dec_cell.call(state=state,inputs=trg_emb)
                output = tf.reshape(dec_outputs,[-1,HIDDEN_SIZE])
                logits = (tf.matmul(output,self.softmax_weight) + self.softmax_bias)
                next_id = tf.argmax(logits,axis=1,output_type=tf.int32)

                trg_ids = trg_ids.write(step+1,next_id[0])
                return next_state,trg_ids,step+1

            state,trg_ids,step = tf.while_loop(
                continue_loop_condition,loop_body,init_loop_var)
            return trg_ids.stack()

def main():
    from stanfordcorenlp import StanfordCoreNLP
    nlp = StanfordCoreNLP("../../stanford-corenlp-full-2018-10-05",lang='en')
    with tf.variable_scope("nmt_model",reuse=None):
        model = NMTModel()
    vocab_file = "../train.tags.en-zh.en.deletehtml.vocab"
    sentence = "It is very beautiful!"
    with open(vocab_file,'r') as f:
        data = f.readlines()
        words = [w.strip() for w in data]
    word_to_id = {k:v for (k,v) in zip(words,range(len(words)))}
    wordlist = nlp.word_tokenize(sentence.strip()) + ["<eos>"]
    # print(wordlist)
    idlist = [str(word_to_id[w]) if w in word_to_id else str(word_to_id["<unk>"]) for w in wordlist]
    idlist = [int(i) for i in idlist]
    # print(idlist)

    output_op = model.inference(idlist)
    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7,allow_growth=True)
    session = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
    saver = tf.train.Saver()
    saver.restore(session,CHECKPOINT_PATH)

    output = session.run(output_op)

    vocab_file2 = "../train.tags.en-zh.zh.deletehtml.vocab"
    with open(vocab_file2,'r') as f2:
        data2 = f2.readlines()
        words = [w.strip() for w in data2]
    id_to_word = {k:v for (k,v) in zip(range(len(words)),words)}
    print([id_to_word[i] for i in output])
    session.close()

    nlp.close()

if __name__ == '__main__':
    main()

下面给出预测结果:

['<sos>', '这', '是', '非常', '美丽', '的', '！', '<eos>']

由于不想让篇幅过长，所以直接上干货，具体原理请参考开头所说的书，有疑问请在下面留言，我的软件配置是python3.6.5 + tensorflow-gpu==1.12 + cuda9.0。