Seq2Seq的那些事

# 文本的预处理工作
# step1 获取出文本内容
data_content,data_title = get_title_content(content_fp,title_fp)
ndexs = np.arange(len(data_content))
# step2 文本清洗工作
for index,content,title in zip(indexs,data_content,data_title):
    data_content[index] = remove_tag(content).encode('utf-8')
    data_title[index] = remove_tag(title).encode('utf-8')
# step3 划分数据，训练集，验证集，测试集
get_train_dev_sets(data_content,data_title,train_rate=0.7,dev_rate=0.1,
                    tr_con_path=src_train_path,tr_title_path=dest_train_path,
                       dev_con_path=src_dev_path,dev_title_path=dest_dev_path,
                       test_con_path=src_test_path,test_title_path=dest_test_path
                       )
# step4 将各个样本的语句进行切分，并将各个语句中的词转换成词汇表中该词对应的id.
prepare_headline_data(root_path,vocabulary_size=80000,tokenizer=jieba_tokenizer)

# 1.Encoder编码器的构造,size为隐藏层单元数,num_layers为LSTM的层数
single_cell = tf.contrib.rnn.GRUCell(size)
if use_lstm:
    single_cell = tf.contrib.rnn.BasicLSTMCell(size, state_is_tuple=True)
cell = single_cell
if num_layers > 1:
    cell = tf.contrib.rnn.MultiRNNCell([single_cell] * num_layers, state_is_tuple=True)
# 1.1进入tf.contrib.legacy_seq2seq.embedding_attention_seq2seq部分
#num_encoder_symbols为编码器部分的词汇表的大小，embedding_size为词向量的大小，encoder_outputs为输入的各个句子的各个词对应的ids
encoder_cell = core_rnn_cell.EmbeddingWrapper(cell,
        embedding_classes=num_encoder_symbols,embedding_size=embedding_size)
#encoder_outputs为编码器的输出, encoder_state为编码器的隐藏层的状态
encoder_outputs, encoder_state = core_rnn.static_rnn(encoder_cell, encoder_inputs, dtype=dtype)

# 2.Decoder解码器的构造+attention,output_projection为Decoder的输出参数元组(W, B)
# num_decoder_symbols为Decoder端的词汇表大小
from tensorflow.python.ops import array_ops
output_size = None
    if output_projection is None:
        decoder_cell = core_rnn_cell.OutputProjectionWrapper(cell, num_decoder_symbols)
        output_size = num_decoder_symbols
# 2.1 将编码器的输出进行concat操作，作为attention的输入
top_states = [array_ops.reshape(e, [-1, 1, cell.output_size]) for e in encoder_outputs]
attention_states = array_ops.concat(top_states, 1)
# 2.2embedding_attention_decoder函数是构造encoder和attention的计算关系
return embedding_attention_decoder(
          decoder_inputs,
          encoder_state,
          attention_states,
          cell,
          num_decoder_symbols,
          embedding_size,
          num_heads=num_heads,
          output_size=output_size,
          output_projection=output_projection,
          feed_previous=feed_previous,
          initial_state_attention=initial_state_attention)

# 真实的labels,此处采用的loss函数为sampled_softmax_loss,后面会讲述到为什么是这个loss
labels = tf.reshape(labels, [-1, 1])
local_w_t = tf.cast(w_t, tf.float32)
local_b = tf.cast(b, tf.float32)
local_inputs = tf.cast(inputs, tf.float32)
loss_op = tf.cast(tf.nn.sampled_softmax_loss(
                weights=local_w_t,
                biases=local_b,
                labels=labels,
                inputs=local_inputs,
                num_sampled=num_samples,
                num_classes=self.target_vocab_size),
            tf.float32)

# Gradients and SGD update operation for training the model.
# 1.获取出tf session中trainable=True的参数变量。
params = tf.trainable_variables()
# 2.设置参数更新优化器
opt = tf.train.GradientDescentOptimizer(self.learning_rate)
# 3.求参数的梯度值，其中self.losses[b]为目标值(代价函数的表达式)
gradients = tf.gradients(self.losses[b], params)
# 4.梯度修剪，修正梯度值，用于控制梯度爆炸的问题。梯度爆炸和梯度弥散的原因一样，
#都是因为链式法则求导的关系，导致梯度的指数级衰减。为了避免梯度爆炸，需要对梯度进行修剪。
clipped_gradients, norm = tf.clip_by_global_norm(gradients,max_gradient_norm)
# 5.求取更新参数的tensorflow的节点
self.updates.append(opt.apply_gradients(zip(clipped_gradients, params), global_step=self.global_step))

# 4.1 根据随机选取的bucket_id，批量选出输入模型的三类feed数据，编码器的inputs和解码器的inputs,target_weights对encoder_inputs进行指示，为1表示已经预测的，为0表示PAD部分。
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
          train_set, bucket_id)
# 4.2 训练的Feed数据的构造
input_feed = {}
for l in xrange(encoder_size):
    input_feed[self.encoder_inputs[l].name] = encoder_inputs[l]
for l in xrange(decoder_size):
    input_feed[self.decoder_inputs[l].name] = decoder_inputs[l]
    input_feed[self.target_weights[l].name] = target_weights[l]
# 4.3 由于目标词是由于Decoder左移一位了，所以需要再添加一位
last_target = self.decoder_inputs[decoder_size].name
input_feed[last_target] = np.zeros([self.batch_size], dtype=np.int32)
train_ops = [self.updates[bucket_id],  # Update Op that does SGD.
             self.gradient_norms[bucket_id],  # Gradient norm.
             self.losses[bucket_id]]  # Loss for this batch.
outputs = session.run(train_ops, input_feed)