蚂蚁金融NLP竞赛——文本语义相似度赛题总结

# step 1 加载原数据
jieba.load_userdict(project.aux_dir + dict_path)
data_local_df = pd.read_csv(project.data_dir + train_all, sep='\t', header=None,names=["index", "s1", "s2", "label"])
data_test_df = pd.read_csv(project.data_dir + test_all, sep='\t', header=None, names=["index", "s1", "s2", "label"])
data_all_df = pd.read_csv(project.data_dir + train_data_all, sep='\t', header=None, names=["index", "s1", "s2", "label"])
# 预训练中文字符向量
pre_train_char_w2v()
# 训练集中的语句的文本处理，去除停用词，根据规则表替换相应的词，使用jieba对语句进行分词处理
preprocessing(data_local_df,'train_0.6_seg')
preprocessing(data_test_df,'test_0.4_seg')
preprocessing(data_all_df,'data_all_seg')

# 保存label
project.save(project.features_dir + 'y_0.4_test.pickle', data_test_df['label'].tolist())
project.save(project.features_dir + 'y_0.6_train.pickle', data_local_df['label'].tolist())
project.save(project.features_dir + 'y_train.pickle', data_all_df['label'].tolist())

# step 2预训练中文词组向量
pre_train_w2v()

# step 3记录训练集中的词汇表中的词对应的词向量表示
process_save_embedding_wv('train_all_w2v_embedding_matrix.pickle',type=2,isStore_ids=True)
# process_save_embedding_wv('zhihu_w2v_embedding_matrix.pickle',type=2,isStore_ids=False)
# process_save_embedding_wv('zhihu_w2v_embedding_matrix.pickle',type=3,isStore_ids=False)

# step 4 char wordembedding
process_save_char_embedding_wv(isStore_ids=True)

class ManDist(Layer):
    def call(self, inputs, **kwargs):
        """This is where the layer's logic lives."""
        self.res  = K.exp(- K.sum(K.abs(inputs[0]-inputs[1]),axis = 1,keepdims = True))
        return self.res
class ConsDist(Layer):
    def call(self, inputs, **kwargs):
        self.res = K.sum(inputs[0] * inputs[1],axis=1,keepdims=True)/(K.sum(inputs[0]**2,axis=1,keepdims=True) * K.sum(inputs[1]**2,axis=1,keepdims=True))
        return self.res

def call(self, inputs, **kwargs):
    encode_s1 = inputs[0]
    encode_s2 = inputs[1]
    sentence_differerce = encode_s1 - encode_s2
    sentece_product = encode_s1 * encode_s2
    self.match_vector = K.concatenate([encode_s1,sentence_differerce,sentece_product,encode_s2],1)
    return self.match_vector

# 提取深度学习特征
extract_feature_siamese_lstm_manDist()
extract_feature_siamese_lstm_attention()
extract_feature_siamese_lstm_dssm()
# 提取NLP特征
extract_sentece_length_diff()
extract_edit_distance()
extract_ngram()
extract_sentence_diff_same()
extract_doubt_sim()
extract_sentence_exist_topic()
extract_word_embedding_sim()

# stacking 第一层模型训练,分别使用基分类器对训练集X_train进行5折交叉验证，在使用训练的模型预测X_test取均值。作为第二层Stacking模型的输入。
gnb_cls = GussianNBClassifier()
gnb_oop_train,gnb_oofp_val = gnb_cls.get_model_out(,y_train,X_test)

rf_cls = RFClassifer()
rf_oop_train, rf_oofp_val = rf_cls.get_model_out(X_train, y_train, X_test)

lg_cls = LogisicClassifier()
lg_oop_train, lg_oofp_val = lg_cls.get_model_out(X_train, y_train, X_test)

dt_cls = DecisionClassifier()
dt_oop_train, dt_oofp_val = dt_cls.get_model_out(X_train, y_train, X_test)
 # 构造第二层Stacking模型的输入
input_train = [gnb_oop_train,rf_oop_train,lg_oop_train,dt_oop_train]
input_test = [gnb_oofp_val,rf_oofp_val,lg_oofp_val,dt_oofp_val]

stacked_train = np.concatenate([data.reshape(-1,1) for data in input_train],axis=1)
stacked_test = np.concatenate([data.reshape(-1,1) for data in input_test],axis=1)

# stacking 第二层模型训练
second_model = DecisionTreeClassifier(max_depth=3,class_weight={0: 1, 1: 4})
second_model.fit(stacked_train,y_train)