用word2vec分析标题，预测文章转发量

流行新闻网站的标题情绪分布图

main_data = pd.read_csv('News_Final.csv')

main_data.head()

# Grab all the titles

article_titles = main_data['Title']

# Create a list of strings, one for each title

titles_list = [title for title in article_titles]

# Collapse the list of strings into a single long string for processing

big_title_string = ' '.join(titles_list)

from nltk.tokenize import word_tokenize

# Tokenize the string into words

tokens = word_tokenize(big_title_string)

# Remove non-alphabetic tokens, such as punctuation

words = [word.lower() for word in tokens if word.isalpha()]

# Filter out stopwords

from nltk.corpus import stopwords

stop_words = set(stopwords.words('english'))

words = [word for word in words if not word in stop_words]

# Print first 10 words

words[:10]

# Loadword2vecmodel (trained on an enormous Google corpus)

model = gensim.models.KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary = True)

# Check dimension of word vectors

model.vector_size

300

economy_vec = model['economy']

economy_vec[:20] # First 20 components

# Filter the list of vectors to include only those that Word2Vec has a vector for

vector_list = [model[word] for word in words if word in model.vocab]

# Create a list of the words corresponding to these vectors

words_filtered = [word for word in words if word in model.vocab]

# Zip the words together with their vector representations

word_vec_zip = zip(words_filtered, vector_list)

# Cast to a dict so we can turn it into a DataFrame

word_vec_dict = dict(word_vec_zip)

df = pd.DataFrame.from_dict(word_vec_dict, orient='index')

df.head(3)

from sklearn.manifold import TSNE

# Initialize t-SNE

tsne = TSNE(n_components = 2, init = 'random', random_state = 10, perplexity = 100)

# Use only 400 rows to shorten processing time

tsne_df = tsne.fit_transform(df[:400])

现在，可以准备测绘减少的词汇向量数组的二维图。在这里，笔者使用adjust_text将单词自动分开。

sns.set()

# Initialize figure

fig, ax = plt.subplots(figsize = (11.7, 8.27))

sns.scatterplot(tsne_df[:, 0], tsne_df[:, 1], alpha = 0.5)

# Import adjustText, initialize list of texts

from adjustText import adjust_text

texts = []

words_to_plot = list(np.arange(0, 400, 10))

# Append words to list

for word in words_to_plot:

texts.append(plt.text(tsne_df[word, 0], tsne_df[word, 1], df.index[word], fontsize = 14))

# Plot text using adjust_text (because overlapping text is hard to read)

adjust_text(texts, force_points = 0.4, force_text = 0.4,

expand_points = (2,1), expand_text = (1,2),

arrowprops = dict(arrowstyle = "-", color = 'black', lw = 0.5))

plt.show()

# Preprocess the corpus

corpus = [preprocess(title) for title in titles_list]

# Remove docs that don't include any words in W2V's vocab

corpus, titles_list = filter_docs(corpus, titles_list, lambda doc: has_vector_representation(model, doc))

# Filter out any empty docs

corpus, titles_list = filter_docs(corpus, titles_list, lambda doc: (len(doc) != 0))

x = []

for doc in corpus: # append the vector for each document

x.append(document_vector(model, doc))

X = np.array(x) # list to array

# Initialize t-SNE

tsne = TSNE(n_components = 2, init = 'random', random_state = 10, perplexity = 100)

# Again use only 400 rows to shorten processing time

tsne_df = tsne.fit_transform(X[:400])

fig, ax = plt.subplots(figsize = (14, 10))

sns.scatterplot(tsne_df[:, 0], tsne_df[:, 1], alpha = 0.5)

from adjustText import adjust_text

texts = []

titles_to_plot = list(np.arange(0, 400, 40)) # plots every 40th title in first 400 titles

# Append words to list

for title in titles_to_plot:

texts.append(plt.text(tsne_df[title, 0], tsne_df[title, 1], titles_list[title], fontsize = 14))

# Plot text using adjust_text

adjust_text(texts, force_points = 0.4, force_text = 0.4,

expand_points = (2,1), expand_text = (1,2),

arrowprops = dict(arrowstyle = "-", color = 'black', lw = 0.5))

plt.show()

# Drop all the rows where the article popularities are unknown (this is only about 11% of the data)

main_data = main_data.drop(main_data[(main_data.Facebook == -1) |

(main_data.GooglePlus == -1) |

(main_data.LinkedIn == -1)].index)

# Also drop all rows where we don't know the source

main_data = main_data.drop(main_data[main_data['Source'].isna()].index)

main_data.shape

# Group by Source, then get descriptive statistics for title sentiment

source_info = main_data.groupby('Source')['SentimentTitle'].describe()

# Recall that `source_names` contains the top 12 sources

# We'll also sort by highest standard deviation

source_info.loc[source_names].sort_values('std', ascending=False)[['std', 'min', 'max']]

from sklearn.decomposition import PCA

pca = PCA(n_components=15, random_state=10)

# as a reminder, x is the array with our 300-dimensional vectors

reduced_vecs = pca.fit_transform(x)

df_w_vectors = pd.DataFrame(reduced_vecs)

df_w_vectors['Title'] = titles_list

# Use pd.concat to match original titles with their vectors

main_w_vectors = pd.concat((df_w_vectors, main_data), axis=1)

# Get rid of vectors that couldn't be matched with the main_df

main_w_vectors.dropna(axis=0, inplace=True)

# Drop all non-numeric, non-dummy columns, for feeding into the models

cols_to_drop = ['IDLink', 'Title', 'TimeSinceEpoch', 'Headline', 'PublishDate', 'Source']

data_only_df = pd.get_dummies(main_w_vectors, columns = ['Topic']).drop(columns=cols_to_drop)

# Standardize DaysSinceEpoch since the raw numbers are larger in magnitude

from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()

# Reshape so we can feed the column to the scaler

standardized_days = np.array(data_only_df['DaysSinceEpoch']).reshape(-1, 1)

data_only_df['StandardizedDays'] = scaler.fit_transform(standardized_days)

# Drop the raw column; we don't need it anymore

data_only_df.drop(columns=['DaysSinceEpoch'], inplace=True)

# Look at the new range

data_only_df['StandardizedDays'].describe()

# Get Facebook data only

fb_data_only_df = data_only_df.drop(columns=['GooglePlus', 'LinkedIn'])

# Separate the features and the response

X = fb_data_only_df.drop('Facebook', axis=1)

y = fb_data_only_df['Facebook']

# 80% of data goes to training

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 10)

from sklearn.metrics import mean_squared_error

# Instantiate an XGBRegressor

xgr = xgb.XGBRegressor(random_state=2)

# Fit the classifier to the training set

xgr.fit(X_train, y_train)

y_pred = xgr.predict(X_test)

mean_squared_error(y_test, y_pred)

353495.42

from sklearn.model_selection import GridSearchCV

# Various hyper-parameters to tune

xgb1 = xgb.XGBRegressor()

parameters = {'nthread':[4],

'objective':['reg:linear'],

'learning_rate': [.03, 0.05, .07],

'max_depth': [5, 6, 7],

'min_child_weight': [4],

'silent': [1],

'subsample': [0.7],

'colsample_bytree': [0.7],

'n_estimators': [250]}

xgb_grid = GridSearchCV(xgb1,

parameters,

cv = 2,

n_jobs = 5,

verbose=True)

xgb_grid.fit(X_train, y_train)

params = {'colsample_bytree': 0.7, 'learning_rate':0.03, 'max_depth': 5, 'min_child_weight': 4,

'n_estimators': 250, 'nthread':4, 'objective': 'reg:linear', 'silent': 1, 'subsample': 0.7}

# Try again with new params

xgr = xgb.XGBRegressor(random_state=2, **params)

# Fit the classifier to the training set

xgr.fit(X_train, y_train)

y_pred = xgr.predict(X_test)

mean_squared_error(y_test, y_pred)

351220.13

# Define a quick function that will return 1 (true) if the article has 0-1 share(s)

def dud_finder(popularity):

if popularity <= 1:

return 1

else:

return 0

# Create target column using the function

fb_data_only_df['is_dud'] = fb_data_only_df['Facebook'].apply(dud_finder)

fb_data_only_df[['Facebook', 'is_dud']].head()

# 28% of articles can be classified as "duds"

fb_data_only_df['is_dud'].sum() / len(fb_data_only_df)

0.28

from sklearn.ensemble import RandomForestClassifier

from sklearn.neighbors import KNeighborsClassifier

from sklearn.model_selection import train_test_split

X = fb_data_only_df.drop(['is_dud', 'Facebook'],axis=1)

y = fb_data_only_df['is_dud']

# 80% of data goes to training

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2,random_state = 10)

# Best params, produced by HP tuning

params = {'colsample_bytree': 0.7, 'learning_rate': 0.03, 'max_depth': 5,'min_child_weight': 4,

'n_estimators': 200, 'nthread':4, 'silent': 1, 'subsample': 0.7}

# Try xgc again with new params

xgc = xgb.XGBClassifier(random_state=10, **params)

rfc = RandomForestClassifier(n_estimators=100, random_state=10)

knn = KNeighborsClassifier()

preds = {}

for model_name, model in zip(['XGClassifier', 'RandomForestClassifier','KNearestNeighbors'], [xgc, rfc, knn]):

model.fit(X_train, y_train)

preds[model_name] =model.predict(X_test)

from sklearn.metrics import classification_report,roc_curve, roc_auc_score

for k in preds:

print("{}performance:".format(k))

print()

print(classification_report(y_test,preds[k]), sep='\n')

averaged_probs = (xgc.predict_proba(X)[:, 1] +

knn.predict_proba(X)[:,1] +

rfc.predict_proba(X)[:,1]) / 3

X['prob_dud'] = averaged_probs

y = fb_data_only_df['Facebook']

xgb.XGBRegressor(random_state=2,**params)

# Fit the classifier to the training set

xgr.fit(X_train, y_train)

y_pred = xgr.predict(X_test)

mean_squared_error(y_test, y_pred)

314551.59