一、新闻推荐之数据探索与数据分析

%matplotlib inline
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
plt.rc('font', family='SimHei', size=13)
import os,gc,re,warnings,sys
warnings.filterwarnings("ignore")

# path = './data/' # 自定义的路径
path='./data_raw/'
trn_click = pd.read_csv(path+'train_click_log.csv')
item_df = pd.read_csv(path+'articles.csv')
item_df = item_df.rename(columns={'article_id': 'click_article_id'})  #重命名，方便后续match
item_emb_df = pd.read_csv(path+'articles_emb.csv')
#####test
tst_click = pd.read_csv(path+'testA_click_log.csv')

trn_click.head()

item_df.head()

item_emb_df.head()

trn_click['rank'] = trn_click.groupby(['user_id'])['click_timestamp'].rank(ascending=False).astype(int)
tst_click['rank'] = tst_click.groupby(['user_id'])['click_timestamp'].rank(ascending=False).astype(int)

trn_click['click_cnts'] = trn_click.groupby(['user_id'])['click_timestamp'].transform('count')
tst_click['click_cnts'] = tst_click.groupby(['user_id'])['click_timestamp'].transform('count')

trn_click.groupby(['user_id'])['click_timestamp'].transform('count')

0          11
1          11
2          11
3          40
4          40
           ..
1112618     4
1112619     2
1112620     2
1112621    14
1112622    14
Name: click_timestamp, Length: 1112623, dtype: int64

trn_click.groupby(['user_id'])['click_timestamp'].agg('count')
• 1

user_id
0          2
1          2
2          2
3          2
4          2
          ..
199995     7
199996    13
199997     2
199998    40
199999    11
Name: click_timestamp, Length: 200000, dtype: int64

trn_click = trn_click.merge(item_df, how='left', on=['click_article_id'])
trn_click.head()

trn_click.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 1112623 entries, 0 to 1112622
Data columns (total 14 columns):
user_id                1112623 non-null int64
click_article_id       1112623 non-null int64
click_timestamp        1112623 non-null int64
click_environment      1112623 non-null int64
click_deviceGroup      1112623 non-null int64
click_os               1112623 non-null int64
click_country          1112623 non-null int64
click_region           1112623 non-null int64
click_referrer_type    1112623 non-null int64
rank                   1112623 non-null int32
click_cnts             1112623 non-null int64
category_id            1112623 non-null int64
created_at_ts          1112623 non-null int64
words_count            1112623 non-null int64
dtypes: int32(1), int64(13)
memory usage: 123.1 MB

trn_click.describe([0.01,0.25,0.5,0.75,0.99])

trn_click.head()
• 1

columns=trn_click.columns.to_list()

len(columns)

14

fig=plt.figure(figsize=(80,60))
for i in range(14):
    plt.subplot(4,4,i+1)
    sns.boxplot(trn_click[columns[i]],orient='v',width=0.5)
    plt.ylabel(columns[i],fontsize=36)
plt.show()

#训练集中的用户数量为20w
trn_click["user_id"].nunique()

200000

trn_click.groupby('user_id')['click_article_id'].count().min()
# 训练集里面每个用户至少点击了两篇文章

2

trn_click.groupby('user_id')['click_article_id'].count()

user_id
0          2
1          2
2          2
3          2
4          2
          ..
199995     7
199996    13
199997     2
199998    40
199999    11
Name: click_article_id, Length: 200000, dtype: int64

trn_click['click_article_id'].value_counts().reset_index()[:10]

plt.figure()
plt.figure(figsize=(15, 20))
i = 1
for col in ['click_article_id', 'click_timestamp', 'click_environment', 'click_deviceGroup', 'click_os', 'click_country', 
            'click_region', 'click_referrer_type', 'rank', 'click_cnts']:
    plot_envs = plt.subplot(5, 2, i)
    i += 1
    v = trn_click[col].value_counts().reset_index()[:10]
    fig = sns.barplot(x=v['index'], y=v[col])
    for item in fig.get_xticklabels():
        item.set_rotation(90)
    plt.title(col)
plt.tight_layout()
plt.show()

<Figure size 432x288 with 0 Axes>

tst_click = tst_click.merge(item_df, how='left', on=['click_article_id'])
tst_click.head()

tst_click.describe([0.01,0.25,0.5,0.75,0.99]).T

#测试集中的用户数量为5w
tst_click.user_id.nunique()

50000

tst_click.groupby('user_id')['click_article_id'].count().min()
# 注意测试集里面有只点击过一次文章的用户

1

#新闻文章数据集浏览
item_df.head().append(item_df.tail())

item_df['words_count'].value_counts()

176     3485
182     3480
179     3463
178     3458
174     3456
        ... 
845        1
710        1
965        1
847        1
1535       1
Name: words_count, Length: 866, dtype: int64

print(item_df['category_id'].nunique())     # 461个文章主题
item_df['category_id'].hist()

461
<matplotlib.axes._subplots.AxesSubplot at 0x21fb84d4248>

item_df.shape       # 364047篇文章

(364047, 4)

item_emb_df.head()

item_emb_df.shape

(364047, 251)

#####merge
user_click_merge = trn_click.append(tst_click)

#用户重复点击
user_click_count = user_click_merge.groupby(['user_id', 'click_article_id'])['click_timestamp'].agg({'count'}).reset_index()
user_click_count[:10]

user_click_count[user_click_count['count']>7]
• 1

user_click_count['count'].unique()

array([ 1,  2,  4,  3,  6,  5, 10,  7, 13], dtype=int64)

#用户点击新闻次数
user_click_count.loc[:,'count'].value_counts() 

1     1605541
2       11621
3         422
4          77
5          26
6          12
10          4
7           3
13          1
Name: count, dtype: int64

def plot_envs(df, cols, r, c):
    plt.figure()
    plt.figure(figsize=(10, 5))
    i = 1
    for col in cols:
        plt.subplot(r, c, i)
        i += 1
        v = df[col].value_counts().reset_index()
        fig = sns.barplot(x=v['index'], y=v[col])
        for item in fig.get_xticklabels():
            item.set_rotation(90)
        plt.title(col)
    plt.tight_layout()
    plt.show()

# 分析用户点击环境变化是否明显，这里随机采样10个用户分析这些用户的点击环境分布
sample_user_ids = np.random.choice(tst_click['user_id'].unique(), size=10, replace=False)
sample_users = user_click_merge[user_click_merge['user_id'].isin(sample_user_ids)]
cols = ['click_environment','click_deviceGroup', 'click_os', 'click_country', 'click_region','click_referrer_type']
for _, user_df in sample_users.groupby('user_id'):
    plot_envs(user_df, cols, 2, 3)

<Figure size 432x288 with 0 Axes>

<Figure size 432x288 with 0 Axes>

<Figure size 432x288 with 0 Axes>

<Figure size 432x288 with 0 Axes>

<Figure size 432x288 with 0 Axes>

<Figure size 432x288 with 0 Axes>
• 1

<Figure size 432x288 with 0 Axes>

<Figure size 432x288 with 0 Axes>

<Figure size 432x288 with 0 Axes>
• 1

<Figure size 432x288 with 0 Axes>

user_click_item_count = sorted(user_click_merge.groupby('user_id')['click_article_id'].count(), reverse=True)
plt.plot(user_click_item_count)

[<matplotlib.lines.Line2D at 0x21fc0b4e3c8>]

#点击次数在前50的用户
plt.plot(user_click_item_count[:50])

[<matplotlib.lines.Line2D at 0x21fc0b15308>]

#点击次数排名在[25000:50000]之间
plt.plot(user_click_item_count[25000:50000])

[<matplotlib.lines.Line2D at 0x21fc0ae2f48>

item_click_count = sorted(user_click_merge.groupby('click_article_id')['user_id'].count(), reverse=True)

plt.plot(item_click_count)

[<matplotlib.lines.Line2D at 0x21f0f68a9c8>]

plt.plot(item_click_count[:100])

[<matplotlib.lines.Line2D at 0x21f0f716948>]

plt.plot(item_click_count[:20])

[<matplotlib.lines.Line2D at 0x21f0f620d08>]

plt.plot(item_click_count[3500:])

[<matplotlib.lines.Line2D at 0x21f9be4a988>]

tmp = user_click_merge.sort_values('click_timestamp')
tmp['next_item'] = tmp.groupby(['user_id'])['click_article_id'].transform(lambda x:x.shift(-1))
union_item = tmp.groupby(['click_article_id','next_item'])['click_timestamp'].agg({'count'}).reset_index().sort_values('count', ascending=False)
union_item[['count']].describe()

#由统计数据可以看出，平均共现次数2.88，最高为1687。
#说明用户看的新闻，相关性是比较强的。
#画个图直观地看一看
x = union_item['click_article_id']
y = union_item['count']
plt.scatter(x, y)

<matplotlib.collections.PathCollection at 0x21f0fedad88>

plt.plot(union_item['count'].values[40000:])

[<matplotlib.lines.Line2D at 0x21f0ff1eb08>]

#不同类型的新闻出现的次数
plt.plot(user_click_merge['category_id'].value_counts().values)

[<matplotlib.lines.Line2D at 0x21f0dda5808>]

#出现次数比较少的新闻类型, 有些新闻类型，基本上就出现过几次
plt.plot(user_click_merge['category_id'].value_counts().values[150:])

[<matplotlib.lines.Line2D at 0x21f58bf0e88>]

#新闻字数的描述性统计
user_click_merge['words_count'].describe(

count    1.630633e+06
mean     2.043012e+02
std      6.382198e+01
min      0.000000e+00
25%      1.720000e+02
50%      1.970000e+02
75%      2.290000e+02
max      6.690000e+03
Name: words_count, dtype: float64

plt.plot(user_click_merge['words_count'].values)

[<matplotlib.lines.Line2D at 0x21f5ad068c8>]

plt.plot(sorted(user_click_merge.groupby('user_id')['category_id'].nunique(), reverse=True))

[<matplotlib.lines.Line2D at 0x21f5ad08708>]

plt.plot(sorted(user_click_merge.groupby('user_id')['words_count'].mean(), reverse=True))

[<matplotlib.lines.Line2D at 0x21fb99c5b88>]

#挑出大多数人的区间仔细看看
plt.plot(sorted(user_click_merge.groupby('user_id')['words_count'].mean(), reverse=True)[1000:45000])

[<matplotlib.lines.Line2D at 0x21f9f9b3388>]

#更加详细的参数
user_click_merge.groupby('user_id')['words_count'].mean().reset_index().describe()

#为了更好的可视化，这里把时间进行归一化操作
from sklearn.preprocessing import MinMaxScaler
mm = MinMaxScaler()
user_click_merge['click_timestamp'] = mm.fit_transform(user_click_merge[['click_timestamp']])
user_click_merge['created_at_ts'] = mm.fit_transform(user_click_merge[['created_at_ts']])
user_click_merge = user_click_merge.sort_values('click_timestamp')

user_click_merge.head()

def mean_diff_time_func(df, col):
    df = pd.DataFrame(df, columns={col})
    df['time_shift1'] = df[col].shift(1).fillna(0)
    df['diff_time'] = abs(df[col] - df['time_shift1'])
    return df['diff_time'].mean()

# 点击时间差的平均值
mean_diff_click_time = user_click_merge.groupby('user_id')['click_timestamp', 'created_at_ts'].apply(lambda x: mean_diff_time_func(x, 'click_timestamp'))

plt.plot(sorted(mean_diff_click_time.values, reverse=True))

# 前后点击文章的创建时间差的平均值
mean_diff_created_time = user_click_merge.groupby('user_id')['click_timestamp', 'created_at_ts'].apply(lambda x: mean_diff_time_func(x, 'created_at_ts'))

plt.plot(sorted(mean_diff_created_time.values, reverse=True))

# 用户前后点击文章的相似性分布
item_idx_2_rawid_dict = dict(zip(item_emb_df['article_id'], item_emb_df.index))

del item_emb_df['article_id']

item_emb_np = np.ascontiguousarray(item_emb_df.values, dtype=np.float32)

# 随机选择5个用户，查看这些用户前后查看文章的相似性
sub_user_ids = np.random.choice(user_click_merge.user_id.unique(), size=15, replace=False)
sub_user_info = user_click_merge[user_click_merge['user_id'].isin(sub_user_ids)]
sub_user_info.head()
• 5

def get_item_sim_list(df):
    sim_list = []
    item_list = df['click_article_id'].values
    for i in range(0, len(item_list)-1):
        emb1 = item_emb_np[item_idx_2_rawid_dict[item_list[i]]]
        emb2 = item_emb_np[item_idx_2_rawid_dict[item_list[i+1]]]
        sim_list.append(np.dot(emb1,emb2)/(np.linalg.norm(emb1)*(np.linalg.norm(emb2))))
    sim_list.append(0)
    return sim_list

for _, user_df in sub_user_info.groupby('user_id'):
    item_sim_list = get_item_sim_list(user_df)
    plt.plot(item_sim_list)