源码请到：自然语言处理练习: 学习自然语言处理时候写的一些代码 (gitee.com)

数据来源：麦卡里价格建议挑战Mercari Price Suggestion Challenge | Kaggle

如果不会使用魔法可以使用百度云

链接：https://pan.baidu.com/s/1EM2MwjX4bLlypLSIJYZqeg?pwd=xqs0

提取码：xqs0

一、加载数据集

拿到数据集后首先对数据集的数据进行一些显示展示，来了解数据集

# 设置pandas显示配置pd.set_option('display.max_columns', 1000)
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)
train=pd.read_csv('data/train.tsv', sep='\t')
test=pd.read_csv('data/test.tsv', sep='\t')
# 查看数据集大小print(train.shape)
print(test.shape)
# 查看数据集列名print(train.dtypes)
# 查看数据集前几行print(train.head())
# 查看商品价格信息print(train.price.describe())

二、分析影响价格的因素

经过第一步已经大致了解了数据集的内容以及价格的范围大小及均值，下面我们进一步对影响价格的因素进行分析

2.1 对价格进行对数变换，比较变换后价格的分布情况

由于价格的分布十分的散乱，所以对价格进行对数变换处理

# 对价格进行对数变换，比较转换前和转换后的分布情况plt.subplot(1, 2, 1)
(train['price']).plot.hist(bins=50, figsize=(20, 10), edgecolor='white', range=[0, 250])
plt.xlabel('price+', fontsize=17)
plt.ylabel('frequency', fontsize=17)
plt.tick_params(labelsize=15)
plt.title('Price Distribution - Training Set', fontsize=17)
plt.subplot(1, 2, 2)
np.log(train['price'] +1).plot.hist(bins=50, figsize=(20, 10), edgecolor='white')
plt.xlabel('log(price+1)', fontsize=17)
plt.ylabel('frequency', fontsize=17)
plt.tick_params(labelsize=15)
plt.title('Log(Price) Distribution - Training Set', fontsize=17)
plt.show()

可以看出，原数据十分不均衡，高价区区间大数量少，经过对数处理的数据更加符合正态分布

2.2 包邮对价格的影响

接下来分析商户包邮是否对价格产生影响

# 运费承担：大概有55%卖家承担运费print(train.shipping.value_counts() /len(train))

可以看出，大概有百分之五十五的商户包邮

# 看下运费不同情况下的价格变化(包邮的价格贵一些）prc_shipBySeller=train.loc[train.shipping==0, 'price']
prc_shipByBuyer=train.loc[train.shipping==1, 'price']
fig, ax=plt.subplots(figsize=(20, 10))
ax.hist(np.log(prc_shipBySeller+1), color='#8CB4E1', alpha=1.0, bins=50, label='Price when Seller pays Shipping')
ax.hist(np.log(prc_shipByBuyer+1), color='#007D00', alpha=0.7, bins=50, label='Price when Buyer pays Shipping')
ax.set(title='Histogram Comparison', ylabel='% of Dataset in Bin')
plt.legend()
plt.xlabel('log(price+1)', fontsize=17)
plt.ylabel('frequency', fontsize=17)
plt.title('Price Distribution by Shipping Type', fontsize=17)
plt.tick_params(labelsize=15)
plt.show()

可以看出，包邮的整体价格比不包邮的价格贵一些

2.3 商品类别对价格的影响

首先我们统计下商品的类别

# 商品类别划分print('There are %d unique values in the category column'%train['category_name'].nunique())
print(train['category_name'].value_counts()[:5])
print('There are %d items that do not have a label'%train['category_name'].isnull().sum())

可以看出总共有1287种商品，并且展示了数量前五的商品，还有6327件商品没有类别标签，我们处理的时候就忽略这些商品

商品类别太多了，可以看到商品类别结构为主类/子类1/子类2的格式，我们进行拆分，将类别合并一些

# 商品类别太多了，合并一下defsplit_cat(text):
try:
returntext.split("/")
except:
return"No Label", "No Label", "No Label"train['general_cat'], train['subcat_1'], train['subcat_2'] =zip(*train['category_name'].apply(lambdax: split_cat(x)))
print(train.head())
test['general_cat'], test['subcat_1'], test['subcat_2'] =zip(*test['category_name'].apply(lambdax: split_cat(x)))
print('There are %d unique general_cat'%train['general_cat'].nunique())
print('There are %d unique first sub-categories'%train['subcat_1'].nunique())
print('There are %d unique second sub-categories'%train['subcat_2'].nunique())

可以看出总共有11个主类，114个子类1，871个子类2

接下来分析下主类的分布情况

# 主类别分布情况x=train['general_cat'].value_counts().index.values.astype('str')
y=train['general_cat'].value_counts().valuespct= [('%.2f'% (v*100)) +'%'forvin (y/len(train))]
tracel=go.Bar(x=x, y=y, text=pct)
layout=dict(title="Number of Items by Main Category", yaxis=dict(title='Count'), xaxis=dict(title='Category'))
fig=dict(data=[tracel], layout=layout)
py.iplot(fig)

可以看出，大量商品是关于女这种类别的，占据了百分之四十五，第二多的是化妆品，第三多的是孩子。

子类的数量很多，我们展示前15个子类的分布

# 前15个子类别分布情况x=train['subcat_1'].value_counts().index.values.astype('str')[:15]
y=train['subcat_1'].value_counts().values[:15]
pct= [('%.2f'% (v*100)) +'%'forvin (y/len(train))][:15]
tracel=go.Bar(x=x, y=y, text=pct, marker=dict(color=y, colorscale='Portland', showscale=True, reversescale=False))
layout=dict(title="Number of Items by Sub Category(Top 15)", yaxis=dict(title='Count'),
xaxis=dict(title='SubCategory'))
fig=dict(data=[tracel], layout=layout)
py.iplot(fig)

接下来看看不同主类商品的价格区间

# 不同类型商品价格浮动区间general_cats=train['general_cat'].unique()
x= [train.loc[train['general_cat'] ==cat, 'price'] forcatingeneral_cats]
data= [go.Box(x=np.log(x[i] +1), name=general_cats[i]) foriinrange(len(general_cats))]
layout=dict(title='Price Distribution by General Category', yaxis=dict(title='Frequency'),
xaxis=dict(title='Category'))
fig=dict(data=data, layout=layout)
py.iplot(fig)

2.4 品牌分布情况

分析一下品牌的分布情况

# 前10品牌名称的数据分布x=train['brand_name'].value_counts().index.values.astype('str')[:10]
y=train['brand_name'].value_counts().values[:10]
tracel=go.Bar(x=x, y=y, marker=dict(color=y, colorscale='Portland', showscale=True, reversescale=False))
layout=dict(title="Top 10 Brand by Number of Items", yaxis=dict(title='Count'), xaxis=dict(title='Brand Name'))
fig=dict(data=[tracel], layout=layout)
py.iplot(fig)

2.5 商品描述长度对商品的影响

统计商品描述的长度，然后研究其对商品的影响

# 商品描述对价格的影响defwordCount(text):
try:
text=text.lower()
regex=re.compile('['+re.escape(string.punctuation) +'0-9\\r\\t\\n]')
txt=regex.sub(' ', text)
words= [wforwintxt.split(" ") ifwnotinstop_words.STOP_WORDSandlen(w) >3]
returnlen(words)
except:
return0train['desc_len'] =train['item_description'].apply(lambdax: wordCount(x))
test['desc_len'] =test['item_description'].apply(lambdax: wordCount(x))
print(train.head())
df=train.groupby('desc_len')['price'].mean().reset_index()
tracel=go.Scatter(x=df['desc_len'], y=np.log(df['price'] +1), mode='lines+markers', name='lines+markers')
layout=dict(title='Average Log(Price) by Description Length', yaxis=dict(title='Average Log(Price)'),
xaxis=dict(title='Description Length'))
fig=dict(data=[tracel], layout=layout)
py.iplot(fig)

可以看到商品描述适中价格越高，描述短的可能因为功能简单所以价格低，描述长的可能因为小众所以价格低

三、商品描述关键字

3.1 统计常用关键字

统计一下商品描述中常用的关键字，注意，有的商品没有商品描述，需要去掉

print(train.item_description.isnull().sum())
# 去掉缺失值train=train[pd.notnull(train['item_description'])]
# 提取每种品牌的描述关键词tokenize=nltk.data.load('tokenizers/punkt/english.pickle')
cat_desc=dict()
forcatingeneral_cats:
text=' '.join(train.loc[train['general_cat'] ==cat, 'item_description'].values)
cat_desc[cat] =tokenize.tokenize(text)
# 统计常用关键词flat_lst= [itemforsublistinlist(cat_desc.values()) foriteminsublist]
allWordsCount=Counter(flat_lst)
all_top10=allWordsCount.most_common(20)
x= [w[0] forwinall_top10]
y= [w[1] forwinall_top10]
tracel=go.Bar(x=x, y=y)
layout=dict(title='Word Frequency', yaxis=dict(title='Count'), xaxis=dict(title='Word'))
fig=dict(data=[tracel], layout=layout)
py.iplot(fig)

3.2 分别展示不同商品的关键字

首先将商品描述进行分词，去掉停用词

# 展示不同商品的关键词stop=set(stopwords.words('english'))
deftokenize(text):
try:
regex=re.compile('['+re.escape(string.punctuation) +'0-9\\r\\t\\n]')
text=regex.sub(' ', text)
tokens_= [word_tokenize(s) forsinsent_tokenize(text)]
tokens= []
fortoken_by_sentintokens_:
tokens+=token_by_senttokens=list(filter(lambdat: t.lower() notinstop, tokens))
filtered_tokens= [wforwintokensifre.search('[a-zA-Z]', w)]
filtered_tokens= [w.lower() forwinfiltered_tokensiflen(w) >=3]
returnfiltered_tokensexceptTypeErrorase:
print(text, e)
train['tokens'] =train['item_description'].map(tokenize)
test['tokens'] =test['item_description'].map(tokenize)
train.reset_index(drop=True, inplace=True)
test.reset_index(drop=True, inplace=True)
fordescription, tokensinzip(train['item_description'].head(), train['tokens'].head()):
print('description:', description)
print('tokens:', tokens)
print()

对每个类别提取数量最多的前100个关键字统计词频生成词云

cat_desc=dict()
forcatingeneral_cats:
text=' '.join(train.loc[train['general_cat'] ==cat, 'item_description'].values)
cat_desc[cat] =tokenize(text)
cat_desc100=dict()
forkey, valueincat_desc.items():
cat_desc100[key] =Counter(value).most_common()
defgenerate_wordcloud(tup):
wordcloud=WordCloud(background_color='white', max_words=50, max_font_size=40, random_state=42).generate(str(tup))
returnwordcloudfig, axes=plt.subplots(len(cat_desc100) //2+1, 2, figsize=(30, 15))
fori, (key, cat) inenumerate(cat_desc100.items()):
ax=axes[i//2, i%2]
ax.imshow(generate_wordcloud(cat), interpolation='bilinear')
ax.axis('off')
ax.set_title("%s Top 100"%key, fontsize=12)
plt.show()

四、tfidf算法

可以看出不同类别出现的关键字有很多是相似的，不能代表这种类别的商品，所以我们使用tf-idf算法进行关键字的挖掘，tf-idf基本思想是词在本文章中出现的次数越多在其他文章中出现的次数越少越可能是关键字。首先将描述数据扩展到180000维度，在进行tf-idf打分

# tf-idfvectorizer=TfidfVectorizer(min_df=10, max_features=180000, tokenizer=tokenize, ngram_range=(1, 2))
all_desc=np.append(train['item_description'].values, test['item_description'].values)
vz=vectorizer.fit_transform(list(all_desc))
print(vz.shape)
tfidf=dict(zip(vectorizer.get_feature_names_out(), vectorizer.idf_))
tfidf=pd.DataFrame(columns=['tfidf']).from_dict(dict(tfidf), orient='index')
tfidf.columns= ['tfidf']
print(tfidf.sort_values(by=['tfidf'], ascending=True).head(10))
print(tfidf.sort_values(by=['tfidf'], ascending=False).head(10))

可以看出停用词的得分值基本上都比较低，因为他们虽然频率高但是不具备什么代表性的价值，而另一批的词得分就很高，可以作为关键词来分析

接下来使用SVD降维将特征的维度降到50，然后使用t-SNE将维度降维到2进行展示

trn=train.copy()
tst=test.copy()
trn['is_train'] =1tst['is_train'] =0sample_sz=15000# 采样combined_df=pd.concat([trn, tst])
combined_sample=combined_df.sample(n=sample_sz)
vz_sample=vectorizer.fit_transform(list(combined_sample['item_description']))
# SVD 降维n_comp=30svd=TruncatedSVD(n_components=n_comp, random_state=42)
svd_tfidf=svd.fit_transform(vz_sample)
# t-SNE降维tsne_model=TSNE(n_components=2, verbose=1, random_state=42, n_iter=500)
tsne_tfidf=tsne_model.fit_transform(svd_tfidf)
plot_tfidf=bp.figure(width=700, height=600, title='tf-idf clustring of the item description',
tools='pan, wheel_zoom, box_zoom, reset, hover', x_axis_type=None, y_axis_type=None,
min_border=1)
combined_sample.reset_index(inplace=True, drop=True)
tfidf_df=pd.DataFrame(tsne_tfidf, columns=['x', 'y'])
tfidf_df['description'] =combined_sample['item_description']
tfidf_df['tokens'] =combined_sample['tokens']
tfidf_df['category'] =combined_sample['general_cat']
plot_tfidf.scatter(x='x', y='y', source=tfidf_df, alpha=0.7)
hover=plot_tfidf.select(dict(type=HoverTool))
hover.tooltips= {'description': '@description', 'tokens': '@tokens', 'category': '@category'}
show(plot_tfidf)

关键词比较接近的就会被绘制在一个点位置

五、分类

5.1 使用聚类算法对上面数据的点可以进行分类

# 聚类分堆num_clusters=10kmeans_model=MiniBatchKMeans(n_clusters=num_clusters, init='k-means++', n_init=1, init_size=10000, batch_size=1000,
verbose=0, max_iter=1000)
kmeans_model.fit(vz_sample)
kmeans_clusters=kmeans_model.predict(vz_sample)
kmeans_distances=kmeans_model.transform(vz_sample)
tsne_kmeans=tsne_model.fit_transform(kmeans_distances)
kmeans_df=pd.DataFrame(tsne_kmeans, columns=['x', 'y'])
kmeans_df['cluster'] =kmeans_clusterskmeans_df['description'] =combined_sample['item_description']
kmeans_df['category'] =combined_sample['general_cat']
plot_kmeans=bp.figure(width=700, height=600, title='KMeans clustering of the description',
tools='pan, wheel_zoom, box_zoom, reset, hover', x_axis_type=None, y_axis_type=None,
min_border=1)
print(kmeans_clusters)
colormap= {'0': 'red', '1': 'green', '2': 'blue', '3': 'black', '4': 'yellow', '5': 'pink', '6': 'purple', '7': 'grey',
'8': 'brown', '9': 'orange'}
defget_color(num):
ifnum==0:
return'red'elifnum==1:
return'green'elifnum==2:
return'blue'elifnum==3:
return'black'elifnum==4:
return'yellow'elifnum==5:
return'pink'elifnum==6:
return'purple'elifnum==7:
return'grey'elifnum==8:
return'brown'elifnum==9:
return'orange'color=pd.Series(kmeans_clusters).apply(get_color)
source=ColumnDataSource(
data=dict(x=kmeans_df['x'], y=kmeans_df['y'], color=color, description=kmeans_df['description'],
category=kmeans_df['category'], cluster=kmeans_df['cluster']))
plot_kmeans.scatter(x='x', y='y', color='color', source=source)
hover=plot_kmeans.select(dict(type=HoverTool))
hover.tooltips= {'description': '@description', 'category': '@category', 'cluster': '@cluster'}
show(plot_kmeans)

5.2 LDA主题模型分类

除了聚类算法外，也可以使用LDA主题模型进行分类

# LDA分堆cvectorizer=CountVectorizer(min_df=4, max_features=180000, tokenizer=tokenize, ngram_range=(1, 2))
cvz=cvectorizer.fit_transform(combined_sample['item_description'])
lda_model=LatentDirichletAllocation(n_components=10, learning_method='online', max_iter=20, random_state=42)
X_topics=lda_model.fit_transform(cvz)
# 获取主题n_top_words=10topic_summaries= []
topic_word=lda_model.components_vocab=cvectorizer.get_feature_names_out()
fori, topic_distinenumerate(topic_word):
topic_words=np.array(vocab)[np.argsort(topic_dist)][:-(n_top_words+1):-1]
topic_summaries.append(' '.join(topic_words))
print('Topic {}:{}'.format(i, '|'.join(topic_words)))
tsne_lda=tsne_model.fit_transform(X_topics)
unnormalized=np.matrix(X_topics)
doc_topic=unnormalized/unnormalized.sum(axis=1)
lda_keys= []
fori, tweetinenumerate(combined_sample['item_description']):
lda_keys+= [doc_topic[i].argmax()]
lda_df=pd.DataFrame(tsne_lda, columns=['x', 'y'])
lda_df['description'] =combined_sample['item_description']
lda_df['category'] =combined_sample['general_cat']
lda_df['topic'] =lda_keyslda_df['topic'] =lda_df['topic'].map(int)
plot_lda=bp.figure(width=700, height=600, title='LDA topic visualization',
tools='pan, wheel_zoom, box_zoom, reset, hover', x_axis_type=None, y_axis_type=None, min_border=1)
source=ColumnDataSource(
data=dict(x=lda_df['x'], y=lda_df['y'], color=color, description=lda_df['description'],
topic=lda_df['topic'], category=lda_df['category']))
plot_lda.scatter(x='x', y='y', color='color', source=source)
hover=plot_lda.select(dict(type=HoverTool))
hover.tooltips= {'description': '@description', 'topic': '@topic', 'category': '@category'}
show(plot_lda)