ML之NB、LoR：基于NB和LoR算法对Kaggle IMDB影评数据集(国外类似豆瓣电影)情感分析进行分类

1. #1、读取数据并做一些基本的预处理(比如说把评论部分的html标签去掉等等)
2. def review_to_wordlist(review):
3. '''
4.     把IMDB的评论转成词序列
5.     '''
6.     review_text = BeautifulSoup(review,"lxml").get_text()  # 去掉HTML标签，拿到内容
7.     review_text = re.sub("[^a-zA-Z]"," ", review_text)   # 用正则表达式取出符合规范的部分
8.     words = review_text.lower().split()   # 小写化所有的词，并转成词list
9. return words   # 返回words
10. 
11. # 使用pandas读入训练和测试csv文件
12. train = pd.read_csv('F:/File_Python/Resources/Kaggle Film critic emotion/labeledTrainData.tsv', header=0, delimiter="\t", quoting=3)
13. test = pd.read_csv('F:/File_Python/Resources/Kaggle Film critic emotion/testData.tsv', header=0, delimiter="\t", quoting=3 )
14. y_train = train['sentiment']  # 取出情感标签，positive/褒 或者 negative/贬
15. train_data = []               # 将训练和测试数据都转成词list
16. for i in range(0,len(train['review'])):
17.     train_data.append(" ".join(review_to_wordlist(train['review'][i])))
18. test_data = []
19. for i in range(0,len(test['review'])):
20.     test_data.append(" ".join(review_to_wordlist(test['review'][i])))
21. print(train_data)
22. print(y_train)
23. 
24. #2、特征处理：从数据里面拿到有区分度的特征，采用TF-IDF向量方法
25. from sklearn.feature_extraction.text import TfidfVectorizer as TFIV
26. # 初始化TFIV对象，去停用词，加2元语言模型
27. tfv = TFIV(min_df=3,  max_features=None, strip_accents='unicode', analyzer='word',token_pattern=r'\w{1,}', ngram_range=(1, 2), use_idf=1,smooth_idf=1,sublinear_tf=1, stop_words = 'english')
28. 
29. X_all = train_data + test_data  # 合并训练和测试集以便进行TFIDF向量化操作
30. len_train = len(train_data)
31. 
32. tfv.fit(X_all)    
33. X_all = tfv.transform(X_all)
34. 
35. X = X_all[:len_train]   # 恢复成训练集和测试集部分
36. X_test = X_all[len_train:]
37. 
38. 
39. # 3、利用NB算法，多项式朴素贝叶斯
40. from sklearn.naive_bayes import MultinomialNB as MNB
41. 
42. model_NB = MNB()
43. model_NB.fit(X, y_train) #特征数据直接灌进来
44. MNB(alpha=1.0, class_prior=None, fit_prior=True)
45. 
46. from sklearn.cross_validation import cross_val_score
47. import numpy as np
48. print ("多项式贝叶斯分类器20折交叉验证得分: ", np.mean(cross_val_score(model_NB, X, y_train, cv=20, scoring='roc_auc')))
49. 
50. 
51. 
52. #4、利用LoR算法
53. from sklearn.linear_model import LogisticRegression as LR
54. from sklearn.grid_search import GridSearchCV
55. 
56. # 设定grid search的参数
57. grid_values = {'C':[30]}  
58. # 设定打分为roc_auc
59. model_LR = GridSearchCV(LR(penalty = 'L2', dual = True, random_state = 0), grid_values, scoring = 'roc_auc', cv = 20) 
60. # 数据灌进来
61. model_LR.fit(X,y_train)
62. # 20折交叉验证，开始漫长的等待...
63. GridSearchCV(cv=20, estimator=LogisticRegression(C=1.0, class_weight=None, dual=True, 
64.              fit_intercept=True, intercept_scaling=1, penalty='L2', random_state=0, tol=0.0001),
65.         fit_params={}, iid=True, loss_func=None, n_jobs=1,
66.         param_grid={'C': [30]}, pre_dispatch='2*n_jobs', refit=True,
67.         score_func=None, scoring='roc_auc', verbose=0)
68. #输出结果
69. print (model_LR.grid_scores_)
70. 
71.