使用神经网络-垃圾邮件检测-LSTM或者CNN(一维卷积)效果都不错【代码有问题,pass】

简介:
复制代码
from sklearn.feature_extraction.text import CountVectorizer
import os
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import train_test_split
from sklearn import metrics
import matplotlib.pyplot as plt
import numpy as np
from sklearn import svm
from sklearn.feature_extraction.text import TfidfTransformer
import tensorflow as tf
import tflearn
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_1d, global_max_pool
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.merge_ops import merge
from tflearn.layers.estimator import regression
from tflearn.data_utils import to_categorical, pad_sequences
from sklearn.neural_network import MLPClassifier
from tflearn.layers.normalization import local_response_normalization
from tensorflow.contrib import learn


max_features=500
max_document_length=1024



def load_one_file(filename):
    x=""
    with open(filename) as f:
        for line in f:
            line=line.strip('\n')
            line = line.strip('\r')
            x+=line
    return x

def load_files_from_dir(rootdir):
    x=[]
    list = os.listdir(rootdir)
    for i in range(0, len(list)):
        path = os.path.join(rootdir, list[i])
        if os.path.isfile(path):
            v=load_one_file(path)
            x.append(v)
    return x

def load_all_files():
    ham=[]
    spam=[]
    for i in range(1,5):
        path="../data/mail/enron%d/ham/" % i
        print "Load %s" % path
        ham+=load_files_from_dir(path)
        path="../data/mail/enron%d/spam/" % i
        print "Load %s" % path
        spam+=load_files_from_dir(path)
    return ham,spam

def get_features_by_wordbag():
    ham, spam=load_all_files()
    x=ham+spam
    y=[0]*len(ham)+[1]*len(spam)
    vectorizer = CountVectorizer(
                                 decode_error='ignore',
                                 strip_accents='ascii',
                                 max_features=max_features,
                                 stop_words='english',
                                 max_df=1.0,
                                 min_df=1 )
    print vectorizer
    x=vectorizer.fit_transform(x)
    x=x.toarray()
    return x,y

def show_diffrent_max_features():
    global max_features
    a=[]
    b=[]
    for i in range(1000,20000,2000):
        max_features=i
        print "max_features=%d" % i
        x, y = get_features_by_wordbag()
        x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.4, random_state=0)
        gnb = GaussianNB()
        gnb.fit(x_train, y_train)
        y_pred = gnb.predict(x_test)
        score=metrics.accuracy_score(y_test, y_pred)
        a.append(max_features)
        b.append(score)
        plt.plot(a, b, 'r')
    plt.xlabel("max_features")
    plt.ylabel("metrics.accuracy_score")
    plt.title("metrics.accuracy_score VS max_features")
    plt.legend()
    plt.show()

def do_nb_wordbag(x_train, x_test, y_train, y_test):
    print "NB and wordbag"
    gnb = GaussianNB()
    gnb.fit(x_train,y_train)
    y_pred=gnb.predict(x_test)
    print metrics.accuracy_score(y_test, y_pred)
    print metrics.confusion_matrix(y_test, y_pred)

def do_svm_wordbag(x_train, x_test, y_train, y_test):
    print "SVM and wordbag"
    clf = svm.SVC()
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)
    print metrics.accuracy_score(y_test, y_pred)
    print metrics.confusion_matrix(y_test, y_pred)

def get_features_by_wordbag_tfidf():
    ham, spam=load_all_files()
    x=ham+spam
    y=[0]*len(ham)+[1]*len(spam)
    vectorizer = CountVectorizer(binary=True,
                                 decode_error='ignore',
                                 strip_accents='ascii',
                                 max_features=max_features,
                                 stop_words='english',
                                 max_df=1.0,
                                 min_df=1 )
    print vectorizer
    x=vectorizer.fit_transform(x)
    x=x.toarray()
    transformer = TfidfTransformer(smooth_idf=False)
    print transformer
    tfidf = transformer.fit_transform(x)
    x = tfidf.toarray()
    return  x,y


def do_cnn_wordbag(trainX, testX, trainY, testY):
    global max_document_length
    print "CNN and tf"

    trainX = pad_sequences(trainX, maxlen=max_document_length, value=0.)
    testX = pad_sequences(testX, maxlen=max_document_length, value=0.)
    # Converting labels to binary vectors
    trainY = to_categorical(trainY, nb_classes=2)
    testY = to_categorical(testY, nb_classes=2)

    # Building convolutional network
    network = input_data(shape=[None,max_document_length], name='input')
    network = tflearn.embedding(network, input_dim=1000000, output_dim=128)
    branch1 = conv_1d(network, 128, 3, padding='valid', activation='relu', regularizer="L2")
    branch2 = conv_1d(network, 128, 4, padding='valid', activation='relu', regularizer="L2")
    branch3 = conv_1d(network, 128, 5, padding='valid', activation='relu', regularizer="L2")
    network = merge([branch1, branch2, branch3], mode='concat', axis=1)
    network = tf.expand_dims(network, 2)
    network = global_max_pool(network)
    network = dropout(network, 0.8)
    network = fully_connected(network, 2, activation='softmax')
    network = regression(network, optimizer='adam', learning_rate=0.001,
                         loss='categorical_crossentropy', name='target')
    # Training
    model = tflearn.DNN(network, tensorboard_verbose=0)
    model.fit(trainX, trainY,
              n_epoch=5, shuffle=True, validation_set=(testX, testY),
              show_metric=True, batch_size=100,run_id="spam")

def do_rnn_wordbag(trainX, testX, trainY, testY):
    global max_document_length
    print "RNN and wordbag"

    trainX = pad_sequences(trainX, maxlen=max_document_length, value=0.)
    testX = pad_sequences(testX, maxlen=max_document_length, value=0.)
    # Converting labels to binary vectors
    trainY = to_categorical(trainY, nb_classes=2)
    testY = to_categorical(testY, nb_classes=2)

    # Network building
    net = tflearn.input_data([None, max_document_length])
    net = tflearn.embedding(net, input_dim=10240000, output_dim=128)
    net = tflearn.lstm(net, 128, dropout=0.8)
    net = tflearn.fully_connected(net, 2, activation='softmax')
    net = tflearn.regression(net, optimizer='adam', learning_rate=0.001,
                             loss='categorical_crossentropy')

    # Training
    model = tflearn.DNN(net, tensorboard_verbose=0)
    model.fit(trainX, trainY, validation_set=(testX, testY), show_metric=True,
              batch_size=10,run_id="spm-run",n_epoch=5)


def do_dnn_wordbag(x_train, x_test, y_train, y_testY):
    print "DNN and wordbag"

    # Building deep neural network
    clf = MLPClassifier(solver='lbfgs',
                        alpha=1e-5,
                        hidden_layer_sizes = (5, 2),
                        random_state = 1)
    print  clf
    clf.fit(x_train, y_train)
    y_pred = clf.predict(x_test)
    print metrics.accuracy_score(y_test, y_pred)
    print metrics.confusion_matrix(y_test, y_pred)



def  get_features_by_tf():
    global  max_document_length
    x=[]
    y=[]
    ham, spam=load_all_files()
    x=ham+spam
    y=[0]*len(ham)+[1]*len(spam)
    vp=tflearn.data_utils.VocabularyProcessor(max_document_length=max_document_length,
                                              min_frequency=0,
                                              vocabulary=None,
                                              tokenizer_fn=None)
    x=vp.fit_transform(x, unused_y=None)
    x=np.array(list(x))
    return x,y



if __name__ == "__main__":
    print "Hello spam-mail"
    #print "get_features_by_wordbag"
    #x,y=get_features_by_wordbag()
    #x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.4, random_state = 0)

    #print "get_features_by_wordbag_tfidf"
    #x,y=get_features_by_wordbag_tfidf()
    #x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.4, random_state = 0)
    #NB
    #do_nb_wordbag(x_train, x_test, y_train, y_test)
    #show_diffrent_max_features()

    #SVM
    #do_svm_wordbag(x_train, x_test, y_train, y_test)

    #DNN
    #do_dnn_wordbag(x_train, x_test, y_train, y_test)

    print "get_features_by_tf"
    x,y=get_features_by_wordbag()
    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.4, random_state = 0)
    #CNN
    do_cnn_wordbag(x_train, x_test, y_train, y_test)


    #RNN
    #do_rnn_wordbag(x_train, x_test, y_train, y_test)
复制代码

自己写检测算法的时候也记得多个算法比较下

posted on  2017-11-28 11:08  bonelee 阅读( 75) 评论( 3编辑  收藏

  
#1楼 [ 楼主2017-12-11 16:12 |  bonelee   
感觉有些问题,因为LSTM要记住序列位置,而词袋模型,破坏了序列先后顺序!
http://www.cnblogs.com/bonelee/p/7639920.html
见这个DGA LSTM的文章!!!才应该是正解!
  
#2楼 [ 楼主2017-12-11 16:23 |  bonelee   
代码确实有问题!
RNN或者CNN都应该是针对原始数据进行编码!用:
Vocabulary Processor

tflearn.data_utils.VocabularyProcessor (max_document_length, min_frequency=0, vocabulary=None, tokenizer_fn=None)

Maps documents to sequences of word ids.



tf.contrib.learn.preprocessing.VocabularyProcessor (max_document_length, min_frequency=0, vocabulary=None, tokenizer_fn=None)

参数:
max_document_length: 文档的最大长度。如果文本的长度大于最大长度,那么它会被剪切,反之则用0填充。
min_frequency: 词频的最小值,出现次数小于最小词频则不会被收录到词表中。
vocabulary: CategoricalVocabulary 对象。
tokenizer_fn:分词函数

代码:

from tensorflow.contrib import learn
import numpy as np
max_document_length = 4
x_text =[
'i love you',
'me too'
]
vocab_processor = learn.preprocessing.VocabularyProcessor(max_document_length)
vocab_processor.fit(x_text)
print next(vocab_processor.transform(['i me too'])).tolist()
x = np.array(list(vocab_processor.fit_transform(x_text)))
print x

[1, 4, 5, 0]
[[1 2 3 0]
[4 5 0 0]]

文档地址: http://tflearn.org/data_utils/
























本文转自张昺华-sky博客园博客,原文链接:http://www.cnblogs.com/bonelee/p/7908573.html,如需转载请自行联系原作者


相关文章
|
9天前
|
机器学习/深度学习 算法 计算机视觉
卷积神经网络(CNN)的工作原理深度解析
【6月更文挑战第14天】本文深度解析卷积神经网络(CNN)的工作原理。CNN由输入层、卷积层、激活函数、池化层、全连接层和输出层构成。卷积层通过滤波器提取特征,激活函数增加非线性,池化层降低维度。全连接层整合特征,输出层根据任务产生预测。CNN通过特征提取、整合、反向传播和优化进行学习。尽管存在计算量大、参数多等问题,但随着技术发展,CNN在计算机视觉领域的潜力将持续增长。
|
3天前
|
机器学习/深度学习 算法 计算机视觉
没有公式,不要代码,让你理解 RCNN:目标检测中的区域卷积神经网络
没有公式,不要代码,让你理解 RCNN:目标检测中的区域卷积神经网络
5 0
没有公式,不要代码,让你理解 RCNN:目标检测中的区域卷积神经网络
|
6天前
|
监控 网络协议 安全
Verilog代码在上网行为管理软件中的网络设备控制与协议分析
**Verilog摘要:** Verilog是硬件描述语言,用于数字电路设计和网络设备控制。它在上网行为管理软件中用于控制路由器、交换机等,例如通过简单模块控制端口状态。此外,Verilog还支持协议分析,如解析以太网帧提取MAC地址。监控数据可结合Verilog实现自动化提交,例如通过HTTP发送请求到服务器,实现实时监控和响应。这些示例展示了Verilog在网络安全和性能优化中的应用潜力。
40 1
|
3天前
|
机器学习/深度学习 算法 语音技术
基于语音信号MFCC特征提取和GRNN神经网络的人员身份检测算法matlab仿真
**语音识别算法概览** MATLAB2022a中实现,结合MFCC与GRNN技术进行说话人身份检测。MFCC利用人耳感知特性提取语音频谱特征,GRNN作为非线性映射工具,擅长序列学习,确保高效识别。预加重、分帧、加窗、FFT、滤波器组、IDCT构成MFCC步骤,GRNN以其快速学习与鲁棒性处理不稳定数据。适用于多种领域。
|
3天前
|
机器学习/深度学习 自然语言处理 TensorFlow
深入浅出:理解和实现深度学习中的卷积神经网络(CNN)
在当今的数据驱动世界,深度学习已经成为许多领域的关键技术。本文将深入探讨卷积神经网络(CNN)的原理、结构和应用,旨在帮助读者全面理解这项强大的技术,并提供实际的实现技巧。
18 0
|
4天前
|
机器学习/深度学习 人工智能 算法
【机器学习】深度神经网络(DNN):原理、应用与代码实践
【机器学习】深度神经网络(DNN):原理、应用与代码实践
15 0
|
5天前
|
机器学习/深度学习 自然语言处理 PyTorch
【自然语言处理NLP】Bert预训练模型、Bert上搭建CNN、LSTM模型的输入、输出详解
【自然语言处理NLP】Bert预训练模型、Bert上搭建CNN、LSTM模型的输入、输出详解
23 0
|
9天前
|
机器学习/深度学习 自然语言处理 前端开发
深度学习-[数据集+完整代码]基于卷积神经网络的缺陷检测
深度学习-[数据集+完整代码]基于卷积神经网络的缺陷检测
|
10天前
|
机器学习/深度学习
【从零开始学习深度学习】23. CNN中的多通道输入及多通道输出计算方式及1X1卷积层介绍
【从零开始学习深度学习】23. CNN中的多通道输入及多通道输出计算方式及1X1卷积层介绍
【从零开始学习深度学习】23. CNN中的多通道输入及多通道输出计算方式及1X1卷积层介绍
|
10天前
|
机器学习/深度学习 Shell
【从零开始学习深度学习】22. 卷积神经网络(CNN)中填充(padding)与步幅(stride)详解,填充、步幅、输入及输出之间的关系
【从零开始学习深度学习】22. 卷积神经网络(CNN)中填充(padding)与步幅(stride)详解,填充、步幅、输入及输出之间的关系

热门文章

最新文章