机器学习自然语言处理之英文NLTK（代码+原理）

from nltk.tokenize import sent_tokenize
text="""Hello Mr. Smith, how are you doing today? The weather is great, and 
city is awesome.The sky is pinkish-blue. You shouldn't eat cardboard"""
tokenized_text=sent_tokenize(text)
print(tokenized_text)
'''
结果：
  ['Hello Mr. Smith, how are you doing today?', 
   'The weather is great, and city is awesome.The sky is pinkish-blue.', 
   "You shouldn't eat cardboard"]
'''

import nltk
sent = "I am almost dead this time"
token = nltk.word_tokenize(sent)
# 结果：token['I','am','almost','dead','this','time']

import string
"""移除标点符号"""
if __name__ == '__main__':
    # 方式一
    # s = 'abc.'
    text_list = "Hello Mr. Smith, how are you doing today? The weather is great, and city is awesome."
    text_list = text_list.translate(str.maketrans(string.punctuation, " " * len(string.punctuation)))  # abc
    print("s: ", text_list)
    # 方式二
    english_punctuations = [',', '.', ':', ';', '?', '(', ')', '[', ']', '&', '!', '*', '@', '#', '$', '%']
    text_list = [word for word in text_list if word not in english_punctuations]
    print("text: ", text_list)

import nltk
from nltk.corpus import stopwords
# nltk.download('stopwords')
# Downloading package stopwords to 
# C:\Users\Administrator\AppData\Roaming\nltk_data\corpora\stopwords.zip.
# Unzipping the stopwords.zip
"""移除停用词""" 
stop_words = stopwords.words("english")
if __name__ == '__main__':
    text = "Hello Mr. Smith, how are you doing today? The weather is great, and city is awesome."
    word_tokens = nltk.tokenize.word_tokenize(text.strip())
    filtered_word = [w for w in word_tokens if not w in stop_words]
    print("word_tokens: ", word_tokens)
    print("filtered_word: ", filtered_word)
    '''
    word_tokens：['Hello', 'Mr.', 'Smith', ',', 'how', 'are', 'you', 'doing', 'today', '?',
     'The', 'weather', 'is', 'great', ',', 'and', 'city', 'is', 'awesome', '.']
    filtered_word：['Hello', 'Mr.', 'Smith', ',', 'today', '?', 'The', 'weather', 'great', ',', 'city', 'awesome', '.']
    '''

from nltk.stem import WordNetLemmatizer  
lemmatizer = WordNetLemmatizer()  
lemmatizer.lemmatize('leaves') 
# 输出：'leaf'

# 基于Porter词干提取算法
from nltk.stem.porter import PorterStemmer  
porter_stemmer = PorterStemmer()  
porter_stemmer.stem('maximum')
# 基于Lancaster 词干提取算法
from nltk.stem.lancaster import LancasterStemmer  
lancaster_stemmer = LancasterStemmer()  
lancaster_stemmer.stem()
# 基于Snowball 词干提取算法
from nltk.stem import SnowballStemmer  
snowball_stemmer = SnowballStemmer("english")  
snowball_stemmer.stem('maximum')

from nltk.stem.wordnet import WordNetLemmatizer  # from nltk.stem import WordNetLemmatizer
lem = WordNetLemmatizer()  # 词形还原
from nltk.stem.porter import PorterStemmer   # from nltk.stem import PorterStemmer
stem = PorterStemmer()   # 词干提取
word = "flying"
print("Lemmatized Word:",lem.lemmatize(word,"v"))
print("Stemmed Word:",stem.stem(word))
'''
Lemmatized Word: fly
Stemmed Word: fli
'''

sent = "Albert Einstein was born in Ulm, Germany in 1879."
tokens = nltk.word_tokenize(sent)
tags = nltk.pos_tag(tokens)
'''
[('Albert', 'NNP'), ('Einstein', 'NNP'), ('was', 'VBD'), ('born', 'VBN'), 
('in', 'IN'), ('Ulm', 'NNP'), (',', ','), ('Germany', 'NNP'), ('in', 'IN'), ('1879', 'CD'), ('.', '.')]
'''

import nltk
nltk.download('wordnet')  # Downloading package wordnet to C:\Users\Administrator\AppData\Roaming\nltk_data...Unzipping corpora\wordnet.zip.
from nltk.corpus import wordnet
word = wordnet.synsets('spectacular')
print(word)
# [Synset('spectacular.n.01'), Synset('dramatic.s.02'), Synset('spectacular.s.02'), Synset('outstanding.s.02')]
print(word[0].definition())
print(word[1].definition())
print(word[2].definition())
print(word[3].definition())
'''
a lavishly produced performance
sensational in appearance or thrilling in effect
characteristic of spectacles or drama
having a quality that thrusts itself into attention
'''

from __future__ import division
import nltk
import matplotlib
from nltk.book import *
from nltk.util import bigrams
# 单词搜索
print('单词搜索')
text1.concordance('boy')
text2.concordance('friends')
# 相似词搜索
print('相似词搜索')
text3.similar('time')
#共同上下文搜索
print('共同上下文搜索')
text2.common_contexts(['monstrous','very'])
# 词汇分布表
print('词汇分布表')
text4.dispersion_plot(['citizens', 'American', 'freedom', 'duties'])
# 词汇计数
print('词汇计数')
print(len(text5))
sorted(set(text5))
print(len(set(text5)))
# 重复词密度
print('重复词密度')
print(len(text8) / len(set(text8)))
# 关键词密度
print('关键词密度')
print(text9.count('girl'))
print(text9.count('girl') * 100 / len(text9))
# 频率分布
fdist = FreqDist(text1)
vocabulary = fdist.keys()
for i in vocabulary:
    print(i)
# 高频前20
fdist.plot(20, cumulative = True)
# 低频词
print('低频词：')
print(fdist.hapaxes())
# 词语搭配
print('词语搭配')
words = list(bigrams(['louder', 'words', 'speak']))
print(words)