自然语言处理实战第二版（MEAP）（二）（2）

>>> import copy
>>> question = "What is algorithmic bias?"
>>> ngram_docs = copy.copy(docs)
>>> ngram_docs.append(question)

>>> question_vec = vectorizer.transform([new_sentence])
>>> question_vec
<1x240 sparse matrix of type '<class 'numpy.int64'>'
    with 3 stored elements in Compressed Sparse Row format>

>>> question_vec.to_array()
array([[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, ... ]])

>>> vocab = list(zip(*sorted((i, tok) for tok, i in
...     vectorizer.vocabulary_.items())))[1]
>>> pd.Series(question_vec.to_array()[0], index=vocab).head(8)
2018           0
ability        0
accurately     0
across         0
addressed      0
advanced       0
algorithm      0
algorithmic    1

>>> cosine_similarity(count_vectors, question_vector)
array([[0.23570226],
       [0.12451456],
       [0.24743583],
       [0.4330127 ],
       [0.12909944],
       ...

>>> docs[3]
The study of algorithmic bias is most concerned with algorithms
that reflect "systematic and unfair" discrimination.

>>> ngram_vectorizer = CountVectorizer(ngram_range=(1, 2))
>>> ngram_vectors = ngram_vectorizer.fit_transform(corpus)
>>> ngram_vectors
<16x616 sparse matrix of type '<class 'numpy.int64'>'
    with 772 stored elements in Compressed Sparse Row format>

>>> vocab = list(zip(*sorted((i, tok) for tok, i in
...     ngram_vectorizer.vocabulary_.items())))[1]
>>> pd.DataFrame(ngram_vectors.toarray(),
...     columns=vocab)['algorithmic bias']

>>> from this import s
>>> print(s)
Gur Mra bs Clguba, ol Gvz Crgref
Ornhgvshy vf orggre guna htyl.
Rkcyvpvg vf orggre guna vzcyvpvg.
Fvzcyr vf orggre guna pbzcyrk.
...
Nygubhtu arire vf bsgra orggre guna *evtug* abj.
Vs gur vzcyrzragngvba vf uneq gb rkcynva, vg'f n onq vqrn.
Vs gur vzcyrzragngvba vf rnfl gb rkcynva, vg znl or n tbbq vqrn.
Anzrfcnprf ner bar ubaxvat terng vqrn -- yrg'f qb zber bs gubfr!

>>> char_vectorizer = CountVectorizer(
...     ngram_range=(1,1), analyzer='char')  # #1
>>> s_char_frequencies = char_vectorizer.fit_transform(s)
>>> generate_histogram(
...     s_char_frequencies, s_char_vectorizer)  # #2

>>> DATA_DIR = ('https://gitlab.com/tangibleai/nlpia/'
...             '-/raw/master/src/nlpia/data')
>>> url = DATA_DIR + '/machine_learning_full_article.txt'
>>> ml_text = requests.get(url).content.decode()
>>> ml_char_frequencies = char_vectorizer.fit_transform(ml_text)
>>> generate_histogram(s_char_frequencies, s_char_vectorizer)

>>> peak_distance = ord('R') - ord('E')
>>> peak_distance
13
>>> chr(ord('v') - peak_distance)  # #1
'I'
>>> chr(ord('n') - peak_distance)  # #2
'A'

>>> chr(ord('W') - peak_distance)
'J'

>>> import codecs
>>> print(codecs.decode(s, 'rot-13'))
The Zen of Python, by Tim Peters
Beautiful is better than ugly.
Explicit is better than implicit.
Simple is better than complex.
Complex is better than complicated.
Flat is better than nested.
Sparse is better than dense.
Readability counts.
Special cases aren't special enough to break the rules.
Although practicality beats purity.
Errors should never pass silently.
Unless explicitly silenced.
In the face of ambiguity, refuse the temptation to guess.
There should be one-- and preferably only one --obvious way to do it.
Although that way may not be obvious at first unless you're Dutch.
Now is better than never.
Although never is often better than *right* now.
If the implementation is hard to explain, it's a bad idea.
If the implementation is easy to explain, it may be a good idea.
Namespaces are one honking great idea -- let's do more of those!

>>> nltk.download('brown')  # #1
>>> from nltk.corpus import brown
>>> brown.words()[:10]  # #2
['The',
 'Fulton',
 'County',
 'Grand',
 'Jury',
 'said',
 'Friday',
 'an',
 'investigation',
 'of']
>>> brown.tagged_words()[:5]  # #3
[('The', 'AT'),
 ('Fulton', 'NP-TL'),
 ('County', 'NN-TL'),
 ('Grand', 'JJ-TL'),
 ('Jury', 'NN-TL')]
>>> len(brown.words())
1161192

>>> from collections import Counter
>>> puncs = set((',', '.', '--', '-', '!', '?',
...     ':', ';', '``', "''", '(', ')', '[', ']'))
>>> word_list = (x.lower() for x in brown.words() if x not in puncs)
>>> token_counts = Counter(word_list)
>>> token_counts.most_common(10)
[('the', 69971),
 ('of', 36412),
 ('and', 28853),
 ('to', 26158),
 ('a', 23195),
 ('in', 21337),
 ('that', 10594),
 ('is', 10109),
 ('was', 9815),
 ('he', 9548)]

>>> DATA_DIR = ('https://gitlab.com/tangibleai/nlpia/'
...             '-/raw/master/src/nlpia/data')
>>> url = DATA_DIR + '/bias_discrimination.txt'
>>> bias_discrimination = requests.get(url).content.decode()
>>> intro_tokens = [token.text for token in nlp(bias_intro.lower())]
>>> disc_tokens = [token.text for token in nlp(bias_discrimination.lower())]
>>> intro_total = len(intro_tokens)
>>> intro_total
479
>>> disc_total = len (disc_tokens)
>>> disc_total
451

>>> intro_tf = {}
>>> disc_tf = {}
>>> intro_counts = Counter(intro_tokens)
>>> intro_tf['bias'] = intro_counts['bias'] / intro_total
>>> disc_counts = Counter(disc_tokens)
>>> disc_tf['bias'] = disc_counts['bias'] / disc_total
>>> 'Term Frequency of "bias" in intro is:{:.4f}'.format(intro_tf['bias'])
Term Frequency of "bias" in intro is:0.0167
>>> 'Term Frequency of "bias" in discrimination chapter is: {:.4f}'\
...     .format(disc_tf['bias'])
'Term Frequency of "bias" in discrimination chapter is: 0.0022'

>>> intro_tf['and'] = intro_counts['and'] / intro_total
>>> disc_tf['and'] = disc_counts['and'] / disc_total
>>> print('Term Frequency of "and" in intro is: {:.4f}'\
...     .format(intro_tf['and']))
Term Frequency of "and" in intro is: 0.0292
>>> print('Term Frequency of "and" in discrimination chapter is: {:.4f}'\
...     .format(disc_tf['and']))
Term Frequency of "and" in discrimination chapter is: 0.0303

2 total documents / 2 documents contain "and"  = 2/2 = 1
2 total documents / 2 documents contain "bias" = 2/2 = 1

2 total documents / 1 document contains "black" = 2/1 = 2

>>> num_docs_containing_and = 0
>>> for doc in [intro_tokens, disc_tokens]:
...     if 'and' in doc:
...         num_docs_containing_and += 1  # #1

>>> intro_tf['black'] = intro_counts['black'] / intro_total
>>> disc_tf['black'] = disc_counts['black'] / disc_total

>>> num_docs = 2
>>> intro_idf = {}
>>> disc_idf = {}
>>> intro_idf['and'] = num_docs / num_docs_containing_and
>>> disc_idf['and'] = num_docs / num_docs_containing_and
>>> intro_idf['bias'] = num_docs / num_docs_containing_bias
>>> disc_idf['bias'] = num_docs / num_docs_containing_bias
>>> intro_idf['black'] = num_docs / num_docs_containing_black
>>> disc_idf['black'] = num_docs / num_docs_containing_black

>>> intro_tfidf = {}
>>> intro_tfidf['and'] = intro_tf['and'] * intro_idf['and']
>>> intro_tfidf['bias'] = intro_tf['bias'] * intro_idf['bias']
>>> intro_tfidf['black'] = intro_tf['black'] * intro_idf['black']

>>> disc_tfidf = {}
>>> disc_tfidf['and'] = disc_tf['and'] * disc_idf['and']
>>> disc_tfidf['bias'] = disc_tf['bias'] * disc_idf['bias']
>>> disc_tfidf['black'] = disc_tf['black'] * disc_idf['black']

>>> log_tf = log(term_occurences_in_doc) -\
...     log(num_terms_in_doc)  # #1
>>> log_log_idf = log(log(total_num_docs) -\
...     log(num_docs_containing_term))  # #2
>>> log_tf_idf = log_tf + log_log_idf  # #3