1.导入引用的python库

# encoding=utf8
import io
import os
import sys
import requests
from collections import OrderedDict
import math
import random
import numpy as np
import paddle
from paddle.nn import Embedding
import paddle.nn.functional as F
import paddle.nn as nn

2.数据处理

# 下载语料用来训练word2vec
def download():
    # 可以从百度云服务器下载一些开源数据集（dataset.bj.bcebos.com）
    corpus_url = "https://dataset.bj.bcebos.com/word2vec/text8.txt"
    # 使用python的requests包下载数据集到本地
    web_request = requests.get(corpus_url)
    corpus = web_request.content
    # 把下载后的文件存储在当前目录的text8.txt文件内
    with open("./text8.txt", "wb") as f:
        f.write(corpus)
    f.close()

download()

2.1 读取数据

# 读取text8数据
def load_text8():
    with open("./text8.txt", "r") as f:
        corpus = f.read().strip("\n")
    f.close()
    return corpus

corpus = load_text8()

# 打印前500个字符，简要看一下这个语料的样子
print(corpus[:500])

anarchism originated as a term of abuse first used against early working class radicals including the diggers of the english revolution and the sans culottes of the french revolution whilst the term is still used in a pejorative way to describe any act that used violent means to destroy the organization of society it has also been taken up as a positive label by self defined anarchists the word anarchism is derived from the greek without archons ruler chief king anarchism as a political philoso

2.2 语料切词

一般来说，在自然语言处理中，需要先对语料进行切词。对于英文来说，可以比较简单地直接使用空格进行切词：

# 对语料进行预处理（分词）
def data_preprocess(corpus):
    # 由于英文单词出现在句首的时候经常要大写，所以我们把所有英文字符都转换为小写，
    # 以便对语料进行归一化处理（Apple vs apple等）
    corpus = corpus.strip().lower()
    corpus = corpus.split(" ")
    return corpus

corpus = data_preprocess(corpus)
print(corpus[:50])

['anarchism', 'originated', 'as', 'a', 'term', 'of', 'abuse', 'first', 'used', 'against', 'early', 'working', 'class', 'radicals', 'including', 'the', 'diggers', 'of', 'the', 'english', 'revolution', 'and', 'the', 'sans', 'culottes', 'of', 'the', 'french', 'revolution', 'whilst', 'the', 'term', 'is', 'still', 'used', 'in', 'a', 'pejorative', 'way', 'to', 'describe', 'any', 'act', 'that', 'used', 'violent', 'means', 'to', 'destroy', 'the']

2.3 排序

重点是学习sorted方法，还是直接文心一言学习，比搜索快，且准确： Python中的sorted()方法是一个内置函数，用于对可迭代对象（如列表、元组、字符串等）进行排序。它的基本语法如下：

sorted(iterable, key=None, reverse=False, key default=None)

其中，iterable 是要排序的可迭代对象，key 参数是可选的，用于指定排序规则，reverse 参数设置为 True 表示降序排序，key 参数默认为 None，表示使用默认的排序规则。key 参数的默认值为 None，表示使用默认的排序规则。

下面是一个使用sorted()方法对列表进行排序的示例：

my_list = [3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5]
# 使用 sorted() 方法进行排序
sorted_list = sorted(my_list)
# 输出排序后的结果
print(sorted_list)

输出结果为：

[1, 1, 2, 3, 3, 4, 5, 5, 5, 6, 9]

在上面的示例中，我们首先定义了一个包含多个元素的列表 my_list，然后使用 sorted() 方法对列表进行排序，并将结果存储在 sorted_list 变量中。最后，我们输出排序后的结果。

my_list = [3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5]
# 使用 sorted() 方法进行排序
sorted_list = sorted(my_list)
# 输出排序后的结果
print(sorted_list)

[1, 1, 2, 3, 3, 4, 5, 5, 5, 6, 9]

对语料进行统计，构造ID
频次越高，ID越小，以方便对词典进行管理

my_dict = {'apple': 1, 'banana': 2, 'orange': 3}
print(my_dict)
# 按照键值对的值进行排序，并翻转结果
sorted_items = sorted(my_dict.items(), key=lambda x: x[1], reverse=True)
# 输出结果
print(sorted_items)  # [('apple', 1), ('banana', 2), ('orange', 3)]

{'apple': 1, 'banana': 2, 'orange': 3}
[('orange', 3), ('banana', 2), ('apple', 1)]

可以使用 key 参数来指定排序规则。在上面的例子中，key 参数使用 lambda x: x[1] 来指定排序规则，其中 x[1] 表示 x 的第二个元素。
reverse 参数设置为 True 表示降序排序，默认情况下是升序排序。如果不设置 reverse 参数，默认情况下会根据键的字母顺序进行排序。
最后得到的结果是按照第二个元素（即值）进行排序，得到的是一个降序排列的字典。

def build_dict(corpus):
    # 定义词典，存储单词出现的频率
    word_freq_dict = dict()
    for word in corpus:
        if word not in word_freq_dict:
            word_freq_dict[word] = 0
        word_freq_dict[word] += 1
    # 对单词出现的频率进行排序
    word_freq_dict = sorted(word_freq_dict.items(), key=lambda x: x[1], reverse=True)
    # 构造3个不同的词典，分别存储，
    # 每个词到id的映射关系：word2id_dict
    # 每个id出现的频率：word2id_freq
    # 每个id到词的映射关系：id2word_dict
    word2id_dict = dict()
    word2id_freq = dict()
    id2word_dict = dict()
    # 按照频率，从高到低，开始遍历每个单词，并为这个单词构造一个独一无二的id
    # key value， 直接对应过去
    for word, freq in word_freq_dict:
        # 牛逼，id直接按长度增加，不过求len和 i++不知道哪个开销少
        curr_id = len(word2id_dict)
        word2id_dict[word] = curr_id
        word2id_freq[word2id_dict[word]] = freq
        id2word_dict[curr_id] = word
    return word2id_freq, word2id_dict, id2word_dict

word2id_freq, word2id_dict, id2word_dict = build_dict(corpus)
vocab_size = len(word2id_freq)
print("there are totoally %d different words in the corpus" % vocab_size)
for _, (word, word_id) in zip(range(50), word2id_dict.items()):
    print("word %s, its id %d, its word freq %d" % (word, word_id, word2id_freq[word_id]))

there are totoally 253854 different words in the corpus
word the, its id 0, its word freq 1061396
word of, its id 1, its word freq 593677
word and, its id 2, its word freq 416629
word one, its id 3, its word freq 411764
word in, its id 4, its word freq 372201
word a, its id 5, its word freq 325873
word to, its id 6, its word freq 316376
word zero, its id 7, its word freq 264975
word nine, its id 8, its word freq 250430
word two, its id 9, its word freq 192644
word is, its id 10, its word freq 183153
word as, its id 11, its word freq 131815
word eight, its id 12, its word freq 125285
word for, its id 13, its word freq 118445
word s, its id 14, its word freq 116710
word five, its id 15, its word freq 115789
word three, its id 16, its word freq 114775
word was, its id 17, its word freq 112807
word by, its id 18, its word freq 111831
word that, its id 19, its word freq 109510
word four, its id 20, its word freq 108182
word six, its id 21, its word freq 102145
word seven, its id 22, its word freq 99683
word with, its id 23, its word freq 95603
word on, its id 24, its word freq 91250
word are, its id 25, its word freq 76527
word it, its id 26, its word freq 73334
word from, its id 27, its word freq 72871
word or, its id 28, its word freq 68945
word his, its id 29, its word freq 62603
word an, its id 30, its word freq 61925
word be, its id 31, its word freq 61281
word this, its id 32, its word freq 58832
word which, its id 33, its word freq 54788
word at, its id 34, its word freq 54576
word he, its id 35, its word freq 53573
word also, its id 36, its word freq 44358
word not, its id 37, its word freq 44033
word have, its id 38, its word freq 39712
word were, its id 39, its word freq 39086
word has, its id 40, its word freq 37866
word but, its id 41, its word freq 35358
word other, its id 42, its word freq 32433
word their, its id 43, its word freq 31523
word its, its id 44, its word freq 29567
word first, its id 45, its word freq 28810
word they, its id 46, its word freq 28553
word some, its id 47, its word freq 28161
word had, its id 48, its word freq 28100
word all, its id 49, its word freq 26229

2.4 转换数据形式

# 把语料转换为id序列
def convert_corpus_to_id(corpus, word2id_dict):
    # 使用一个循环，将语料中的每个词替换成对应的id，以便于神经网络进行处理
    corpus = [word2id_dict[word] for word in corpus]
    return corpus
corpus = convert_corpus_to_id(corpus, word2id_dict)
print("%d tokens in the corpus" % len(corpus))
print(corpus[:50])

17005207 tokens in the corpus
[5233, 3080, 11, 5, 194, 1, 3133, 45, 58, 155, 127, 741, 476, 10571, 133, 0, 27349, 1, 0, 102, 854, 2, 0, 15067, 58112, 1, 0, 150, 854, 3580, 0, 194, 10, 190, 58, 4, 5, 10712, 214, 6, 1324, 104, 454, 19, 58, 2731, 362, 6, 3672, 0]

2.5 构建训练数据

使用二次采样法处理原始文本

二次采样法的主要思想是降低高频词在语料中出现的频次。
方法是随机将高频的词抛弃，频率越高，被抛弃的概率就越大；
频率越低，被抛弃的概率就越小。
标点符号或冠词这样的高频词就会被抛弃，从而优化整个词表的词向量训练效果

这个鬼公式我得查查。。。。。。

# 使用二次采样算法（subsampling）处理语料，强化训练效果
def subsampling(corpus, word2id_freq):
    # 这个discard函数决定了一个词会不会被替换，这个函数是具有随机性的，每次调用结果不同，这是个什么鬼公式转的？？？
    # 如果一个词的频率很大，那么它被遗弃的概率就很大
    def discard(word_id):
        return random.uniform(0, 1) < 1 - math.sqrt(
            1e-4 / word2id_freq[word_id] * len(corpus))
    # 随即丢，丢完了再重新组
    corpus = [word for word in corpus if not discard(word)]
    return corpus

corpus = subsampling(corpus, word2id_freq)
print("%d tokens in the corpus" % len(corpus))
print(corpus[:50])

8742730 tokens in the corpus
[5233, 3080, 3133, 155, 741, 476, 10571, 133, 27349, 854, 15067, 58112, 854, 3580, 10, 10712, 214, 1324, 454, 19, 2731, 362, 3672, 0, 708, 40, 539, 97, 1423, 2757, 567, 686, 7088, 247, 5233, 1052, 320, 248, 44611, 2877, 186, 5233, 602, 1134, 2621, 8983, 279, 4147, 141, 6437]

在完成语料数据预处理之后，需要构造训练数据。根据上面的描述，我们需要使用一个滑动窗口对语料从左到右扫描，在每个窗口内，中心词需要预测它的上下文，并形成训练数据。

在实际操作中，由于词表往往很大（50000，100000等），对大词表的一些矩阵运算（如softmax）需要消耗巨大的资源，因此可以通过负采样的方式模拟softmax的结果。

给定一个中心词和一个需要预测的上下文词，把这个上下文词作为正样本。
通过词表随机采样的方式，选择若干个负样本。
把一个大规模分类问题转化为一个2分类问题，通过这种方式优化计算速度。

# 构造数据，准备模型训练
# max_window_size代表了最大的window_size的大小，程序会根据max_window_size从左到右扫描整个语料
# negative_sample_num代表了对于每个正样本，我们需要随机采样多少负样本用于训练，
# 一般来说，negative_sample_num的值越大，训练效果越稳定，但是训练速度越慢。
def build_data(corpus, word2id_dict, word2id_freq, max_window_size = 3, negative_sample_num = 4):
    # 使用一个list存储处理好的数据
    dataset = []
    # 从左到右，开始枚举每个中心点的位置
    for center_word_idx in range(len(corpus)):
        # 以max_window_size为上限，随机采样一个window_size，这样会使得训练更加稳定
        window_size = random.randint(1, max_window_size)
        # 当前的中心词就是center_word_idx所指向的词
        center_word = corpus[center_word_idx]
        # 以当前中心词为中心，左右两侧在window_size内的词都可以看成是正样本
        positive_word_range = (max(0, center_word_idx - window_size), min(len(corpus) - 1, center_word_idx + window_size))
        positive_word_candidates = [corpus[idx] for idx in range(positive_word_range[0], positive_word_range[1]+1) if idx != center_word_idx]
        # 对于每个正样本来说，随机采样negative_sample_num个负样本，用于训练
        for positive_word in positive_word_candidates:
            # 首先把（中心词，正样本，label=1）的三元组数据放入dataset中，
            # 这里label=1表示这个样本是个正样本
            dataset.append((center_word, positive_word, 1))
            # 开始负采样
            i = 0
            while i < negative_sample_num:
                negative_word_candidate = random.randint(0, vocab_size-1)
                if negative_word_candidate not in positive_word_candidates:
                    # 把（中心词，正样本，label=0）的三元组数据放入dataset中，
                    # 这里label=0表示这个样本是个负样本
                    dataset.append((center_word, negative_word_candidate, 0))
                    i += 1
    return dataset
corpus_light = corpus[:int(len(corpus)*0.2)]
dataset = build_data(corpus_light, word2id_dict, word2id_freq)
for _, (center_word, target_word, label) in zip(range(50), dataset):
    print("center_word %s, target %s, label %d" % (id2word_dict[center_word],
                                                   id2word_dict[target_word], label))

center_word anarchism, target originated, label 1
center_word anarchism, target hisakazu, label 0
center_word anarchism, target liverwurst, label 0
center_word anarchism, target bilisi, label 0
center_word anarchism, target tolerability, label 0
center_word anarchism, target abuse, label 1
center_word anarchism, target ztat, label 0
center_word anarchism, target saskatchewann, label 0
center_word anarchism, target kinara, label 0
center_word anarchism, target ndbm, label 0
center_word originated, target anarchism, label 1
center_word originated, target qinling, label 0
center_word originated, target gerbert, label 0
center_word originated, target petroglyphs, label 0
center_word originated, target hanegbi, label 0
center_word originated, target abuse, label 1
center_word originated, target ipac, label 0
center_word originated, target bartley, label 0
center_word originated, target ushpizin, label 0
center_word originated, target ghattas, label 0
center_word originated, target against, label 1
center_word originated, target predidate, label 0
center_word originated, target kavvana, label 0
center_word originated, target zeuxippus, label 0
center_word originated, target ribbon, label 0
center_word originated, target working, label 1
center_word originated, target tbk, label 0
center_word originated, target huella, label 0
center_word originated, target amarantos, label 0
center_word originated, target mustique, label 0
center_word abuse, target anarchism, label 1
center_word abuse, target masking, label 0
center_word abuse, target moviemistakes, label 0
center_word abuse, target reichstein, label 0
center_word abuse, target misunderstood, label 0
center_word abuse, target originated, label 1
center_word abuse, target cicada, label 0
center_word abuse, target pamyat, label 0
center_word abuse, target cohanite, label 0
center_word abuse, target hovertank, label 0
center_word abuse, target against, label 1
center_word abuse, target gonzal, label 0
center_word abuse, target sclc, label 0
center_word abuse, target preproduction, label 0
center_word abuse, target arsonval, label 0
center_word abuse, target working, label 1
center_word abuse, target cilau, label 0
center_word abuse, target anchoretism, label 0
center_word abuse, target plentitude, label 0
center_word abuse, target stingray, label 0

2.6 组装mini-batch数据

训练数据准备好后，把训练数据都组装成mini-batch，并准备输入到网络中进行训练，代码如下：

# 构造mini-batch，准备对模型进行训练
# 我们将不同类型的数据放到不同的tensor里，便于神经网络进行处理
# 并通过numpy的array函数，构造出不同的tensor来，并把这些tensor送入神经网络中进行训练
def build_batch(dataset, batch_size, epoch_num):
    # center_word_batch缓存batch_size个中心词
    center_word_batch = []
    # target_word_batch缓存batch_size个目标词（可以是正样本或者负样本）
    target_word_batch = []
    # label_batch缓存了batch_size个0或1的标签，用于模型训练
    label_batch = []
    for epoch in range(epoch_num):
        # 每次开启一个新epoch之前，都对数据进行一次随机打乱，提高训练效果
        random.shuffle(dataset)
        for center_word, target_word, label in dataset:
            # 遍历dataset中的每个样本，并将这些数据送到不同的tensor里
            center_word_batch.append([center_word])
            target_word_batch.append([target_word])
            label_batch.append(label)
            # 当样本积攒到一个batch_size后，我们把数据都返回回来
            # 在这里我们使用numpy的array函数把list封装成tensor
            # 并使用python的迭代器机制，将数据yield出来
            # 使用迭代器的好处是可以节省内存
            if len(center_word_batch) == batch_size:
                yield np.array(center_word_batch).astype("int64"), \
                    np.array(target_word_batch).astype("int64"), \
                    np.array(label_batch).astype("float32")
                center_word_batch = []
                target_word_batch = []
                label_batch = []
    if len(center_word_batch) > 0:
        yield np.array(center_word_batch).astype("int64"), \
            np.array(target_word_batch).astype("int64"), \
            np.array(label_batch).astype("float32")
for _, batch in zip(range(10), build_batch(dataset, 128, 3)):
    print(batch)
    break

(array([[    90],
       [   588],
       [   982],
       [   334],
       [  3770],
       [   344],
       [  1201],
       [ 34373],
       [  3522],
       [  1996],
       [   384],
       [  1089],
       [  1098],
       [   965],
       [  3847],
       [  8111],
       [  4971],
       [  8975],
       [  1238],
       [  1430],
       [ 27871],
       [ 20604],
       [ 14278],
       [   281],
       [  2996],
       [  2295],
       [ 42539],
       [  1587],
       [ 22266],
       [ 33013],
       [  1150],
       [  1067],
       [   767],
       [  4663],
       [   240],
       [   592],
       [  6285],
       [   854],
       [   172],
       [  3193],
       [   392],
       [  3178],
       [  5803],
       [   413],
       [     0],
       [ 13138],
       [ 20100],
       [   276],
       [  1577],
       [    28],
       [ 17444],
       [   905],
       [   281],
       [  5896],
       [  9164],
       [   421],
       [ 40051],
       [  2255],
       [ 16329],
       [  3420],
       [  5711],
       [  2163],
       [    98],
       [  3715],
       [  1761],
       [  3581],
       [  2420],
       [   504],
       [ 58477],
       [   160],
       [  3794],
       [   978],
       [    59],
       [  1965],
       [ 19106],
       [  6462],
       [   550],
       [   138],
       [ 14207],
       [  4669],
       [108313],
       [  1663],
       [   785],
       [  1775],
       [  2299],
       [ 11068],
       [   681],
       [  4947],
       [  9482],
       [ 54617],
       [  4755],
       [  7502],
       [  1113],
       [   688],
       [    98],
       [ 22978],
       [   885],
       [   711],
       [ 45986],
       [   368],
       [ 10402],
       [  3574],
       [  6930],
       [   695],
       [  3264],
       [   841],
       [ 18505],
       [   267],
       [   466],
       [    64],
       [ 28452],
       [   728],
       [  2490],
       [  2425],
       [  1507],
       [   760],
       [  4600],
       [  1879],
       [  1271],
       [ 34440],
       [  2270],
       [ 11137],
       [  1852],
       [  9232],
       [   257],
       [  4270],
       [  1512],
       [ 31547]], dtype=int64), array([[  2827],
       [ 13796],
       [103439],
       [ 16480],
       [ 29571],
       [144095],
       [ 75916],
       [  4541],
       [   930],
       [  2708],
       [103147],
       [ 20194],
       [ 10286],
       [  4975],
       [162742],
       [109192],
       [   968],
       [  8801],
       [   650],
       [  4299],
       [ 56979],
       [ 66745],
       [ 90132],
       [ 89302],
       [183085],
       [211328],
       [215867],
       [226421],
       [230690],
       [  2027],
       [181287],
       [ 68227],
       [  1886],
       [177493],
       [ 14429],
       [ 19151],
       [158341],
       [238722],
       [ 56248],
       [ 19939],
       [   721],
       [170988],
       [ 60351],
       [201496],
       [135780],
       [228575],
       [ 36277],
       [ 89419],
       [197125],
       [222194],
       [ 33692],
       [ 50198],
       [160937],
       [224291],
       [187053],
       [ 89955],
       [ 14144],
       [188683],
       [172533],
       [201584],
       [193150],
       [234380],
       [110731],
       [  2842],
       [ 97179],
       [143441],
       [ 72314],
       [   191],
       [227196],
       [214612],
       [   725],
       [144038],
       [124484],
       [  3315],
       [123621],
       [173700],
       [ 18402],
       [ 98103],
       [   238],
       [145282],
       [   592],
       [188998],
       [ 81425],
       [ 40827],
       [ 23973],
       [194500],
       [237439],
       [144121],
       [211876],
       [234473],
       [ 37290],
       [ 63905],
       [  2655],
       [232259],
       [   929],
       [179475],
       [197707],
       [ 17272],
       [188889],
       [  8810],
       [164693],
       [ 54438],
       [  5120],
       [174645],
       [ 27173],
       [ 87828],
       [  9932],
       [241565],
       [   173],
       [  3957],
       [ 63051],
       [   141],
       [ 41401],
       [183427],
       [ 46558],
       [104789],
       [216617],
       [   448],
       [109615],
       [ 16352],
       [  7107],
       [ 82236],
       [177176],
       [  2441],
       [  1023],
       [193967],
       [ 56406],
       [154326]], dtype=int64), array([1., 1., 0., 1., 1., 0., 0., 0., 1., 1., 0., 0., 1., 1., 0., 0., 1.,       1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,       0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 1.,       0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0.,       0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 0., 0., 0., 0.,       1., 0., 0., 0., 1., 0., 1., 1., 0., 1., 0., 0., 0., 0., 0., 1., 0.,       0., 0., 0., 0., 0., 1., 0., 0., 1.], dtype=float32))