自然语言处理实战第二版（MEAP）（三）（4）

>>> import torchtext
>>> dsets = torchtext.datasets.WikiText2()
>>> num_texts = 10000
>>> filepath = DATA_DIR / f'WikiText2-{num_texts}.txt'
>>> with open(filepath, 'wt') as fout:
...     fout.writelines(list(dsets[0])[:num_texts])

>>> !tail -n 3 ~/nessvec-data/WikiText2-10000.txt
When Marge leaves Dr. Zweig 's office , she says ,
" Whenever the wind whistles through the leaves ,
I 'll think , Lowenstein , Lowenstein … " .
This is a reference to The Prince of Tides ; the <unk> is Dr. Lowenstein .
= = Reception = =

>>> import datasets
>>> dset = datasets.load_dataset('text', data_files=str(filepath))
>>> dset
DatasetDict({
    train: Dataset({
        features: ['text'],
        num_rows: 10000
    })
})

def tokenize_row(row):
    row['all_tokens'] = row['text'].lower().split()
    return row

>>> dset = dset.map(tokenize_row)
>>> dset
DatasetDict({
    train: Dataset({
        features: ['text', 'tokens'],
        num_rows: 10000
    })
})

>>> vocab = list(set(
...     [tok for row in dset['train']['tokens'] for tok in row]))
>>> vocab[:4]
['cast', 'kaifeng', 'recovered', 'doctorate']
>>> id2tok = dict(enumerate(vocab))
>>> list(id2tok.items())[:4]
[(0, 'cast'), (1, 'kaifeng'), (2, 'recovered'), (3, 'doctorate')]
>>> tok2id = {tok: i for (i, tok) in id2tok.items()}
>>> list(tok2id.items())[:4]
[('cast', 0), ('kaifeng', 1), ('recovered', 2), ('doctorate', 3)]

WINDOW_WIDTH = 10
>>> def windowizer(row, wsize=WINDOW_WIDTH):
    """ Compute sentence (str) to sliding-window of skip-gram pairs. """
...    doc = row['tokens']
...    out = []
...    for i, wd in enumerate(doc):
...        target = tok2id[wd]
...        window = [
...            i + j for j in range(-wsize, wsize + 1, 1)
...            if (i + j >= 0) & (i + j < len(doc)) & (j != 0)
...        ]
...        out += [(target, tok2id[doc[w]]) for w in window]
...    row['moving_window'] = out
...    return row

>>> dset = dset.map(windowizer)
>>> dset
DatasetDict({
    train: Dataset({
        features: ['text', 'tokens', 'window'],
        num_rows: 10000
    })
})

>>> def skip_grams(tokens, window_width=WINDOW_WIDTH):
...    pairs = []
...    for i, wd in enumerate(tokens):
...        target = tok2id[wd]
...        window = [
...            i + j for j in
...            range(-window_width, window_width + 1, 1)
...            if (i + j >= 0)
...            & (i + j < len(tokens))
...            & (j != 0)
...        ]
...        pairs.extend([(target, tok2id[tokens[w]]) for w in window])
    # huggingface datasets are dictionaries for every text element
...    return pairs

>>> from torch.utils.data import Dataset
>>> class Word2VecDataset(Dataset):
...    def __init__(self, dataset, vocab_size, wsize=WINDOW_WIDTH):
...        self.dataset = dataset
...        self.vocab_size = vocab_size
...        self.data = [i for s in dataset['moving_window'] for i in s]
...
...    def __len__(self):
...        return len(self.data)
...
...    def __getitem__(self, idx):
...        return self.data[idx]

from torch.utils.data import DataLoader
dataloader = {}
for k in dset.keys():
    dataloader = {
        k: DataLoader(
            Word2VecDataset(
                dset[k],
                vocab_size=len(vocab)),
            batch_size=BATCH_SIZE,
            shuffle=True,
            num_workers=CPU_CORES - 1)
    }

def one_hot_encode(input_id, size):
    vec = torch.zeros(size).float()
    vec[input_id] = 1.0
    return vec

from torch import nn
EMBED_DIM = 100  # #1
class Word2Vec(nn.Module):
    def __init__(self, vocab_size=len(vocab), embedding_size=EMBED_DIM):
        super().__init__()
        self.embed = nn.Embedding(vocab_size, embedding_size)  # #2
        self.expand = nn.Linear(embedding_size, vocab_size, bias=False)
    def forward(self, input):
        hidden = self.embed(input)  # #3
        logits = self.expand(hidden)  # #4
        return logits

>>> model = Word2Vec()
>>> model
Word2Vec(
  (embed): Embedding(20641, 100)
  (expand): Linear(in_features=100, out_features=20641, bias=False)
)

>>> import torch
>>> if torch.cuda.is_available():
...     device = torch.device('cuda')
>>> else:
...     device = torch.device('cpu')
>>> device
device(type='cpu')

>>> model.to(device)
Word2Vec(
  (embed): Embedding(20641, 100)
  (expand): Linear(in_features=100, out_features=20641, bias=False)
)

>>> from tqdm import tqdm  # noqa
>>> EPOCHS = 10
>>> LEARNING_RATE = 5e-4
EPOCHS = 10
loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE)

running_loss = []
pbar = tqdm(range(EPOCHS * len(dataloader['train'])))
for epoch in range(EPOCHS):
    epoch_loss = 0
    for sample_num, (center, context) in enumerate(dataloader['train']):
        if sample_num % len(dataloader['train']) == 2:
            print(center, context)
            # center: tensor([ 229,    0, 2379,  ...,  402,  553,  521])
            # context: tensor([ 112, 1734,  802,  ...,   28,  852,  363])
        center, context = center.to(device), context.to(device)
        optimizer.zero_grad()
        logits = model(input=context)
        loss = loss_fn(logits, center)
        if not sample_num % 10000:
            # print(center, context)
            pbar.set_description(f'loss[{sample_num}] = {loss.item()}')
        epoch_loss += loss.item()
        loss.backward()
        optimizer.step()
        pbar.update(1)
    epoch_loss /= len(dataloader['train'])
    running_loss.append(epoch_loss)
save_model(model, loss)

>>> from nlpia.data.loaders import get_data
>>> word_vectors = get_data('word2vec')

>>> from gensim.models.keyedvectors import KeyedVectors
>>> word_vectors = KeyedVectors.load_word2vec_format(\
...     '/path/to/GoogleNews-vectors-negative300.bin.gz', binary=True)

>>> from gensim.models.keyedvectors import KeyedVectors
>>> from nlpia.loaders import get_data
>>> word_vectors = get_data('w2v', limit=200000)  # #1

>>> word_vectors.most_similar(positive=['cooking', 'potatoes'], topn=5)
[('cook', 0.6973530650138855),
 ('oven_roasting', 0.6754530668258667),
 ('Slow_cooker', 0.6742032170295715),
 ('sweet_potatoes', 0.6600279808044434),
 ('stir_fry_vegetables', 0.6548759341239929)]
>>> word_vectors.most_similar(positive=['germany', 'france'], topn=1)
[('europe', 0.7222039699554443)]

>>> word_vectors.doesnt_match("potatoes milk cake computer".split())
'computer'

>>> word_vectors.most_similar(positive=['king', 'woman'],
...     negative=['man'], topn=2)
[('queen', 0.7118192315101624), ('monarch', 0.6189674139022827)]

>>> word_vectors.similarity('princess', 'queen')
0.70705315983704509

>>> word_vectors['phone']
array([-0.01446533, -0.12792969, -0.11572266, -0.22167969, -0.07373047,
       -0.05981445, -0.10009766, -0.06884766,  0.14941406,  0.10107422,
       -0.03076172, -0.03271484, -0.03125   , -0.10791016,  0.12158203,
        0.16015625,  0.19335938,  0.0065918 , -0.15429688,  0.03710938,
        ...

>>> token_list
[
  ['to', 'provide', 'early', 'intervention/early', 'childhood', 'special',
   'education', 'services', 'to', 'eligible', 'children', 'and', 'their',
   'families'],
  ['essential', 'job', 'functions'],
  ['participate', 'as', 'a', 'transdisciplinary', 'team', 'member', 'to',
   'complete', 'educational', 'assessments', 'for']
  ...
]

>>> from gensim.models.word2vec import Word2Vec

>>> num_features = 300  # #1
>>> min_word_count = 3  # #2
>>> num_workers = 2  # #3
>>> window_size = 6  # #4
>>> subsampling = 1e-3  # #5

>>> model = Word2Vec(
...     token_list,
...     workers=num_workers,
...     size=num_features,
...     min_count=min_word_count,
...     window=window_size,
...     sample=subsampling)

>>> model.init_sims(replace=True)

>>> model_name = "my_domain_specific_word2vec_model"
>>> model.save(model_name)

>>> from gensim.models.word2vec import Word2Vec
>>> model_name = "my_domain_specific_word2vec_model"
>>> model = Word2Vec.load(model_name)
>>> model.most_similar('radiology')