# 先克隆仓库代码
git clone https://github.com/pytorch/fairseq
# 进入文件夹里
cd fairseq
# 执行命令，这个命令我不太清楚什么意思，不过必须要执行,否则之后使用的时候会报错。
# 猜测：安装Fairseq项目到python
pip install --editable ./ -i https://pypi.mirrors.ustc.edu.cn/simple/

# 
git clone https://github.com/NVIDIA/apex
cd apex
pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" \
  --global-option="--deprecated_fused_adam" --global-option="--xentropy" \
  --global-option="--fast_multihead_attn" ./
# 查看显卡信息
nvidia-smi

# --destdir： 预处理后的二进制文件会默认保存在data-bin目录下，可以通过destdir参数将生成的数据存放在其他位置。
# --thresholdsrc/--thresholdtgt: 分别对应源端（source）和目标端（target）的词表的最低词频，词频低于这个阈值的单词将不会出现在词表中，而是统一使用一个unknown标签来代替。
# --nwordssrc/--nwordstgt，源端和目标端词表的大小，在对单词根据词频排序后，取前n个词来构建词表，剩余的单词使用一个统一的unknown标签代替。
# --source-lang: 源
# --target-lang：目标
# --trainpref：训练文件前缀（也用于建立词典），即路径和文件名的前缀。
# --validpref：验证文件前缀。   
# --testpref: 测试文件前缀。 
# --joined-dictionary: 源端和目标端使用同一个词表，对于相似语言（如英语和西班牙语）来说，有很多的单词是相同的，使用同一个词表可以降低词表和参数的总规模。
# --tgtdict: 重用给定的目标词典
# --srcdict：重用给定的源词典，参数为文件名，即使用已有的词典，而不去根据文本数据中单词的词频去构建词表
# --workers: 并行进程数。
eg: TEXT=iwslt14.tokenized.de-en
  fairseq-preprocess --source-lang de --target-lang en \
    --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \
    --destdir data-bin/iwslt14.tokenized.de-en \
    --joined-dictionary --workers 20

# --arch：所使用的模型结构
# --optimizer: 可以选择的优化器：adadelta, adafactor, adagrad, adam, adamax, composite, cpu_adam, lamb, nag, sgd
# --clip-norm: 梯度减少阈值，默认为0
# --lr： 前N个批次的学习率，默认为0.25
# --lr-scheduler： 学习率缩减的方式，可选： cosine, fixed, inverse_sqrt, manual, pass_through, polynomial_decay, reduce_lr_on_plateau, step, tri_stage, triangular，默认为fixed。
# --criterion: 指定使用的损失函数，选择：adaptive_loss, composite_loss, cross_entropy, ctc, fastspeech2, hubert, label_smoothed_cross_entropy, latency_augmented_label_smoothed_cross_entropy, label_smoothed_cross_entropy_with_alignment, label_smoothed_cross_entropy_with_ctc, legacy_masked_lm_loss, masked_lm, model, nat_loss, sentence_prediction, sentence_prediction_adapters, sentence_ranking, tacotron2, speech_to_unit, speech_to_spectrogram, speech_unit_lm_criterion, wav2vec, vocab_parallel_cross_entropy
# --max-tokens: 按照词的数量来分batch，每个batch包含多少个词。
# --fp 16: 若使用的GPU支持半精度，可以通过--fp16来进行混合精度训练，可以极大提高模型训练的速度。通过torch.cuda.get_device_capablity(0)[0]可以确定GPU是否支持半精度（值小于7则不支持，大于7则支持。）
# --no-epoch-checkpoints: 只储存最后和最好的检查点
# --save-dir: 训练过程中保存中间模型，默认为checkpoints。
# --label-smoothing 0.1：将label_smoothed_cross_entropy损失默认为0的label-smoothing值改为0.1
# --reset-dataloader: 如果已设置，则不从检查点重新加载数据加载器状态, 默认值:False
# --reset-meters: 如果设置，则不从检查点加载仪表，默认值:False
# --reset-optimizer:如果设置，则不从检查点加载优化器状态，默认值:False
# --no-progress-bar参数可以改为逐行打印日志，方便保存。默认情况下，每训练100步之后会打印一次

# --gen-subset train：翻译整个训练数据
# --gen-subset: 默认解码测试部分。
# --beam: 设置beam search中的beam size
# --lenpen: 设置beam search中的长度惩罚
# --remove-bpe: 指定对翻译结果后处理，由于在准备数据时，使用了BPE切分，该参数会把BPE切分的词合并为完整的单词。如果不添加该参数，那么输出的翻译结果和BLEU打分都是按照未合并BPE进行的。
# --unkpen: unk惩罚。

> cd examples/translation/
# 在机器翻译中，需要双语平行数据来进行模型的训练，在这里使用fairseq中提供的数据，这个脚本会下载IWSLT 14 英语和德语的平行数据，并进行分词、BPE等操作。
> bash prepare-iwslt14.sh
> 
> cd ../..
> TEXT=examples/translation/iwslt14.tokenized.de-en
# 设置训练文件前缀、验证文件前缀、测试文件前缀等
# data-bin：预处理后的文件保存再哪里
# joined dictionary: 源和目标使用同一个词典，对于相似语言来说，有很多的单词是相同的，使用同一个词表可以降低词表和参数的总规模。
# fairseq-preprocess：将文本数据转化为二进制文件。
> fairseq-preprocess --source-lang de --target-lang en \
    --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \
    --destdir data-bin/iwslt14.tokenized.de-en

# arch: 所使用的模型结构
# optimizer：可以选择的优化器
# --clip-norm：梯度减少阈值
# lr：前N个批次的学习率。
# --lr-scheduler：学习率缩减的方式
# criterion：指定使用的损失函数。
# --max--tokens：按照词的数量来分batch，每个batch包含多少个词。
# 训练之后会生成pt后缀的文件，这个文件可以用于后续生成翻译结果。
> mkdir -p checkpoints/fconv
> CUDA_VISIBLE_DEVICES=0 fairseq-train data-bin/iwslt14.tokenized.de-en \
    --optimizer nag --lr 0.25 --clip-norm 0.1 --dropout 0.2 --max-tokens 4000 \
    --arch fconv_iwslt_de_en --save-dir checkpoints/fconv

# --gen-subset 
# --beam: 设置beam search中的beam size
# --lenpen: 设置beam search中的长度惩罚
# --remove-bpe: 指定对翻译结果进行后处理，该参数会把BPE切分的词合并起来。
# --path：模型路径
> fairseq-generate data-bin/iwslt14.tokenized.de-en \
    --path checkpoints/fconv/checkpoint_best.pt \
    --batch-size 128 --beam 5
| [de] dictionary: 35475 types
| [en] dictionary: 24739 types
| data-bin/iwslt14.tokenized.de-en test 6750 examples
| model fconv
| loaded checkpoint trainings/fconv/checkpoint_best.pt
S-721   danke .
T-721   thank you .
...

import torch.nn as nn
from fairseq import utils
from fairseq.models import FairseqEncoder
import torch
from fairseq.models import FairseqDecoder
from fairseq.models import FairseqEncoderDecoderModel, register_model
# Note: the register_model "decorator" should immediately precede the
# definition of the Model class.
class SimpleLSTMEncoder(FairseqEncoder):
    def __init__(
        self, args, dictionary, embed_dim=128, hidden_dim=128, dropout=0.1,
    ):
        super().__init__(dictionary)
        self.args = args
        # Our encoder will embed the inputs before feeding them to the LSTM.
        self.embed_tokens = nn.Embedding(
            num_embeddings=len(dictionary),
            embedding_dim=embed_dim,
            padding_idx=dictionary.pad(),
        )
        self.dropout = nn.Dropout(p=dropout)
        # We'll use a single-layer, unidirectional LSTM for simplicity.
        self.lstm = nn.LSTM(
            input_size=embed_dim,
            hidden_size=hidden_dim,
            num_layers=1,
            bidirectional=False,
            batch_first=True,
        )
    def forward(self, src_tokens, src_lengths):
        # The inputs to the ``forward()`` function are determined by the
        # Task, and in particular the ``'net_input'`` key in each
        # mini-batch. We discuss Tasks in the next tutorial, but for now just
        # know that *src_tokens* has shape `(batch, src_len)` and *src_lengths*
        # has shape `(batch)`.
        # Note that the source is typically padded on the left. This can be
        # configured by adding the `--left-pad-source "False"` command-line
        # argument, but here we'll make the Encoder handle either kind of
        # padding by converting everything to be right-padded.
        if self.args.left_pad_source:
            # Convert left-padding to right-padding.
            src_tokens = utils.convert_padding_direction(
                src_tokens,
                padding_idx=self.dictionary.pad(),
                left_to_right=True
            )
        # Embed the source.
        x = self.embed_tokens(src_tokens)
        # Apply dropout.
        x = self.dropout(x)
        # Pack the sequence into a PackedSequence object to feed to the LSTM.
        x = nn.utils.rnn.pack_padded_sequence(x, src_lengths, batch_first=True)
        # Get the output from the LSTM.
        _outputs, (final_hidden, _final_cell) = self.lstm(x)
        # Return the Encoder's output. This can be any object and will be
        # passed directly to the Decoder.
        return {
            # this will have shape `(bsz, hidden_dim)`
            'final_hidden': final_hidden.squeeze(0),
        }
    # Encoders are required to implement this method so that we can rearrange
    # the order of the batch elements during inference (e.g., beam search).
    def reorder_encoder_out(self, encoder_out, new_order):
        """
        Reorder encoder output according to `new_order`.
        Args:
            encoder_out: output from the ``forward()`` method
            new_order (LongTensor): desired order
        Returns:
            `encoder_out` rearranged according to `new_order`
        """
        final_hidden = encoder_out['final_hidden']
        return {
            'final_hidden': final_hidden.index_select(0, new_order),
        }
class SimpleLSTMDecoder(FairseqDecoder):
    def __init__(
        self, dictionary, encoder_hidden_dim=128, embed_dim=128, hidden_dim=128,
        dropout=0.1,
    ):
        super().__init__(dictionary)
        # Our decoder will embed the inputs before feeding them to the LSTM.
        self.embed_tokens = nn.Embedding(
            num_embeddings=len(dictionary),
            embedding_dim=embed_dim,
            padding_idx=dictionary.pad(),
        )
        self.dropout = nn.Dropout(p=dropout)
        # We'll use a single-layer, unidirectional LSTM for simplicity.
        self.lstm = nn.LSTM(
            # For the first layer we'll concatenate the Encoder's final hidden
            # state with the embedded target tokens.
            input_size=encoder_hidden_dim + embed_dim,
            hidden_size=hidden_dim,
            num_layers=1,
            bidirectional=False,
        )
        # Define the output projection.
        self.output_projection = nn.Linear(hidden_dim, len(dictionary))
    # During training Decoders are expected to take the entire target sequence
    # (shifted right by one position) and produce logits over the vocabulary.
    # The *prev_output_tokens* tensor begins with the end-of-sentence symbol,
    # ``dictionary.eos()``, followed by the target sequence.
    def forward(self, prev_output_tokens, encoder_out):
        """
        Args:
            prev_output_tokens (LongTensor): previous decoder outputs of shape
                `(batch, tgt_len)`, for teacher forcing
            encoder_out (Tensor, optional): output from the encoder, used for
                encoder-side attention
        Returns:
            tuple:
                - the last decoder layer's output of shape
                  `(batch, tgt_len, vocab)`
                - the last decoder layer's attention weights of shape
                  `(batch, tgt_len, src_len)`
        """
        bsz, tgt_len = prev_output_tokens.size()
        # Extract the final hidden state from the Encoder.
        final_encoder_hidden = encoder_out['final_hidden']
        # Embed the target sequence, which has been shifted right by one
        # position and now starts with the end-of-sentence symbol.
        x = self.embed_tokens(prev_output_tokens)
        # Apply dropout.
        x = self.dropout(x)
        # Concatenate the Encoder's final hidden state to *every* embedded
        # target token.
        x = torch.cat(
            [x, final_encoder_hidden.unsqueeze(1).expand(bsz, tgt_len, -1)],
            dim=2,
        )
        # Using PackedSequence objects in the Decoder is harder than in the
        # Encoder, since the targets are not sorted in descending length order,
        # which is a requirement of ``pack_padded_sequence()``. Instead we'll
        # feed nn.LSTM directly.
        initial_state = (
            final_encoder_hidden.unsqueeze(0),  # hidden
            torch.zeros_like(final_encoder_hidden).unsqueeze(0),  # cell
        )
        output, _ = self.lstm(
            x.transpose(0, 1),  # convert to shape `(tgt_len, bsz, dim)`
            initial_state,
        )
        x = output.transpose(0, 1)  # convert to shape `(bsz, tgt_len, hidden)`
        # Project the outputs to the size of the vocabulary.
        x = self.output_projection(x)
        # Return the logits and ``None`` for the attention weights
        return x, None
# 注册模型
@register_model('simple_lstm')
class SimpleLSTMModel(FairseqEncoderDecoderModel):
    @staticmethod
    def add_args(parser):
        # Models can override this method to add new command-line arguments.
        # Here we'll add some new command-line arguments to configure dropout
        # and the dimensionality of the embeddings and hidden states.
        parser.add_argument(
            '--encoder-embed-dim', type=int, metavar='N',
            help='dimensionality of the encoder embeddings',
        )
        parser.add_argument(
            '--encoder-hidden-dim', type=int, metavar='N',
            help='dimensionality of the encoder hidden state',
        )
        parser.add_argument(
            '--encoder-dropout', type=float, default=0.1,
            help='encoder dropout probability',
        )
        parser.add_argument(
            '--decoder-embed-dim', type=int, metavar='N',
            help='dimensionality of the decoder embeddings',
        )
        parser.add_argument(
            '--decoder-hidden-dim', type=int, metavar='N',
            help='dimensionality of the decoder hidden state',
        )
        parser.add_argument(
            '--decoder-dropout', type=float, default=0.1,
            help='decoder dropout probability',
        )
    @classmethod
    def build_model(cls, args, task):
        # Fairseq initializes models by calling the ``build_model()``
        # function. This provides more flexibility, since the returned model
        # instance can be of a different type than the one that was called.
        # In this case we'll just return a SimpleLSTMModel instance.
        # Initialize our Encoder and Decoder.
        encoder = SimpleLSTMEncoder(
            args=args,
            dictionary=task.source_dictionary,
            embed_dim=args.encoder_embed_dim,
            hidden_dim=args.encoder_hidden_dim,
            dropout=args.encoder_dropout,
        )
        decoder = SimpleLSTMDecoder(
            dictionary=task.target_dictionary,
            encoder_hidden_dim=args.encoder_hidden_dim,
            embed_dim=args.decoder_embed_dim,
            hidden_dim=args.decoder_hidden_dim,
            dropout=args.decoder_dropout,
        )
        model = SimpleLSTMModel(encoder, decoder)
        # Print the model architecture.
        print(model)
        return model
    # We could override the ``forward()`` if we wanted more control over how
    # the encoder and decoder interact, but it's not necessary for this
    # tutorial since we can inherit the default implementation provided by
    # the FairseqEncoderDecoderModel base class, which looks like:
    #
    # def forward(self, src_tokens, src_lengths, prev_output_tokens):
    #     encoder_out = self.encoder(src_tokens, src_lengths)
    #     decoder_out = self.decoder(prev_output_tokens, encoder_out)
    #     return decoder_out

# Download and prepare the unidirectional data
bash prepare-iwslt14.sh
# Preprocess/binarize the unidirectional data
TEXT=iwslt14.tokenized.de-en
fairseq-preprocess --source-lang de --target-lang en \
    --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test \
    --destdir data-bin/iwslt14.tokenized.de-en \
    --joined-dictionary --workers 20

import torch
from fairseq.models import FairseqIncrementalDecoder
class SimpleLSTMDecoder(FairseqIncrementalDecoder):
    def __init__(
        self, dictionary, encoder_hidden_dim=128, embed_dim=128, hidden_dim=128,
        dropout=0.1,
    ):
        # This remains the same as before.
        super().__init__(dictionary)
        self.embed_tokens = nn.Embedding(
            num_embeddings=len(dictionary),
            embedding_dim=embed_dim,
            padding_idx=dictionary.pad(),
        )
        self.dropout = nn.Dropout(p=dropout)
        self.lstm = nn.LSTM(
            input_size=encoder_hidden_dim + embed_dim,
            hidden_size=hidden_dim,
            num_layers=1,
            bidirectional=False,
        )
        self.output_projection = nn.Linear(hidden_dim, len(dictionary))
    # We now take an additional kwarg (*incremental_state*) for caching the
    # previous hidden and cell states.
    def forward(self, prev_output_tokens, encoder_out, incremental_state=None):
        if incremental_state is not None:
            # If the *incremental_state* argument is not ``None`` then we are
            # in incremental inference mode. While *prev_output_tokens* will
            # still contain the entire decoded prefix, we will only use the
            # last step and assume that the rest of the state is cached.
            prev_output_tokens = prev_output_tokens[:, -1:]
        # This remains the same as before.
        bsz, tgt_len = prev_output_tokens.size()
        final_encoder_hidden = encoder_out['final_hidden']
        x = self.embed_tokens(prev_output_tokens)
        x = self.dropout(x)
        x = torch.cat(
            [x, final_encoder_hidden.unsqueeze(1).expand(bsz, tgt_len, -1)],
            dim=2,
        )
        # We will now check the cache and load the cached previous hidden and
        # cell states, if they exist, otherwise we will initialize them to
        # zeros (as before). We will use the ``utils.get_incremental_state()``
        # and ``utils.set_incremental_state()`` helpers.
        initial_state = utils.get_incremental_state(
            self, incremental_state, 'prev_state',
        )
        if initial_state is None:
            # first time initialization, same as the original version
            initial_state = (
                final_encoder_hidden.unsqueeze(0),  # hidden
                torch.zeros_like(final_encoder_hidden).unsqueeze(0),  # cell
            )
        # Run one step of our LSTM.
        output, latest_state = self.lstm(x.transpose(0, 1), initial_state)
        # Update the cache with the latest hidden and cell states.
        utils.set_incremental_state(
            self, incremental_state, 'prev_state', latest_state,
        )
        # This remains the same as before
        x = output.transpose(0, 1)
        x = self.output_projection(x)
        return x, None
    # The ``FairseqIncrementalDecoder`` interface also requires implementing a
    # ``reorder_incremental_state()`` method, which is used during beam search
    # to select and reorder the incremental state.
    def reorder_incremental_state(self, incremental_state, new_order):
        # Load the cached state.
        prev_state = utils.get_incremental_state(
            self, incremental_state, 'prev_state',
        )
        # Reorder batches according to *new_order*.
        reordered_state = (
            prev_state[0].index_select(1, new_order),  # hidden
            prev_state[1].index_select(1, new_order),  # cell
        )
        # Update the cached state.
        utils.set_incremental_state(
            self, incremental_state, 'prev_state', reordered_state,
        )
# 下一个案例有时间再分析吧，有些许疲惫。

在Linux下，建立好simple_lstm.py文件并将代码复制后，需要给与执行权限chomd +x simple_lstm.py, 之后再执行一下该文件（python simple_lstm.py）才算注册模型完成。