经典神经网络架构参考 v1.0(3)https://developer.aliyun.com/article/1489285
4.4 卷积自编码器
主体：
digraph ConvAutoEncoder { rankdir=BT node [ style=filled, color=Black fontcolor=White, fillcolor="#30638e", fontname="SimHei", fontsize=32, width=5, height=2, shape="box", ] inp [label="输入\n[BatchSize,\n W=256, H=256, C=3]", shape="Mrecord"] conv1 [label="ConvBlock1\n[In=3, Out=16]"] featmap_e1 [label="[BatchSize,\nW=128, H=128, C=16]", shape="Mrecord"] conv2 [label="ConvBlock2\n[In=16, Out=32]"] featmap_e2 [label="[BatchSize,\nW=64, H=64, C=32]", shape="Mrecord"] conv3 [label="ConvBlock3\n[In=32, Out=32]"] featmap_e3 [label="[BatchSize,\nW=32, H=32, C=32]", shape="Mrecord"] upconv3 [label="UpConvBlock3\n[In=32, Out=32]"] featmap_d2 [label="[BatchSize,\nW=64, H=64, C=32]", shape="Mrecord"] upconv2 [label="UpConvBlock2\n[In=32, Out=16]"] featmap_d1 [label="[BatchSize,\nW=128, H=128, C=16", shape="Mrecord"] upconv1 [label="UpConvBlock1\n[In=16, Out=3]"] tanh [label="Tanh"] oup [label="输出\n[BatchSize,\n W=256, H=256, C=3]", shape="Mrecord"] inp -> conv1 -> featmap_e1 -> conv2 -> featmap_e2 -> conv3 -> featmap_e3 -> upconv3 -> featmap_d2 -> upconv2 -> featmap_d1 -> upconv1 -> tanh -> oup }
卷积块：
digraph CAEConvBlock { rankdir=BT node [ style=filled, color=Black fontcolor=White, fillcolor="#30638e", fontname="SimHei", fontsize=32, width=5, height=2, shape="box", ] inp [label="输入\n[BatchSize,\n W=BlockW,\n H=BlockH,\n C=BlockIn]", shape="Mrecord"] conv [label="Conv2D\n[K=3, P=1,\n In=BlockIn,\n Out=BlockOut]"] batchnorm [label="BatchNorm(BlockOut)"] relu [label="Relu"] maxpool [label="MaxPool2D\n[K=2, S=2]"] oup [label="输出\n[BatchSize,\n W=BlockW/2,\n H=BlockH/2,\n C=BlockOut]", shape="Mrecord"] inp -> conv -> batchnorm -> relu -> maxpool -> oup }
反卷积块：
digraph CAEDeconvBlock { rankdir=BT node [ style=filled, color=Black fontcolor=White, fillcolor="#30638e", fontname="SimHei", fontsize=32, width=5, height=2, shape="box", ] inp [label="输入\n[BatchSize,\n W=BlockW,\n H=BlockH,\n C=BlockIn]", shape="Mrecord"] upsamp [label="UpSample x2"] conv [label="Conv2D\n[K=3, P=1,\n In=BlockIn,\n Out=BlockOut]"] batchnorm [label="BatchNorm(BlockOut)"] relu [label="Relu"] oup [label="输出\n[BatchSize,\n W=BlockW*2,\n H=BlockH*2,\n C=BlockOut]", shape="Mrecord"] inp -> upsamp -> conv -> batchnorm -> relu -> oup }
五、循环神经网络
5.1 时序 RNN
digraph TimeRNN { rankdir=BT node [ style=filled, color=Black fontcolor=White, fillcolor="#30638e", fontname="SimHei", fontsize=32, width=5, height=2, shape="box", ] inp [label="输入\n[BatchSize,\n SeqLen, NFeature]", shape="Mrecord"] rnn1 [label="RNNCell1\n[NFeature, NHidden]"] rnn_rest [label="RNNCell x (NLayers-1)\n[NHidden, NHidden]"] hidst [label="[BatchSize,\n SeqLen, NHidden]", shape="Mrecord"] ll [label="Linear\n[NHidden, NLabels]"] oup [label="输出\n[BatchSize,\n SeqLen, NLabels]", shape="Mrecord"] inp -> rnn1 -> rnn_rest -> hidst -> ll -> oup }
5.2 NLP RNN
digraph NLPRNN { rankdir=BT node [ style=filled, color=Black fontcolor=White, fillcolor="#30638e", fontname="SimHei", fontsize=32, width=5, height=2, shape="box", ] inp [label="输入（单词ID）\n[BatchSize, SeqLen]", shape="Mrecord"] emb [label="Embedding\n[VocabSize, NHidden]"] inp_emb [label="[BatchSize,\n SeqLen, NHidden]", shape="Mrecord"] rnn [label="RNNCell x NLayers\n[NHidden, NHidden]"] hidst [label="[BatchSize,\n SeqLen, NHidden]", shape="Mrecord"] emb_w_t [label="@ Embedding.w.T\n[NHidden, VocabSize]"] oup [label="输出（单词概率）\n[BatchSize,\n SeqLen, VocabSize]", shape="Mrecord"] inp -> emb -> inp_emb -> rnn -> hidst -> emb_w_t -> oup } • 1 • 2 • 3 • 4 • 5 • 6 • 7 • 8 • 9 • 10 • 11 • 12 • 13 • 14 • 15 • 16 • 17 • 18 • 19 • 20 • 21 • 22 • 23 • 24
5.3 RNN 单元
digraph RNNCell { rankdir=BT node [ style=filled, color=Black fontcolor=White, fillcolor="#30638e", fontname="SimHei", fontsize=32, width=5, height=2, shape="box", ] x [label="X[t]（输入）\n[BatchSize, NIn]", shape="Mrecord"] his [label="H[t-1]（历史）\n[BatchSize, NOut]", shape="Mrecord"] w_x [label="W（参数）\n[NIn, NOut]", shape="Mrecord"] w_h [label="U（参数）\n[NOut, NOut]", shape="Mrecord"] b [label="b（参数）\n[1, NOut]", shape="Mrecord"] σ [label="Sigmoid"] matmul1 [label="@"] matmul2 [label="@"] plus [label="+"] h [label="H[t]（输出）\n[BatchSize, NOut]", shape="Mrecord"] x -> matmul1 w_x -> matmul1 his -> matmul2 w_h -> matmul2 matmul1 -> plus matmul2 -> plus b -> plus plus -> σ σ -> h }
5.4 LSTM 单元
digraph GRUCell { rankdir=BT node [ style=filled, color=Black fontcolor=White, fillcolor="#30638e", fontname="SimHei", fontsize=32, width=5, height=2, shape="box", ] x [label="X[t]（输入）\n[BatchSize, NIn]", shape="Mrecord"] his_h [label="H[t-1]（短期历史）\n[BatchSize, NOut]", shape="Mrecord"] his_c [label="C[t-1]（长期历史）\n[BatchSize, NOut]", shape="Mrecord"] calc_f [label="σ(X[t] @ W[f]\n + H[t-1] @ U[f]\n + b[f])"] calc_i [label="σ(X[t] @ W[i]\n + H[t-1] @ U[i]\n + b[i])"] calc_c_cand [label="tanh(X[t] @ W[c]\n + H[t-1] @ U[c]\n + b[c])"] calc_o [label="σ(X[t] @ W[o]\n + H[t-1] @ U[o]\n + b[o])"] f [label="F[t]（遗忘门）\n[BatchSize, NOut]", shape="Mrecord"] i [label="I[t]（记忆门）\n[BatchSize, NOut]", shape="Mrecord"] c_cand [label="CCand[t]（长期候补）\n[BatchSize, NOut]", shape="Mrecord"] o [label="O[t]（输出门）\n[BatchSize, NOut]", shape="Mrecord"] calc_c [label="F[t] * C[t-1]\n + I[t] * CCand[t]"] calc_h [label="C[t] * tanh(O[t])"] h [label="H[t]（输出&短期记忆）\n[BatchSize, NOut]", shape="Mrecord"] c [label="C[t]（长期记忆）\n[BatchSize, NOut]", shape="Mrecord"] x -> calc_f x -> calc_i x -> calc_c_cand x -> calc_o his_h -> calc_f -> f his_h -> calc_i -> i his_h -> calc_c_cand -> c_cand his_h -> calc_o -> o f -> calc_c i -> calc_c c_cand -> calc_c calc_c -> c c -> calc_h o -> calc_h calc_h -> h }
5.5 GRU 单元
digraph LSTMCell { rankdir=BT node [ style=filled, color=Black fontcolor=White, fillcolor="#30638e", fontname="SimHei", fontsize=32, width=5, height=2, shape="box", ] x [label="X[t]（输入）\n[BatchSize, NIn]", shape="Mrecord"] his_h [label="H[t-1]（历史）\n[BatchSize, NOut]", shape="Mrecord"] calc_z [label="σ(X[t] @ W[z]\n + H[t-1] @ U[x]\n + b[z])"] calc_r [label="σ(X[t] @ W[r]\n + H[t-1] @ U[r]\n + b[r])"] calc_h_cand [label="tanh(X[t] @ W[h]\n + (H[t-1] * R[t]) @ U[h]\n + b[h])"] z [label="Z[t]（更新门）\n[BatchSize, NOut]", shape="Mrecord"] r [label="R[t]（复位门）\n[BatchSize, NOut]", shape="Mrecord"] h_cand [label="HCand[t]（候补）\n[BatchSize, NOut]", shape="Mrecord"] calc_h [label="(1 - Z[t]) * H[t-1]\n + Z[t] * HCand[t]"] h [label="H[t]（输出）\n[BatchSize, NOut]", shape="Mrecord"] x -> calc_z x -> calc_r x -> calc_h_cand his_h -> calc_z -> z his_h -> calc_r -> r his_h -> calc_h_cand r -> calc_h_cand -> h_cand z -> calc_h h_cand -> calc_h calc_h -> h } • 1 • 2 • 3 • 4 • 5 • 6 • 7 • 8 • 9 • 10 • 11 • 12 • 13 • 14 • 15 • 16 • 17 • 18 • 19 • 20 • 21 • 22 • 23 • 24 • 25 • 26 • 27 • 28 • 29 • 30 • 31 • 32 • 33 • 34 • 35 • 36 • 37 • 38 • 39 • 40 • 41
