目标尝试用pytorch实现一个transformer层。内容importmathimporttorchimporttorch.nnasnn# 1. 多头自注意力机制classMultiHeadSelfAttention(nn.Module):def__init__(self,d_model32,num_heads4):super().__init__()assertd_model%num_heads0self.num_headsnum_heads self.head_dimd_model//num_heads# Q、K、V 的线性映射self.W_qnn.Linear(d_model,d_model)self.W_knn.Linear(d_model,d_model)self.W_vnn.Linear(d_model,d_model)# 多头拼接后的输出映射self.W_onn.Linear(d_model,d_model)defforward(self,x):batch_size,seq_len,d_modelx.shape Qself.W_q(x)Kself.W_k(x)Vself.W_v(x)# 拆分多个注意力头# [B, S, D] - [B, H, S, head_dim]defsplit_heads(tensor):returntensor.view(batch_size,seq_len,self.num_heads,self.head_dim).transpose(1,2)Qsplit_heads(Q)Ksplit_heads(K)Vsplit_heads(V)# 计算注意力得分scorestorch.matmul(Q,K.transpose(-2,-1))/math.sqrt(self.head_dim)# Softmax 转换为注意力权重attention_weightstorch.softmax(scores,dim-1)# 加权求和contexttorch.matmul(attention_weights,V)# 拼接多个注意力头contextcontext.transpose(1,2)contextcontext.contiguous().view(batch_size,seq_len,d_model)returnself.W_o(context)# 2. Transformer Encoder LayerclassTransformerLayer(nn.Module):def__init__(self,d_model32,num_heads4,d_ff128,dropout0.1):super().__init__()self.attentionMultiHeadSelfAttention(d_model,num_heads)self.norm1nn.LayerNorm(d_model)self.norm2nn.LayerNorm(d_model)self.ffnnn.Sequential(nn.Linear(d_model,d_ff),nn.ReLU(),nn.Linear(d_ff,d_model))self.dropoutnn.Dropout(dropout)defforward(self,x):# 第一步多头自注意力attention_outputself.attention(x)# 第二步残差连接 LayerNormxself.norm1(xself.dropout(attention_output))# 第三步前馈神经网络ffn_outputself.ffn(x)# 第四步残差连接 LayerNormxself.norm2(xself.dropout(ffn_output))returnx# 3. 验证 Transformer 层if__name____main__:torch.manual_seed(42)modelTransformerLayer(d_model32,num_heads4,d_ff128)# 2个样本每个样本5个Token# 每个Token使用32维向量表示xtorch.randn(2,5,32)outputmodel(x)print(输入形状,x.shape)print(输出形状,output.shape)# 验证反向传播lossoutput.square().mean()loss.backward()assertoutput.shapex.shapeassertmodel.attention.W_q.weight.gradisnotNoneprint(Transformer层测试成功)