多任务高斯过程数学原理和Pytorch实现示例

多任务高斯过程

Intrinsic model of coregionalization（ICM）

ICM（共区域化的内在模型）方法通过引入核心区域化矩阵 (B) 来推广独立多输出高斯过程，该矩阵模型化任务之间的相关性。ICM方法中的协方差函数定义如下：

ICM方法可以学习任务之间的共享结构。任务之间的皮尔逊相关系数可以表示为：

Linear model of coregionalization (LMC)

PyTorch实现

 %pip install torch gpytorch matplotlib seaborn numpy pandas

import torch
import gpytorch
from matplotlib import pyplot as plt
import numpy as np
import seaborn as sns
import pandas as pd


 # Define the kernel with coregionalization
def __init__(self, train_x, train_y, likelihood, num_tasks):
)
)

def forward(self, x):
mean_x = self.mean_module(x)
covar_x = self.covar_module(x)

# Training data
f1 = lambda x:  torch.sin(x * (2 * torch.pi))
f2 = lambda x: torch.sin((x - 0.1) * (2 * torch.pi))
train_x = torch.linspace(0, 1, 10)
train_y = torch.stack([
f1(train_x),
f2(train_x)
]).T
# Define the noise covariance matrix with correlation = 0.3
sigma2 = 0.1**2
Sigma = torch.tensor([[sigma2, 0.3 * sigma2], [0.3 * sigma2, sigma2]])
# Add noise to the training data
train_y += torch.tensor(np.random.multivariate_normal(mean=[0,0], cov=Sigma, size=len(train_x)))

# Model and likelihood

# Training the model
model.train()
likelihood.train()

mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model)

scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.5)

num_iter = 500
for i in range(num_iter):
output = model(train_x)
loss = -mll(output, train_y)
loss.backward()
optimizer.step()
scheduler.step()

# Evaluation
model.eval()
likelihood.eval()

test_x = torch.linspace(0, 1, 100)

pred_multi = likelihood(model(test_x))

# Plot predictions
fig, ax = plt.subplots()

colors = ['blue', 'red']
ax.plot(test_x, pred_multi.mean[:, i], label=f'Mean prediction (Task {i+1})', color=colors[i])
ax.plot(test_x, [f1(test_x), f2(test_x)][i], linestyle='--', label=f'True function (Task {i+1})')
lower = pred_multi.confidence_region()[0][:, i].detach().numpy()
upper = pred_multi.confidence_region()[1][:, i].detach().numpy()
ax.fill_between(
test_x,
lower,
upper,
alpha=0.2,
color=colors[i]
)

ax.scatter(train_x, train_y[:, 0], color='black', label=f'Training data (Task 1)')
ax.scatter(train_x, train_y[:, 1], color='gray', label=f'Training data (Task 2)')

ax.legend(loc='lower center', bbox_to_anchor=(0.5, -0.2),
ncol=3, fancybox=True)


GPyTorch


MultitaskMean


MultitaskKernel


MultitaskGaussianLikelihood


 W = model.covar_module.task_covar_module.covar_factor
B = W @ W.T

fig, ax = plt.subplots()
sns.heatmap(B.detach().numpy(), annot=True, ax=ax, cbar=False, square=True)
ax.set_title('Coregionalization matrix B')
fig.show()

Sigma = L @ L.T

fig, ax = plt.subplots()
sns.heatmap(Sigma, annot=True, ax=ax, cbar=False, square=True)
ax.set_title('Noise covariance matrix')
fig.show()


比较

 class IndependentGPModel(gpytorch.models.ExactGP):
def __init__(self, train_x, train_y, likelihood):
super(IndependentGPModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = gpytorch.means.ConstantMean()
self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel())

def forward(self, x):
mean_x = self.mean_module(x)
covar_x = self.covar_module(x)
return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)

# Create models and likelihoods for each task
likelihoods = [gpytorch.likelihoods.GaussianLikelihood() for _ in range(num_tasks)]
models = [IndependentGPModel(train_x, train_y[:, i], likelihoods[i]) for i in range(num_tasks)]

# Training the independent models
for i, (model, likelihood) in enumerate(zip(models, likelihoods)):
model.train()
likelihood.train()
mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.5)

for _ in range(num_iter):
output = model(train_x)
loss = -mll(output, train_y[:, i])
loss.backward()
optimizer.step()
scheduler.step()

# Evaluation
for model, likelihood in zip(models, likelihoods):
model.eval()
likelihood.eval()

pred_inde = [likelihood(model(test_x)) for model, likelihood in zip(models, likelihoods)]

# Plot predictions
fig, ax = plt.subplots()

ax.plot(test_x, pred_inde[i].mean, label=f'Mean prediction (Task {i+1})', color=colors[i])
ax.plot(test_x, [f1(test_x), f2(test_x)][i], linestyle='--', label=f'True function (Task {i+1})')
lower = pred_inde[i].confidence_region()[0]
upper = pred_inde[i].confidence_region()[1]
ax.fill_between(
test_x,
lower,
upper,
alpha=0.2,
color=colors[i]
)

ax.scatter(train_x, train_y[:, 0], color='black', label='Training data (Task 1)')
ax.scatter(train_x, train_y[:, 1], color='gray', label='Training data (Task 2)')

ax.set_title('Independent GPs')
ax.legend(loc='lower center', bbox_to_anchor=(0.5, -0.2),
ncol=3, fancybox=True)


 mean_multi = pred_multi.mean.numpy()
mean_inde = np.stack([pred.mean.numpy() for pred in pred_inde]).T

test_y = torch.stack([f1(test_x), f2(test_x)]).T.numpy()
MSE_multi = np.mean((mean_multi - test_y) ** 2)
MSE_inde = np.mean((mean_inde - test_y) ** 2)

df = pd.DataFrame({
'Model': ['ICM', 'Independent'],
'MSE': [MSE_multi, MSE_inde]
})
df


https://avoid.overfit.cn/post/f804e93bd5dd4c4ab9ede5bf1bc9b6c8

|
3月前
|

209 0
|
21天前
|

Pytorch的编译新特性TorchDynamo的工作原理和使用示例
PyTorch的TorchDynamo是一个即时编译器，用于优化动态图执行，提高运行效率。它在运行时分析和转换代码，应用优化技术，如操作符融合，然后编译成高效机器码。通过一个包含特征工程、超参数调整、交叉验证的合成数据集示例，展示了TorchDynamo如何减少训练时间并提高模型性能。它易于集成，只需对现有PyTorch代码进行小改动，即可利用其性能提升。TorchDynamo的优化包括动态捕获计算图、应用优化和编译，适用于实时应用和需要快速响应的场景。
34 11
|
3月前
|

57 1
|
3月前
|

PyTorch实战：图像分类任务的实现与优化
【4月更文挑战第17天】本文介绍了使用PyTorch实现图像分类任务的步骤，包括数据集准备（如使用CIFAR-10数据集）、构建简单的CNN模型、训练与优化模型以及测试模型性能。在训练过程中，使用了交叉熵损失和SGD优化器。此外，文章还讨论了提升模型性能的策略，如调整模型结构、数据增强、正则化和利用预训练模型。通过本文，读者可掌握基础的PyTorch图像分类实践。
189 1
|
3月前
|

【4月更文挑战第17天】本文深入解析PyTorch的自动微分机制，重点讨论反向传播的原理和实现。反向传播利用链式法则计算神经网络的梯度，包括前向传播、梯度计算、反向传播及参数更新。PyTorch通过autograd模块实现自动微分，使用Tensor和计算图记录操作历史以自动计算梯度。通过示例展示了如何在PyTorch中创建张量、定义计算过程及求梯度。掌握这些有助于提升深度学习模型的训练效率。
255 1
|
3月前
|
PyTorch 算法框架/工具

85 6
|
10月前
|

PyTorch Geometric (PyG)是构建图神经网络模型和实验各种图卷积的主要工具。在本文中我们将通过链接预测来对其进行介绍。
39 0
|
3月前
|

【PyTorch实战演练】深入剖析MTCNN(多任务级联卷积神经网络)并使用30行代码实现人脸识别
【PyTorch实战演练】深入剖析MTCNN(多任务级联卷积神经网络)并使用30行代码实现人脸识别
306 2
|
3月前
|

PyTorch在NLP任务中的应用：文本分类、序列生成等
【4月更文挑战第18天】PyTorch在NLP中应用于文本分类和序列生成，支持RNN、CNN、Transformer等模型构建。其动态计算图、丰富API及强大社区使其在NLP研究中备受欢迎。预训练模型和多模态学习的发展将进一步拓宽PyTorch在NLP的应用前景。
44 0
|
3月前
|

513 2