182 lines
6.6 KiB
Python
182 lines
6.6 KiB
Python
|
"""
|
||
|
|
Loss functions for frame prediction and representation learning
|
||
|
|
"""
|
||
|
|
import torch
|
||
|
|
import torch.nn as nn
|
||
|
|
import torch.nn.functional as F
|
||
|
|
import math
|
||
|
|
|
||
|
|
|
||
|
|
class SSIMLoss(nn.Module):
|
||
|
|
"""
|
||
|
|
Structural Similarity Index Measure Loss
|
||
|
|
Based on: https://github.com/Po-Hsun-Su/pytorch-ssim
|
||
|
|
"""
|
||
|
|
def __init__(self, window_size=11, size_average=True):
|
||
|
|
super().__init__()
|
||
|
|
self.window_size = window_size
|
||
|
|
self.size_average = size_average
|
||
|
|
self.channel = 3
|
||
|
|
self.window = self.create_window(window_size, self.channel)
|
||
|
|
|
||
|
|
def create_window(self, window_size, channel):
|
||
|
|
def gaussian(window_size, sigma):
|
||
|
|
gauss = torch.Tensor([math.exp(-(x - window_size//2)**2/float(2*sigma**2)) for x in range(window_size)])
|
||
|
|
return gauss/gauss.sum()
|
||
|
|
|
||
|
|
_1D_window = gaussian(window_size, 1.5).unsqueeze(1)
|
||
|
|
_2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0)
|
||
|
|
window = _2D_window.expand(channel, 1, window_size, window_size).contiguous()
|
||
|
|
return window
|
||
|
|
|
||
|
|
def forward(self, img1, img2):
|
||
|
|
# Ensure window is on correct device
|
||
|
|
if self.window.device != img1.device:
|
||
|
|
self.window = self.window.to(img1.device)
|
||
|
|
|
||
|
|
mu1 = F.conv2d(img1, self.window, padding=self.window_size//2, groups=self.channel)
|
||
|
|
mu2 = F.conv2d(img2, self.window, padding=self.window_size//2, groups=self.channel)
|
||
|
|
|
||
|
|
mu1_sq = mu1.pow(2)
|
||
|
|
mu2_sq = mu2.pow(2)
|
||
|
|
mu1_mu2 = mu1 * mu2
|
||
|
|
|
||
|
|
sigma1_sq = F.conv2d(img1*img1, self.window, padding=self.window_size//2, groups=self.channel) - mu1_sq
|
||
|
|
sigma2_sq = F.conv2d(img2*img2, self.window, padding=self.window_size//2, groups=self.channel) - mu2_sq
|
||
|
|
sigma12 = F.conv2d(img1*img2, self.window, padding=self.window_size//2, groups=self.channel) - mu1_mu2
|
||
|
|
|
||
|
|
C1 = 0.01**2
|
||
|
|
C2 = 0.03**2
|
||
|
|
|
||
|
|
ssim_map = ((2*mu1_mu2 + C1)*(2*sigma12 + C2)) / ((mu1_sq + mu2_sq + C1)*(sigma1_sq + sigma2_sq + C2))
|
||
|
|
|
||
|
|
if self.size_average:
|
||
|
|
return 1 - ssim_map.mean()
|
||
|
|
else:
|
||
|
|
return 1 - ssim_map.mean(1).mean(1).mean(1)
|
||
|
|
|
||
|
|
|
||
|
|
class FramePredictionLoss(nn.Module):
|
||
|
|
"""
|
||
|
|
Combined loss for frame prediction
|
||
|
|
"""
|
||
|
|
def __init__(self, l1_weight=1.0, ssim_weight=0.1, use_ssim=True):
|
||
|
|
super().__init__()
|
||
|
|
self.l1_weight = l1_weight
|
||
|
|
self.ssim_weight = ssim_weight
|
||
|
|
self.use_ssim = use_ssim
|
||
|
|
|
||
|
|
self.l1_loss = nn.L1Loss()
|
||
|
|
if use_ssim:
|
||
|
|
self.ssim_loss = SSIMLoss()
|
||
|
|
|
||
|
|
def forward(self, pred, target):
|
||
|
|
"""
|
||
|
|
Args:
|
||
|
|
pred: predicted frame [B, 3, H, W] in range [-1, 1]
|
||
|
|
target: target frame [B, 3, H, W] in range [-1, 1]
|
||
|
|
Returns:
|
||
|
|
total_loss, loss_dict
|
||
|
|
"""
|
||
|
|
loss_dict = {}
|
||
|
|
|
||
|
|
# L1 loss
|
||
|
|
l1_loss = self.l1_loss(pred, target)
|
||
|
|
loss_dict['l1'] = l1_loss
|
||
|
|
total_loss = self.l1_weight * l1_loss
|
||
|
|
|
||
|
|
# SSIM loss
|
||
|
|
if self.use_ssim:
|
||
|
|
ssim_loss = self.ssim_loss(pred, target)
|
||
|
|
loss_dict['ssim'] = ssim_loss
|
||
|
|
total_loss += self.ssim_weight * ssim_loss
|
||
|
|
|
||
|
|
loss_dict['total'] = total_loss
|
||
|
|
return total_loss, loss_dict
|
||
|
|
|
||
|
|
|
||
|
|
class ContrastiveLoss(nn.Module):
|
||
|
|
"""
|
||
|
|
Contrastive loss for representation learning
|
||
|
|
Positive pairs: representations from adjacent frames
|
||
|
|
Negative pairs: representations from distant frames
|
||
|
|
"""
|
||
|
|
def __init__(self, temperature=0.1, margin=1.0):
|
||
|
|
super().__init__()
|
||
|
|
self.temperature = temperature
|
||
|
|
self.margin = margin
|
||
|
|
self.cosine_similarity = nn.CosineSimilarity(dim=-1)
|
||
|
|
|
||
|
|
def forward(self, representations, temporal_indices):
|
||
|
|
"""
|
||
|
|
Args:
|
||
|
|
representations: [B, D] representation vectors
|
||
|
|
temporal_indices: [B] temporal indices of each sample
|
||
|
|
Returns:
|
||
|
|
contrastive_loss
|
||
|
|
"""
|
||
|
|
batch_size = representations.size(0)
|
||
|
|
|
||
|
|
# Compute similarity matrix
|
||
|
|
sim_matrix = torch.matmul(representations, representations.T) / self.temperature
|
||
|
|
|
||
|
|
# Create positive mask (adjacent frames)
|
||
|
|
indices_expanded = temporal_indices.unsqueeze(0)
|
||
|
|
diff = torch.abs(indices_expanded - indices_expanded.T)
|
||
|
|
positive_mask = (diff == 1).float()
|
||
|
|
|
||
|
|
# Create negative mask (distant frames)
|
||
|
|
negative_mask = (diff > 2).float()
|
||
|
|
|
||
|
|
# Positive loss
|
||
|
|
pos_sim = sim_matrix * positive_mask
|
||
|
|
pos_loss = -torch.log(torch.exp(pos_sim) / torch.exp(sim_matrix).sum(dim=-1, keepdim=True) + 1e-8)
|
||
|
|
pos_loss = (pos_loss * positive_mask).sum() / (positive_mask.sum() + 1e-8)
|
||
|
|
|
||
|
|
# Negative loss (push apart)
|
||
|
|
neg_sim = sim_matrix * negative_mask
|
||
|
|
neg_loss = torch.relu(neg_sim - self.margin).mean()
|
||
|
|
|
||
|
|
return pos_loss + 0.1 * neg_loss
|
||
|
|
|
||
|
|
|
||
|
|
class MultiTaskLoss(nn.Module):
|
||
|
|
"""
|
||
|
|
Multi-task loss combining frame prediction and representation learning
|
||
|
|
"""
|
||
|
|
def __init__(self, frame_weight=1.0, contrastive_weight=0.1,
|
||
|
|
l1_weight=1.0, ssim_weight=0.1, use_contrastive=True):
|
||
|
|
super().__init__()
|
||
|
|
self.frame_weight = frame_weight
|
||
|
|
self.contrastive_weight = contrastive_weight
|
||
|
|
self.use_contrastive = use_contrastive
|
||
|
|
|
||
|
|
self.frame_loss = FramePredictionLoss(l1_weight=l1_weight, ssim_weight=ssim_weight)
|
||
|
|
if use_contrastive:
|
||
|
|
self.contrastive_loss = ContrastiveLoss()
|
||
|
|
|
||
|
|
def forward(self, pred_frame, target_frame, representations=None, temporal_indices=None):
|
||
|
|
"""
|
||
|
|
Args:
|
||
|
|
pred_frame: predicted frame [B, 3, H, W]
|
||
|
|
target_frame: target frame [B, 3, H, W]
|
||
|
|
representations: [B, D] representation vectors (optional)
|
||
|
|
temporal_indices: [B] temporal indices (optional)
|
||
|
|
Returns:
|
||
|
|
total_loss, loss_dict
|
||
|
|
"""
|
||
|
|
loss_dict = {}
|
||
|
|
|
||
|
|
# Frame prediction loss
|
||
|
|
frame_loss, frame_loss_dict = self.frame_loss(pred_frame, target_frame)
|
||
|
|
loss_dict.update({f'frame_{k}': v for k, v in frame_loss_dict.items()})
|
||
|
|
total_loss = self.frame_weight * frame_loss
|
||
|
|
|
||
|
|
# Contrastive loss (if representations provided)
|
||
|
|
if self.use_contrastive and representations is not None and temporal_indices is not None:
|
||
|
|
contrastive_loss = self.contrastive_loss(representations, temporal_indices)
|
||
|
|
loss_dict['contrastive'] = contrastive_loss
|
||
|
|
total_loss += self.contrastive_weight * contrastive_loss
|
||
|
|
|
||
|
|
loss_dict['total'] = total_loss
|
||
|
|
return total_loss, loss_dict
|