tijayantML's picture
Add flow and pixel IDM weights (Apache 2.0)
1d7ff26 verified
Raw History Blame Contribute Delete
9.42 kB
"""Pixel IDM: predict W/A/S/D/Shift and yaw/pitch per frame from a video.
Usage: python inference.py VIDEO [--weights model.safetensors] [--config config.json]
Pipeline: decode -> 128x128 RGB (squashed) -> ImageNet normalisation -> windows of
window_frames -> per-frame IMPALA CNN + temporal attention -> per-frame heads.
Overlapping windows are averaged. Output is JSON on stdout.
Needs: torch, av, opencv-python, safetensors, numpy.
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import av
import cv2
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from safetensors.torch import load_file
YAW_BIN_CENTERS = [-70, -2.2, -0.7, -0.28, -0.12, -0.04, 0, 0.04, 0.12, 0.28, 0.7, 2.2, 70]
PITCH_BIN_CENTERS = [-16, -0.35, -0.13, -0.06, -0.02, 0, 0.02, 0.06, 0.13, 0.35, 16]
IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
class ResBlock(nn.Module):
def __init__(self, ch: int, gn_groups: int) -> None:
super().__init__()
self.gn1 = nn.GroupNorm(gn_groups, ch)
self.conv1 = nn.Conv2d(ch, ch, 3, padding=1, bias=False)
self.gn2 = nn.GroupNorm(gn_groups, ch)
self.conv2 = nn.Conv2d(ch, ch, 3, padding=1, bias=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
out = self.conv1(F.relu(self.gn1(x)))
out = self.conv2(F.relu(self.gn2(out)))
return out + x
class ImpalaBlock(nn.Module):
def __init__(self, in_ch: int, out_ch: int, gn_groups: int) -> None:
super().__init__()
self.conv = nn.Conv2d(in_ch, out_ch, 3, padding=1)
self.pool = nn.MaxPool2d(3, stride=2, padding=1)
self.res1 = ResBlock(out_ch, gn_groups)
self.res2 = ResBlock(out_ch, gn_groups)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.res2(self.res1(self.pool(self.conv(x))))
class ImpalaCNN(nn.Module):
def __init__(self, channels: list[int], gn_groups: int, pool: int) -> None:
super().__init__()
chs = [3, *channels]
self.blocks = nn.Sequential(*[ImpalaBlock(chs[i], chs[i + 1], gn_groups) for i in range(len(channels))])
self.pool = nn.AdaptiveAvgPool2d(pool)
self.feat_dim = channels[-1] * pool * pool
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.pool(self.blocks(x)).flatten(1)
class PointwiseMLP(nn.Module):
def __init__(self, dim: int) -> None:
super().__init__()
self.norm = nn.LayerNorm(dim)
self.fc1 = nn.Linear(dim, dim * 4)
self.fc2 = nn.Linear(dim * 4, dim)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return x + self.fc2(F.gelu(self.fc1(self.norm(x))))
class ResidualAttentionBlock(nn.Module):
def __init__(self, dim: int, num_heads: int) -> None:
super().__init__()
self.norm = nn.LayerNorm(dim)
self.attn = nn.MultiheadAttention(dim, num_heads, batch_first=True)
self.mlp = PointwiseMLP(dim)
def forward(self, x: torch.Tensor) -> torch.Tensor:
n = self.norm(x)
x = x + self.attn(n, n, n, need_weights=False)[0]
return self.mlp(x)
def sub_head(feat_dim: int, hidden: int, dropout: float, out_dim: int) -> nn.Sequential:
return nn.Sequential(
nn.ReLU(), nn.Linear(feat_dim, hidden), nn.ReLU(inplace=True), nn.Dropout(dropout), nn.Linear(hidden, out_dim)
)
class HybridRotationHead(nn.Module):
"""Output (B, 26): yaw bin logits (13), yaw residual, pitch bin logits (11), pitch residual."""
def __init__(self, feat_dim: int, hidden: int, dropout: float) -> None:
super().__init__()
self.yaw_cls = sub_head(feat_dim, hidden, dropout, len(YAW_BIN_CENTERS))
self.yaw_res = sub_head(feat_dim, hidden, dropout, 1)
self.pitch_cls = sub_head(feat_dim, hidden, dropout, len(PITCH_BIN_CENTERS))
self.pitch_res = sub_head(feat_dim, hidden, dropout, 1)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return torch.cat([self.yaw_cls(x), self.yaw_res(x), self.pitch_cls(x), self.pitch_res(x)], dim=1)
class IDMImpalaVPT(nn.Module):
"""Per-frame IMPALA CNN -> temporal attention over the window -> per-frame heads."""
def __init__(self, cfg: dict) -> None:
super().__init__()
hid = cfg["temporal_hidden"]
self.cnn = ImpalaCNN(cfg["channels"], cfg["gn_groups"], cfg["spatial_pool_size"])
self.cnn_proj = nn.Linear(self.cnn.feat_dim, hid)
self.pre_temporal_norm = nn.LayerNorm(hid)
self.temporal_blocks = nn.Sequential(
*[ResidualAttentionBlock(hid, cfg["num_heads"]) for _ in range(cfg["temporal_layers"])]
)
self.post_temporal = nn.Sequential(nn.ReLU(), nn.Linear(hid, hid), nn.LayerNorm(hid))
self.key_head = sub_head(hid, cfg["head_hidden"], cfg["dropout"], cfg["num_keys"])
assert cfg["rotation_head_type"] == "hybrid"
self.rotation_head = HybridRotationHead(hid, cfg["head_hidden"], cfg["dropout"])
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""x: (B, N, 3, H, W) normalised frames -> key logits (B, N, 5), rotation raw (B, N, 26)."""
b, n = x.shape[:2]
f = self.cnn(x.flatten(0, 1)).view(b, n, -1)
f = self.post_temporal(self.temporal_blocks(self.pre_temporal_norm(self.cnn_proj(f))))
flat = f.reshape(b * n, -1)
return self.key_head(flat).view(b, n, -1), self.rotation_head(flat).view(b, n, -1)
def decode_hybrid_rotation(raw: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
"""(N, 26) -> (yaw_deg, pitch_deg): argmax bin centre plus residual."""
ny = len(YAW_BIN_CENTERS)
yaw = np.array(YAW_BIN_CENTERS, dtype=np.float32)[raw[:, :ny].argmax(1)] + raw[:, ny]
pitch_logits = raw[:, ny + 1 : ny + 1 + len(PITCH_BIN_CENTERS)]
pitch = np.array(PITCH_BIN_CENTERS, dtype=np.float32)[pitch_logits.argmax(1)] + raw[:, -1]
return yaw, pitch
def load_model(weights: Path, cfg: dict) -> IDMImpalaVPT:
model = IDMImpalaVPT(cfg)
model.load_state_dict(load_file(str(weights)), strict=True)
return model.eval()
def decode_frames(video: Path, resolution: int) -> tuple[np.ndarray, float]:
"""All frames as (N, res, res, 3) uint8 RGB, squashed to a square with INTER_AREA."""
frames: list[np.ndarray] = []
with av.open(str(video)) as container:
rate = container.streams.video[0].average_rate
fps = float(rate) if rate else 24.0
for frame in container.decode(video=0):
img = frame.to_ndarray(format="rgb24")
frames.append(cv2.resize(img, (resolution, resolution), interpolation=cv2.INTER_AREA))
assert frames, f"no frames decoded from {video}"
return np.stack(frames), fps
def generate_windows(total: int, size: int, stride: int, min_frames: int) -> list[tuple[int, int]]:
if total < min_frames:
return []
if total < size:
return [(0, total)]
windows, start = [], 0
while start < total:
end = min(start + size, total)
windows.append((start, end))
if end == total:
break
start += stride
return windows
@torch.no_grad()
def predict(video: Path, weights: Path, cfg: dict, device: torch.device) -> dict:
model = load_model(weights, cfg).to(device)
frames, fps = decode_frames(video, cfg["resolution"])
total = len(frames)
windows = generate_windows(total, cfg["window_frames"], cfg["window_stride"], cfg["min_frames"])
assert windows, f"video has {total} frames; need at least {cfg['min_frames']}"
norm = torch.from_numpy(((frames.astype(np.float32) / 255.0 - IMAGENET_MEAN) / IMAGENET_STD)).permute(0, 3, 1, 2)
dim = cfg["num_keys"] + 26
acc, count = np.zeros((total, dim), np.float64), np.zeros(total, np.float64)
for start, end in windows:
x = norm[start:end].unsqueeze(0).to(device)
key_logits, rot = model(x)
raw = torch.cat([torch.sigmoid(key_logits), rot], dim=-1)[0].cpu().numpy()
acc[start:end] += raw
count[start:end] += 1
acc /= count[:, None]
nk = cfg["num_keys"]
yaw, pitch = decode_hybrid_rotation(acc[:, nk:].astype(np.float32))
prob = acc[:, :nk]
rows = [
{
"frame": i,
"t_s": round(i / fps, 3),
"key_probabilities": dict(zip(cfg["key_order"], (round(float(p), 4) for p in prob[i]))),
"keys_pressed": [k for k, p in zip(cfg["key_order"], prob[i]) if p > cfg["key_threshold"]],
"yaw_deg": round(float(yaw[i]), 4),
"pitch_deg": round(float(pitch[i]), 4),
}
for i in range(total)
]
return {"video": str(video), "fps": fps, "num_frames": total, "frames": rows}
def main() -> None:
here = Path(__file__).parent
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("video", type=Path)
ap.add_argument("--weights", type=Path, default=here / "model.safetensors")
ap.add_argument("--config", type=Path, default=here / "config.json")
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
args = ap.parse_args()
cfg = json.loads(args.config.read_text())
print(json.dumps(predict(args.video, args.weights, cfg, torch.device(args.device)), indent=2))
if __name__ == "__main__":
main()