Download pixel/inference.py from RekaAI/Reka-Inverse-Dynamics-Model: direct link, hf CLI and curl.
- Browser
- Download file 9.42 kB
-
https://huggingface.co/RekaAI/Reka-Inverse-Dynamics-Model/resolve/main/pixel/inference.py
- Command line
-
hf download hf://RekaAI/Reka-Inverse-Dynamics-Model/pixel/inference.py
-
curl -L -o inference.py https://huggingface.co/RekaAI/Reka-Inverse-Dynamics-Model/resolve/main/pixel/inference.py
9.42 kB
| """Pixel IDM: predict W/A/S/D/Shift and yaw/pitch per frame from a video. | |
| Usage: python inference.py VIDEO [--weights model.safetensors] [--config config.json] | |
| Pipeline: decode -> 128x128 RGB (squashed) -> ImageNet normalisation -> windows of | |
| window_frames -> per-frame IMPALA CNN + temporal attention -> per-frame heads. | |
| Overlapping windows are averaged. Output is JSON on stdout. | |
| Needs: torch, av, opencv-python, safetensors, numpy. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import av | |
| import cv2 | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from safetensors.torch import load_file | |
| YAW_BIN_CENTERS = [-70, -2.2, -0.7, -0.28, -0.12, -0.04, 0, 0.04, 0.12, 0.28, 0.7, 2.2, 70] | |
| PITCH_BIN_CENTERS = [-16, -0.35, -0.13, -0.06, -0.02, 0, 0.02, 0.06, 0.13, 0.35, 16] | |
| IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32) | |
| IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32) | |
| class ResBlock(nn.Module): | |
| def __init__(self, ch: int, gn_groups: int) -> None: | |
| super().__init__() | |
| self.gn1 = nn.GroupNorm(gn_groups, ch) | |
| self.conv1 = nn.Conv2d(ch, ch, 3, padding=1, bias=False) | |
| self.gn2 = nn.GroupNorm(gn_groups, ch) | |
| self.conv2 = nn.Conv2d(ch, ch, 3, padding=1, bias=False) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| out = self.conv1(F.relu(self.gn1(x))) | |
| out = self.conv2(F.relu(self.gn2(out))) | |
| return out + x | |
| class ImpalaBlock(nn.Module): | |
| def __init__(self, in_ch: int, out_ch: int, gn_groups: int) -> None: | |
| super().__init__() | |
| self.conv = nn.Conv2d(in_ch, out_ch, 3, padding=1) | |
| self.pool = nn.MaxPool2d(3, stride=2, padding=1) | |
| self.res1 = ResBlock(out_ch, gn_groups) | |
| self.res2 = ResBlock(out_ch, gn_groups) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return self.res2(self.res1(self.pool(self.conv(x)))) | |
| class ImpalaCNN(nn.Module): | |
| def __init__(self, channels: list[int], gn_groups: int, pool: int) -> None: | |
| super().__init__() | |
| chs = [3, *channels] | |
| self.blocks = nn.Sequential(*[ImpalaBlock(chs[i], chs[i + 1], gn_groups) for i in range(len(channels))]) | |
| self.pool = nn.AdaptiveAvgPool2d(pool) | |
| self.feat_dim = channels[-1] * pool * pool | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return self.pool(self.blocks(x)).flatten(1) | |
| class PointwiseMLP(nn.Module): | |
| def __init__(self, dim: int) -> None: | |
| super().__init__() | |
| self.norm = nn.LayerNorm(dim) | |
| self.fc1 = nn.Linear(dim, dim * 4) | |
| self.fc2 = nn.Linear(dim * 4, dim) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return x + self.fc2(F.gelu(self.fc1(self.norm(x)))) | |
| class ResidualAttentionBlock(nn.Module): | |
| def __init__(self, dim: int, num_heads: int) -> None: | |
| super().__init__() | |
| self.norm = nn.LayerNorm(dim) | |
| self.attn = nn.MultiheadAttention(dim, num_heads, batch_first=True) | |
| self.mlp = PointwiseMLP(dim) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| n = self.norm(x) | |
| x = x + self.attn(n, n, n, need_weights=False)[0] | |
| return self.mlp(x) | |
| def sub_head(feat_dim: int, hidden: int, dropout: float, out_dim: int) -> nn.Sequential: | |
| return nn.Sequential( | |
| nn.ReLU(), nn.Linear(feat_dim, hidden), nn.ReLU(inplace=True), nn.Dropout(dropout), nn.Linear(hidden, out_dim) | |
| ) | |
| class HybridRotationHead(nn.Module): | |
| """Output (B, 26): yaw bin logits (13), yaw residual, pitch bin logits (11), pitch residual.""" | |
| def __init__(self, feat_dim: int, hidden: int, dropout: float) -> None: | |
| super().__init__() | |
| self.yaw_cls = sub_head(feat_dim, hidden, dropout, len(YAW_BIN_CENTERS)) | |
| self.yaw_res = sub_head(feat_dim, hidden, dropout, 1) | |
| self.pitch_cls = sub_head(feat_dim, hidden, dropout, len(PITCH_BIN_CENTERS)) | |
| self.pitch_res = sub_head(feat_dim, hidden, dropout, 1) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return torch.cat([self.yaw_cls(x), self.yaw_res(x), self.pitch_cls(x), self.pitch_res(x)], dim=1) | |
| class IDMImpalaVPT(nn.Module): | |
| """Per-frame IMPALA CNN -> temporal attention over the window -> per-frame heads.""" | |
| def __init__(self, cfg: dict) -> None: | |
| super().__init__() | |
| hid = cfg["temporal_hidden"] | |
| self.cnn = ImpalaCNN(cfg["channels"], cfg["gn_groups"], cfg["spatial_pool_size"]) | |
| self.cnn_proj = nn.Linear(self.cnn.feat_dim, hid) | |
| self.pre_temporal_norm = nn.LayerNorm(hid) | |
| self.temporal_blocks = nn.Sequential( | |
| *[ResidualAttentionBlock(hid, cfg["num_heads"]) for _ in range(cfg["temporal_layers"])] | |
| ) | |
| self.post_temporal = nn.Sequential(nn.ReLU(), nn.Linear(hid, hid), nn.LayerNorm(hid)) | |
| self.key_head = sub_head(hid, cfg["head_hidden"], cfg["dropout"], cfg["num_keys"]) | |
| assert cfg["rotation_head_type"] == "hybrid" | |
| self.rotation_head = HybridRotationHead(hid, cfg["head_hidden"], cfg["dropout"]) | |
| def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: | |
| """x: (B, N, 3, H, W) normalised frames -> key logits (B, N, 5), rotation raw (B, N, 26).""" | |
| b, n = x.shape[:2] | |
| f = self.cnn(x.flatten(0, 1)).view(b, n, -1) | |
| f = self.post_temporal(self.temporal_blocks(self.pre_temporal_norm(self.cnn_proj(f)))) | |
| flat = f.reshape(b * n, -1) | |
| return self.key_head(flat).view(b, n, -1), self.rotation_head(flat).view(b, n, -1) | |
| def decode_hybrid_rotation(raw: np.ndarray) -> tuple[np.ndarray, np.ndarray]: | |
| """(N, 26) -> (yaw_deg, pitch_deg): argmax bin centre plus residual.""" | |
| ny = len(YAW_BIN_CENTERS) | |
| yaw = np.array(YAW_BIN_CENTERS, dtype=np.float32)[raw[:, :ny].argmax(1)] + raw[:, ny] | |
| pitch_logits = raw[:, ny + 1 : ny + 1 + len(PITCH_BIN_CENTERS)] | |
| pitch = np.array(PITCH_BIN_CENTERS, dtype=np.float32)[pitch_logits.argmax(1)] + raw[:, -1] | |
| return yaw, pitch | |
| def load_model(weights: Path, cfg: dict) -> IDMImpalaVPT: | |
| model = IDMImpalaVPT(cfg) | |
| model.load_state_dict(load_file(str(weights)), strict=True) | |
| return model.eval() | |
| def decode_frames(video: Path, resolution: int) -> tuple[np.ndarray, float]: | |
| """All frames as (N, res, res, 3) uint8 RGB, squashed to a square with INTER_AREA.""" | |
| frames: list[np.ndarray] = [] | |
| with av.open(str(video)) as container: | |
| rate = container.streams.video[0].average_rate | |
| fps = float(rate) if rate else 24.0 | |
| for frame in container.decode(video=0): | |
| img = frame.to_ndarray(format="rgb24") | |
| frames.append(cv2.resize(img, (resolution, resolution), interpolation=cv2.INTER_AREA)) | |
| assert frames, f"no frames decoded from {video}" | |
| return np.stack(frames), fps | |
| def generate_windows(total: int, size: int, stride: int, min_frames: int) -> list[tuple[int, int]]: | |
| if total < min_frames: | |
| return [] | |
| if total < size: | |
| return [(0, total)] | |
| windows, start = [], 0 | |
| while start < total: | |
| end = min(start + size, total) | |
| windows.append((start, end)) | |
| if end == total: | |
| break | |
| start += stride | |
| return windows | |
| def predict(video: Path, weights: Path, cfg: dict, device: torch.device) -> dict: | |
| model = load_model(weights, cfg).to(device) | |
| frames, fps = decode_frames(video, cfg["resolution"]) | |
| total = len(frames) | |
| windows = generate_windows(total, cfg["window_frames"], cfg["window_stride"], cfg["min_frames"]) | |
| assert windows, f"video has {total} frames; need at least {cfg['min_frames']}" | |
| norm = torch.from_numpy(((frames.astype(np.float32) / 255.0 - IMAGENET_MEAN) / IMAGENET_STD)).permute(0, 3, 1, 2) | |
| dim = cfg["num_keys"] + 26 | |
| acc, count = np.zeros((total, dim), np.float64), np.zeros(total, np.float64) | |
| for start, end in windows: | |
| x = norm[start:end].unsqueeze(0).to(device) | |
| key_logits, rot = model(x) | |
| raw = torch.cat([torch.sigmoid(key_logits), rot], dim=-1)[0].cpu().numpy() | |
| acc[start:end] += raw | |
| count[start:end] += 1 | |
| acc /= count[:, None] | |
| nk = cfg["num_keys"] | |
| yaw, pitch = decode_hybrid_rotation(acc[:, nk:].astype(np.float32)) | |
| prob = acc[:, :nk] | |
| rows = [ | |
| { | |
| "frame": i, | |
| "t_s": round(i / fps, 3), | |
| "key_probabilities": dict(zip(cfg["key_order"], (round(float(p), 4) for p in prob[i]))), | |
| "keys_pressed": [k for k, p in zip(cfg["key_order"], prob[i]) if p > cfg["key_threshold"]], | |
| "yaw_deg": round(float(yaw[i]), 4), | |
| "pitch_deg": round(float(pitch[i]), 4), | |
| } | |
| for i in range(total) | |
| ] | |
| return {"video": str(video), "fps": fps, "num_frames": total, "frames": rows} | |
| def main() -> None: | |
| here = Path(__file__).parent | |
| ap = argparse.ArgumentParser(description=__doc__) | |
| ap.add_argument("video", type=Path) | |
| ap.add_argument("--weights", type=Path, default=here / "model.safetensors") | |
| ap.add_argument("--config", type=Path, default=here / "config.json") | |
| ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") | |
| args = ap.parse_args() | |
| cfg = json.loads(args.config.read_text()) | |
| print(json.dumps(predict(args.video, args.weights, cfg, torch.device(args.device)), indent=2)) | |
| if __name__ == "__main__": | |
| main() | |