Video Analysis and Tracking

Introduction to Video Processing

Video adds the temporal dimension to vision, enabling understanding of motion, actions, and events over time. Video analysis tasks include object tracking, action recognition, video classification, and anomaly detection.

Key Differences from Images

• Temporal coherence - Adjacent frames are highly correlated
• Motion information - Optical flow reveals object movement
• Computational cost - Processing 30 FPS requires real-time optimization
• Memory requirements - Videos consume significantly more storage

Object Tracking

Object tracking maintains the identity of objects across video frames.

Tracking Algorithms

1. Mean Shift and CAMShift

import cv2

# Read video
cap = cv2.VideoCapture('video.mp4')

# Select ROI (Region of Interest)
ret, frame = cap.read()
bbox = cv2.selectROI(frame, False)

# Initialize tracker
tracker = cv2.TrackerCSRT_create()  # CSRT is most accurate
# Other options: TrackerKCF, TrackerMOSSE, TrackerBoosting
tracker.init(frame, bbox)

while True:
    ret, frame = cap.read()
    if not ret:
        break

    # Update tracker
    success, bbox = tracker.update(frame)

    if success:
        x, y, w, h = [int(v) for v in bbox]
        cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)

    cv2.imshow('Tracking', frame)
    if cv2.waitKey(30) & 0xFF == ord('q'):
        break

Deep Learning Tracking

DeepSORT

Combines SORT (Simple Online Realtime Tracking) with deep appearance features.

from deep_sort_realtime.deepsort_tracker import DeepSort

# Initialize DeepSORT
tracker = DeepSort(max_age=30, n_init=3, nms_max_overlap=1.0)

# Detection + Tracking loop
while True:
    ret, frame = cap.read()
    detections = detector.detect(frame)  # [[x1, y1, x2, y2, conf, cls], ...]

    # Convert to DeepSORT format
    det_list = []
    for det in detections:
        x1, y1, x2, y2, conf, cls = det
        det_list.append(([x1, y1, x2-x1, y2-y1], conf, cls))

    # Update tracker
    tracks = tracker.update_tracks(det_list, frame=frame)

    for track in tracks:
        if not track.is_confirmed():
            continue

        track_id = track.track_id
        ltrb = track.to_ltrb()
        x1, y1, x2, y2 = [int(v) for v in ltrb]

        cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 255, 0), 2)
        cv2.putText(frame, f"ID: {track_id}", (x1, y1-10),
                   cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)

ByteTrack

State-of-the-art multi-object tracking using low-confidence detections.

Optical Flow

Optical flow computes the motion vector field between consecutive frames.

Lucas-Kanade Method

import cv2
import numpy as np

cap = cv2.VideoCapture('video.mp4')
ret, old_frame = cap.read()
old_gray = cv2.cvtColor(old_frame, cv2.COLOR_BGR2GRAY)

# Detect good features to track
p0 = cv2.goodFeaturesToTrack(old_gray, mask=None,
                              maxCorners=100, qualityLevel=0.3,
                              minDistance=7, blockSize=7)

# Create mask for drawing
mask = np.zeros_like(old_frame)

while True:
    ret, frame = cap.read()
    if not ret:
        break

    frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Calculate optical flow
    p1, st, err = cv2.calcOpticalFlowPyrLK(old_gray, frame_gray,
                                           p0, None)

    # Select good points
    good_new = p1[st==1]
    good_old = p0[st==1]

    # Draw tracks
    for i, (new, old) in enumerate(zip(good_new, good_old)):
        a, b = new.ravel()
        c, d = old.ravel()
        mask = cv2.line(mask, (int(a), int(b)), (int(c), int(d)),
                       (0, 255, 0), 2)
        frame = cv2.circle(frame, (int(a), int(b)), 5, (0, 0, 255), -1)

    img = cv2.add(frame, mask)
    cv2.imshow('Optical Flow', img)

    # Update previous frame
    old_gray = frame_gray.copy()
    p0 = good_new.reshape(-1, 1, 2)

Farneback Dense Optical Flow

# Compute dense optical flow
flow = cv2.calcOpticalFlowFarneback(
    prev_gray, curr_gray, None,
    pyr_scale=0.5, levels=3, winsize=15,
    iterations=3, poly_n=5, poly_sigma=1.2,
    flags=0
)

# Visualize flow
mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
hsv = np.zeros_like(frame)
hsv[...,1] = 255
hsv[...,0] = ang * 180 / np.pi / 2
hsv[...,2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
flow_rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)

Action Recognition

Classify actions or activities in video clips.

3D Convolutional Networks

import torch
import torch.nn as nn

class C3D(nn.Module):
    """3D Convolutional Network for video understanding"""
    def __init__(self, num_classes=101):
        super(C3D, self).__init__()

        self.conv1 = nn.Conv3d(3, 64, kernel_size=3, padding=1)
        self.pool1 = nn.MaxPool3d(kernel_size=(1, 2, 2), stride=(1, 2, 2))

        self.conv2 = nn.Conv3d(64, 128, kernel_size=3, padding=1)
        self.pool2 = nn.MaxPool3d(kernel_size=2, stride=2)

        self.conv3a = nn.Conv3d(128, 256, kernel_size=3, padding=1)
        self.conv3b = nn.Conv3d(256, 256, kernel_size=3, padding=1)
        self.pool3 = nn.MaxPool3d(kernel_size=2, stride=2)

        self.fc1 = nn.Linear(256 * 4 * 7 * 7, 4096)
        self.fc2 = nn.Linear(4096, 4096)
        self.fc3 = nn.Linear(4096, num_classes)

        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0.5)

    def forward(self, x):
        # x shape: (batch, channels, depth, height, width)
        x = self.relu(self.conv1(x))
        x = self.pool1(x)

        x = self.relu(self.conv2(x))
        x = self.pool2(x)

        x = self.relu(self.conv3a(x))
        x = self.relu(self.conv3b(x))
        x = self.pool3(x)

        x = x.view(x.size(0), -1)

        x = self.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.relu(self.fc2(x))
        x = self.dropout(x)
        x = self.fc3(x)

        return x

Two-Stream Networks

Process spatial (RGB) and temporal (optical flow) information separately.

class TwoStreamNetwork(nn.Module):
    def __init__(self, num_classes):
        super().__init__()
        # Spatial stream (RGB frames)
        self.spatial_stream = models.resnet50(pretrained=True)
        self.spatial_stream.fc = nn.Linear(2048, num_classes)

        # Temporal stream (optical flow)
        self.temporal_stream = models.resnet50(pretrained=True)
        self.temporal_stream.conv1 = nn.Conv2d(
            20, 64, kernel_size=7, stride=2, padding=3, bias=False
        )  # 20 = 10 flow frames * 2 (x, y)
        self.temporal_stream.fc = nn.Linear(2048, num_classes)

    def forward(self, rgb, flow):
        spatial_out = self.spatial_stream(rgb)
        temporal_out = self.temporal_stream(flow)
        return (spatial_out + temporal_out) / 2  # Average fusion

SlowFast Networks

Dual pathway network processing at different frame rates.

Video Classification

from torchvision.models.video import r3d_18, mc3_18, r2plus1d_18

# Load pre-trained video classification model
model = r3d_18(pretrained=True)
model.eval()

# Prepare video clip (e.g., 16 frames)
video_tensor = torch.randn(1, 3, 16, 112, 112)
# Shape: (batch, channels, frames, height, width)

with torch.no_grad():
    output = model(video_tensor)

# Get predictions
probs = torch.softmax(output, dim=1)
top5 = torch.topk(probs, 5)

Temporal Action Detection

Detect when and where actions occur in untrimmed videos.

BMN (Boundary Matching Network)

Generates temporal action proposals with precise boundaries.

AFSD (Anchor-Free Single-Shot Detection)

Detects actions without predefined anchors.

Video Object Segmentation

Segment objects across all frames in a video.

STCN (Spatio-Temporal Correspondence Network)

Propagates segmentation masks across frames using memory networks.

Anomaly Detection in Video

Identify unusual events or behaviors in surveillance footage.

# Simple frame difference for motion detection
def detect_motion(prev_frame, curr_frame, threshold=30):
    # Convert to grayscale
    prev_gray = cv2.cvtColor(prev_frame, cv2.COLOR_BGR2GRAY)
    curr_gray = cv2.cvtColor(curr_frame, cv2.COLOR_BGR2GRAY)

    # Compute difference
    diff = cv2.absdiff(prev_gray, curr_gray)

    # Threshold
    _, thresh = cv2.threshold(diff, threshold, 255, cv2.THRESH_BINARY)

    # Find contours
    contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL,
                                   cv2.CHAIN_APPROX_SIMPLE)

    # Filter by area
    motion_detected = False
    for contour in contours:
        if cv2.contourArea(contour) > 500:
            motion_detected = True
            x, y, w, h = cv2.boundingRect(contour)
            cv2.rectangle(curr_frame, (x, y), (x+w, y+h), (0, 255, 0), 2)

    return motion_detected, curr_frame

Real-Time Video Processing

Optimization Techniques

• Frame skipping - Process every Nth frame
• Resolution reduction - Downscale frames before processing
• Model quantization - Use INT8 instead of FP32
• GPU acceleration - Leverage CUDA/TensorRT
• Batch processing - Process multiple frames simultaneously

# Efficient video processing
cap = cv2.VideoCapture('video.mp4')
fps = cap.get(cv2.CAP_PROP_FPS)
skip_frames = 2  # Process every 3rd frame

frame_count = 0
while True:
    ret, frame = cap.read()
    if not ret:
        break

    frame_count += 1
    if frame_count % skip_frames != 0:
        continue

    # Resize for faster processing
    frame_small = cv2.resize(frame, (640, 480))

    # Process frame
    result = model(frame_small)

    # Display result
    cv2.imshow('Video', result)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

Practical Applications

Surveillance Systems

• Multi-object tracking across cameras
• Activity recognition for security alerts
• Crowd counting and flow analysis

Sports Analytics

• Player tracking and performance metrics
• Ball tracking and trajectory prediction
• Action recognition for highlights generation

Healthcare

• Gait analysis for rehabilitation
• Fall detection for elderly care
• Surgical video analysis