3D Vision and Depth Estimation

Introduction to 3D Vision

3D vision enables computers to understand the three-dimensional structure of the world from 2D images. This is crucial for robotics, autonomous vehicles, AR/VR, and scene understanding.

Key Concepts

• Depth - Distance from camera to objects
• Disparity - Pixel offset between stereo image pairs
• Point cloud - 3D representation as collection of points
• Mesh - Surface representation with vertices and faces
• Voxels - 3D pixels representing volumetric data

Stereo Vision

Stereo vision uses two cameras to triangulate depth, similar to human binocular vision.

Stereo Calibration

import cv2
import numpy as np

# Prepare object points (e.g., chessboard corners)
objp = np.zeros((6*9, 3), np.float32)
objp[:,:2] = np.mgrid[0:9, 0:6].T.reshape(-1, 2)

# Arrays to store points
objpoints = []  # 3D points in real world
imgpoints_left = []  # 2D points in left image
imgpoints_right = []  # 2D points in right image

# Find corners in calibration images
for left_img, right_img in calibration_images:
    gray_left = cv2.cvtColor(left_img, cv2.COLOR_BGR2GRAY)
    gray_right = cv2.cvtColor(right_img, cv2.COLOR_BGR2GRAY)

    ret_left, corners_left = cv2.findChessboardCorners(gray_left, (9,6))
    ret_right, corners_right = cv2.findChessboardCorners(gray_right, (9,6))

    if ret_left and ret_right:
        objpoints.append(objp)
        imgpoints_left.append(corners_left)
        imgpoints_right.append(corners_right)

# Stereo calibration
retval, cameraMatrix1, distCoeffs1, cameraMatrix2, distCoeffs2, R, T, E, F = \
    cv2.stereoCalibrate(objpoints, imgpoints_left, imgpoints_right,
                        None, None, None, None, gray_left.shape[::-1])

# Stereo rectification
R1, R2, P1, P2, Q, roi1, roi2 = cv2.stereoRectify(
    cameraMatrix1, distCoeffs1, cameraMatrix2, distCoeffs2,
    gray_left.shape[::-1], R, T
)

Disparity Map Computation

# Create StereoBM (Block Matching) or StereoSGBM
stereo = cv2.StereoSGBM_create(
    minDisparity=0,
    numDisparities=16*10,  # Max disparity (must be divisible by 16)
    blockSize=5,
    P1=8 * 3 * 5**2,
    P2=32 * 3 * 5**2,
    disp12MaxDiff=1,
    uniquenessRatio=10,
    speckleWindowSize=100,
    speckleRange=32
)

# Compute disparity
disparity = stereo.compute(left_gray, right_gray).astype(np.float32) / 16.0

# Compute depth from disparity
# depth = (focal_length * baseline) / disparity
focal_length = cameraMatrix1[0, 0]
baseline = np.linalg.norm(T)
depth_map = (focal_length * baseline) / (disparity + 1e-6)

Monocular Depth Estimation

Estimate depth from a single image using deep learning.

MiDaS (Towards Robust Monocular Depth Estimation)

import torch
import cv2
from torchvision.transforms import Compose

# Load MiDaS model
model_type = "DPT_Large"  # or "DPT_Hybrid", "MiDaS_small"
midas = torch.hub.load("intel-isl/MiDaS", model_type)

device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
midas.to(device)
midas.eval()

# Load transforms
midas_transforms = torch.hub.load("intel-isl/MiDaS", "transforms")
transform = midas_transforms.dpt_transform

# Load image
img = cv2.imread("image.jpg")
img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

# Prepare input
input_batch = transform(img_rgb).to(device)

# Predict depth
with torch.no_grad():
    prediction = midas(input_batch)
    prediction = torch.nn.functional.interpolate(
        prediction.unsqueeze(1),
        size=img_rgb.shape[:2],
        mode="bicubic",
        align_corners=False,
    ).squeeze()

depth_map = prediction.cpu().numpy()

# Normalize for visualization
depth_min = depth_map.min()
depth_max = depth_map.max()
depth_normalized = (depth_map - depth_min) / (depth_max - depth_min)
depth_colored = cv2.applyColorMap((depth_normalized * 255).astype(np.uint8),
                                  cv2.COLORMAP_MAGMA)

ZoeDepth

Zero-shot transfer for metric depth estimation.

Depth Anything

Foundation model for monocular depth estimation with strong generalization.

Structure from Motion (SfM)

Reconstruct 3D structure from multiple images taken from different viewpoints.

COLMAP

# COLMAP pipeline for 3D reconstruction
# 1. Feature extraction
colmap feature_extractor \
    --database_path database.db \
    --image_path images/

# 2. Feature matching
colmap exhaustive_matcher \
    --database_path database.db

# 3. Sparse reconstruction
colmap mapper \
    --database_path database.db \
    --image_path images/ \
    --output_path sparse/

# 4. Dense reconstruction
colmap image_undistorter \
    --image_path images/ \
    --input_path sparse/0 \
    --output_path dense/

colmap patch_match_stereo \
    --workspace_path dense/

colmap stereo_fusion \
    --workspace_path dense/ \
    --output_path dense/fused.ply

Point Cloud Processing

Loading and Visualizing Point Clouds

import open3d as o3d

# Load point cloud
pcd = o3d.io.read_point_cloud("pointcloud.ply")

# Visualize
o3d.visualization.draw_geometries([pcd])

# Get point cloud statistics
print(f"Number of points: {len(pcd.points)}")
print(f"Point cloud center: {pcd.get_center()}")
print(f"Bounding box: {pcd.get_axis_aligned_bounding_box()}")

# Downsample point cloud
pcd_down = pcd.voxel_down_sample(voxel_size=0.05)

# Estimate normals
pcd.estimate_normals(
    search_param=o3d.geometry.KDTreeSearchParamHybrid(
        radius=0.1, max_nn=30
    )
)

# Remove outliers
cl, ind = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=2.0)
pcd_clean = pcd.select_by_index(ind)

Point Cloud Registration

# ICP (Iterative Closest Point) registration
def register_point_clouds(source, target):
    threshold = 0.02
    trans_init = np.identity(4)

    # Coarse registration with RANSAC
    ransac_result = o3d.pipelines.registration.registration_ransac_based_on_feature_matching(
        source, target,
        source_features, target_features,
        True, threshold,
        o3d.pipelines.registration.TransformationEstimationPointToPoint(False),
        3,
        [o3d.pipelines.registration.CorrespondenceCheckerBasedOnEdgeLength(0.9),
         o3d.pipelines.registration.CorrespondenceCheckerBasedOnDistance(threshold)],
        o3d.pipelines.registration.RANSACConvergenceCriteria(100000, 0.999)
    )

    # Fine registration with ICP
    icp_result = o3d.pipelines.registration.registration_icp(
        source, target, threshold, ransac_result.transformation,
        o3d.pipelines.registration.TransformationEstimationPointToPoint()
    )

    return icp_result.transformation

3D Object Detection

PointNet

Deep learning on point clouds for classification and segmentation.

import torch
import torch.nn as nn

class PointNet(nn.Module):
    def __init__(self, num_classes):
        super(PointNet, self).__init__()

        # Shared MLP
        self.conv1 = nn.Conv1d(3, 64, 1)
        self.conv2 = nn.Conv1d(64, 128, 1)
        self.conv3 = nn.Conv1d(128, 1024, 1)

        # Classification head
        self.fc1 = nn.Linear(1024, 512)
        self.fc2 = nn.Linear(512, 256)
        self.fc3 = nn.Linear(256, num_classes)

        self.bn1 = nn.BatchNorm1d(64)
        self.bn2 = nn.BatchNorm1d(128)
        self.bn3 = nn.BatchNorm1d(1024)
        self.bn4 = nn.BatchNorm1d(512)
        self.bn5 = nn.BatchNorm1d(256)

        self.dropout = nn.Dropout(0.3)

    def forward(self, x):
        # x: (batch, 3, num_points)
        x = F.relu(self.bn1(self.conv1(x)))
        x = F.relu(self.bn2(self.conv2(x)))
        x = F.relu(self.bn3(self.conv3(x)))

        # Max pooling for global feature
        x = torch.max(x, 2, keepdim=True)[0]
        x = x.view(-1, 1024)

        # Classification
        x = F.relu(self.bn4(self.fc1(x)))
        x = F.relu(self.bn5(self.dropout(self.fc2(x))))
        x = self.fc3(x)

        return x

VoxelNet and SECOND

Voxel-based 3D detection for autonomous driving.

Neural Radiance Fields (NeRF)

Novel view synthesis by representing scenes as continuous 5D functions.

# NeRF concept (simplified)
class NeRF(nn.Module):
    def __init__(self, D=8, W=256):
        super(NeRF, self).__init__()

        # Position encoding
        self.pos_encoder = PositionalEncoder()

        # MLP layers
        self.layers = nn.ModuleList()
        for i in range(D):
            if i == 0:
                self.layers.append(nn.Linear(63, W))  # 3*10*2 + 3
            elif i == D//2:
                self.layers.append(nn.Linear(W + 63, W))
            else:
                self.layers.append(nn.Linear(W, W))

        self.rgb_layer = nn.Linear(W, 3)
        self.density_layer = nn.Linear(W, 1)

    def forward(self, x, d):
        # x: position (batch, 3)
        # d: viewing direction (batch, 3)

        x = self.pos_encoder(x)
        h = x

        for i, layer in enumerate(self.layers):
            h = layer(h)
            h = F.relu(h)
            if i == len(self.layers) // 2:
                h = torch.cat([h, x], -1)

        density = F.relu(self.density_layer(h))
        rgb = torch.sigmoid(self.rgb_layer(h))

        return rgb, density

3D Reconstruction from Images

Multi-View Stereo (MVS)

Dense reconstruction from calibrated images.

Photogrammetry

Create 3D models from photographs.

Depth Sensors

RGB-D Cameras

• Intel RealSense - Active stereo depth sensing
• Microsoft Kinect - Time-of-flight (ToF) sensing
• LiDAR - Laser-based ranging

import pyrealsense2 as rs
import numpy as np

# Configure RealSense pipeline
pipeline = rs.pipeline()
config = rs.config()
config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)
config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30)

# Start streaming
pipeline.start(config)

try:
    while True:
        # Wait for frames
        frames = pipeline.wait_for_frames()
        depth_frame = frames.get_depth_frame()
        color_frame = frames.get_color_frame()

        if not depth_frame or not color_frame:
            continue

        # Convert to numpy arrays
        depth_image = np.asanyarray(depth_frame.get_data())
        color_image = np.asanyarray(color_frame.get_data())

        # Process depth (in millimeters)
        depth_colormap = cv2.applyColorMap(
            cv2.convertScaleAbs(depth_image, alpha=0.03),
            cv2.COLORMAP_JET
        )

        # Stack images side by side
        images = np.hstack((color_image, depth_colormap))
        cv2.imshow('RealSense', images)

        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
finally:
    pipeline.stop()

Applications

Autonomous Vehicles

• 3D object detection and tracking
• Obstacle avoidance
• Road surface estimation

Augmented Reality

• Depth-aware occlusion
• Scene understanding for object placement
• Real-time SLAM (Simultaneous Localization and Mapping)

Robotics

• Grasp planning
• Navigation and obstacle avoidance
• 3D scene reconstruction