Shortcuts

Source code for mmaction.models.utils.graph

# Copyright (c) OpenMMLab. All rights reserved.
from typing import List, Tuple, Union

import numpy as np
import torch


def k_adjacency(A: Union[torch.Tensor, np.ndarray],
                k: int,
                with_self: bool = False,
                self_factor: float = 1) -> np.ndarray:
    """Construct k-adjacency matrix.

    Args:
        A (torch.Tensor or np.ndarray): The adjacency matrix.
        k (int): The number of hops.
        with_self (bool): Whether to add self-loops to the
            k-adjacency matrix. The self-loops is critical
            for learning the relationships between the current
            joint and its k-hop neighbors. Defaults to False.
        self_factor (float): The scale factor to the added
            identity matrix. Defaults to 1.

    Returns:
        np.ndarray: The k-adjacency matrix.
    """
    # A is a 2D square array
    if isinstance(A, torch.Tensor):
        A = A.data.cpu().numpy()
    assert isinstance(A, np.ndarray)
    Iden = np.eye(len(A), dtype=A.dtype)
    if k == 0:
        return Iden
    Ak = np.minimum(np.linalg.matrix_power(A + Iden, k), 1) - np.minimum(
        np.linalg.matrix_power(A + Iden, k - 1), 1)
    if with_self:
        Ak += (self_factor * Iden)
    return Ak


def edge2mat(edges: List[Tuple[int, int]], num_node: int) -> np.ndarray:
    """Get adjacency matrix from edges.

    Args:
        edges (list[tuple[int, int]]): The edges of the graph.
        num_node (int): The number of nodes of the graph.

    Returns:
        np.ndarray: The adjacency matrix.
    """
    A = np.zeros((num_node, num_node))
    for i, j in edges:
        A[j, i] = 1
    return A


def normalize_digraph(A: np.ndarray, dim: int = 0) -> np.ndarray:
    """Normalize the digraph according to the given dimension.

    Args:
        A (np.ndarray): The adjacency matrix.
        dim (int): The dimension to perform normalization.
            Defaults to 0.

    Returns:
        np.ndarray: The normalized adjacency matrix.
    """
    # A is a 2D square array
    Dl = np.sum(A, dim)
    h, w = A.shape
    Dn = np.zeros((w, w))

    for i in range(w):
        if Dl[i] > 0:
            Dn[i, i] = Dl[i]**(-1)

    AD = np.dot(A, Dn)
    return AD


def get_hop_distance(num_node: int,
                     edges: List[Tuple[int, int]],
                     max_hop: int = 1) -> np.ndarray:
    """Get n-hop distance matrix by edges.

    Args:
        num_node (int): The number of nodes of the graph.
        edges (list[tuple[int, int]]): The edges of the graph.
        max_hop (int): The maximal distance between two connected nodes.
            Defaults to 1.

    Returns:
        np.ndarray: The n-hop distance matrix.
    """
    A = np.eye(num_node)

    for i, j in edges:
        A[i, j] = 1
        A[j, i] = 1

    # compute hop steps
    hop_dis = np.zeros((num_node, num_node)) + np.inf
    transfer_mat = [np.linalg.matrix_power(A, d) for d in range(max_hop + 1)]
    arrive_mat = (np.stack(transfer_mat) > 0)
    for d in range(max_hop, -1, -1):
        hop_dis[arrive_mat[d]] = d
    return hop_dis


[docs]class Graph: """The Graph to model the skeletons. Args: layout (str or dict): must be one of the following candidates: 'openpose', 'nturgb+d', 'coco', or a dict with the following keys: 'num_node', 'inward', and 'center'. Defaults to ``'coco'``. mode (str): must be one of the following candidates: 'stgcn_spatial', 'spatial'. Defaults to ``'spatial'``. max_hop (int): the maximal distance between two connected nodes. Defaults to 1. """ def __init__(self, layout: Union[str, dict] = 'coco', mode: str = 'spatial', max_hop: int = 1) -> None: self.max_hop = max_hop self.layout = layout self.mode = mode if isinstance(layout, dict): assert 'num_node' in layout assert 'inward' in layout assert 'center' in layout else: assert layout in ['openpose', 'nturgb+d', 'coco'] self.set_layout(layout) self.hop_dis = get_hop_distance(self.num_node, self.inward, max_hop) assert hasattr(self, mode), f'Do Not Exist This Mode: {mode}' self.A = getattr(self, mode)() def __str__(self): return self.A
[docs] def set_layout(self, layout: str) -> None: """Initialize the layout of candidates.""" if layout == 'openpose': self.num_node = 18 self.inward = [(4, 3), (3, 2), (7, 6), (6, 5), (13, 12), (12, 11), (10, 9), (9, 8), (11, 5), (8, 2), (5, 1), (2, 1), (0, 1), (15, 0), (14, 0), (17, 15), (16, 14)] self.center = 1 elif layout == 'nturgb+d': self.num_node = 25 neighbor_base = [(1, 2), (2, 21), (3, 21), (4, 3), (5, 21), (6, 5), (7, 6), (8, 7), (9, 21), (10, 9), (11, 10), (12, 11), (13, 1), (14, 13), (15, 14), (16, 15), (17, 1), (18, 17), (19, 18), (20, 19), (22, 8), (23, 8), (24, 12), (25, 12)] self.inward = [(i - 1, j - 1) for (i, j) in neighbor_base] self.center = 21 - 1 elif layout == 'coco': self.num_node = 17 self.inward = [(15, 13), (13, 11), (16, 14), (14, 12), (11, 5), (12, 6), (9, 7), (7, 5), (10, 8), (8, 6), (5, 0), (6, 0), (1, 0), (3, 1), (2, 0), (4, 2)] self.center = 0 elif isinstance(layout, dict): self.num_node = layout['num_node'] self.inward = layout['inward'] self.center = layout['center'] else: raise ValueError(f'Do Not Exist This Layout: {layout}') self.self_link = [(i, i) for i in range(self.num_node)] self.outward = [(j, i) for (i, j) in self.inward] self.neighbor = self.inward + self.outward
[docs] def stgcn_spatial(self) -> np.ndarray: """ST-GCN spatial mode.""" adj = np.zeros((self.num_node, self.num_node)) adj[self.hop_dis <= self.max_hop] = 1 normalize_adj = normalize_digraph(adj) hop_dis = self.hop_dis center = self.center A = [] for hop in range(self.max_hop + 1): a_close = np.zeros((self.num_node, self.num_node)) a_further = np.zeros((self.num_node, self.num_node)) for i in range(self.num_node): for j in range(self.num_node): if hop_dis[j, i] == hop: if hop_dis[j, center] >= hop_dis[i, center]: a_close[j, i] = normalize_adj[j, i] else: a_further[j, i] = normalize_adj[j, i] A.append(a_close) if hop > 0: A.append(a_further) return np.stack(A)
[docs] def spatial(self) -> np.ndarray: """Standard spatial mode.""" Iden = edge2mat(self.self_link, self.num_node) In = normalize_digraph(edge2mat(self.inward, self.num_node)) Out = normalize_digraph(edge2mat(self.outward, self.num_node)) A = np.stack((Iden, In, Out)) return A
[docs] def binary_adj(self) -> np.ndarray: """Construct an adjacency matrix for an undirected graph.""" A = edge2mat(self.neighbor, self.num_node) return A[None]