Add resonator network and sparse distributed memory (#123)

mikeheddes · github-actions[bot] · web-flow · commit e9c0b485388f · 2023-02-22T22:25:36.000-08:00
* Implement resonator network and sparse distributed memory

* [github-action] formatting fixes

* Add memory and models modules to conda

* Add missing scipy dependency

---------

Co-authored-by: github-actions[bot] &lt;github-actions[bot]@users.noreply.github.com&gt;
diff --git a/conda/meta.yaml b/conda/meta.yaml
@@ -20,6 +20,7 @@ requirements:
   run:
     - python>=3.6
     - pytorch
+    - scipy
     - pandas
     - requests
     - tqdm
@@ -30,6 +31,8 @@ test:
     - torchhd.functional
     - torchhd.embeddings
     - torchhd.structures
+    - torchhd.models
+    - torchhd.memory
     - torchhd.datasets
 
 about:
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -1,5 +1,6 @@
 torch>=1.9.0
 torchvision
+scipy
 pandas
 requests
 numpy
diff --git a/docs/index.rst b/docs/index.rst
@@ -1,7 +1,7 @@
 Welcome to the Torchhd documentation!
 =====================================
 
-Torchhd is a Python library dedicated to *Hyperdimensional Computing* (also known as *Vector Symbolic Architectures*).
+Torchhd is a Python library dedicated to *Hyperdimensional Computing* also known as *Vector Symbolic Architectures*.
 
 .. toctree::
    :glob:
@@ -19,6 +19,7 @@ Torchhd is a Python library dedicated to *Hyperdimensional Computing* (also know
    embeddings
    structures
    models
+   memory
    datasets
    utils
 
diff --git a/docs/memory.rst b/docs/memory.rst
@@ -0,0 +1,12 @@
+.. _memory:
+
+torchhd.memory
+==================
+
+.. currentmodule:: torchhd.memory
+
+.. autosummary::
+    :toctree: generated/
+    :template: class.rst
+
+    SDM
diff --git a/docs/models.rst b/docs/models.rst
@@ -10,4 +10,3 @@ torchhd.models
     :template: class.rst
 
     Centroid
-    
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -1,6 +1,7 @@
 torch
 torchvision
 pandas
+scipy
 requests
 tqdm
 numpy
diff --git a/docs/torchhd.rst b/docs/torchhd.rst
@@ -38,6 +38,7 @@ Operations
     randsel
     multirandsel
     create_random_permute
+    resonator
     soft_quantize
     hard_quantize
 
diff --git a/setup.py b/setup.py
@@ -20,6 +20,7 @@
     license="MIT",
     install_requires=[
         "torch>=1.9.0",
+        "scipy",
         "pandas",
         "numpy",
         "requests",
diff --git a/torchhd/__init__.py b/torchhd/__init__.py
@@ -2,6 +2,7 @@
 import torchhd.embeddings as embeddings
 import torchhd.structures as structures
 import torchhd.models as models
+import torchhd.memory as memory
 import torchhd.datasets as datasets
 import torchhd.utils as utils
 
@@ -42,6 +43,7 @@
     cross_product,
     ngrams,
     graph,
+    resonator,
     map_range,
     value_to_index,
     index_to_value,
@@ -60,6 +62,7 @@
     "embeddings",
     "structures",
     "models",
+    "memory",
     "datasets",
     "utils",
     "as_vsa_model",
@@ -92,6 +95,7 @@
     "cross_product",
     "ngrams",
     "graph",
+    "resonator",
     "map_range",
     "value_to_index",
     "index_to_value",
diff --git a/torchhd/datasets/ccpp.py b/torchhd/datasets/ccpp.py
@@ -12,6 +12,20 @@ class CyclePowerPlant(data.Dataset):
     """`Combined cycle power planet <https://archive.ics.uci.edu/ml/datasets/combined+cycle+power+plant>`_ dataset.
         Features consist of hourly average ambient variables Temperature (T), Ambient Pressure (AP), Relative Humidity (RH) and Exhaust Vacuum (V) to predict the net hourly electrical energy output (EP) of the plant.
 
+    .. list-table::
+       :widths: 10 10 10 10
+       :align: center
+       :header-rows: 1
+
+       * - Instances
+         - Attributes
+         - Task
+         - Area
+       * - 9568
+         - 4
+         - Regression
+         - Computer
+
     Args:
         root (string): Root directory of dataset where downloaded dataset exists
         download (bool, optional): If True, downloads the dataset from the internet and
diff --git a/torchhd/functional.py b/torchhd/functional.py
@@ -1,5 +1,5 @@
 import math
-from typing import Type, Union, Callable
+from typing import Type, Callable
 import torch
 from torch import LongTensor, FloatTensor, Tensor
 from collections import deque
@@ -39,6 +39,7 @@
     "ngrams",
     "hash_table",
     "graph",
+    "resonator",
     "map_range",
     "value_to_index",
     "index_to_value",
@@ -1394,6 +1395,97 @@ def cleanup(input: VSA_Model, memory: VSA_Model, threshold=0.0) -> VSA_Model:
     return torch.index_select(memory, -2, index)
 
 
+def resonator(input: VSA_Model, estimates: VSA_Model, domains: VSA_Model) -> VSA_Model:
+    """A step of the resonator network that factorizes the input.
+
+    Given current estimates for each factor, it returns the next estimates for those factors.
+
+    Shapes:
+        - Input: :math:`(*, d)`
+        - Estimates: :math:`(*, n, d)`
+        - Domains: :math:`(*, n, m, d)`
+        - Output: :math:`(*, n, d)`
+
+    Examples::
+
+        >>> X = torchhd.random_hv(5, 100)
+        >>> Y = torchhd.random_hv(5, 100)
+        >>> Z = torchhd.random_hv(5, 100)
+        >>> domains = torch.stack((X, Y, Z), dim=0)
+        >>> domains.shape
+        torch.Size([3, 5, 100])
+        >>> x_hat = torchhd.multiset(X)
+        >>> y_hat = torchhd.multiset(Y)
+        >>> z_hat = torchhd.multiset(Z)
+        >>> estimates = torch.stack((x_hat, y_hat, z_hat), dim=0)
+        >>> estimates.shape
+        torch.Size([3, 100])
+        >>> # look at similarity of estimates with the domain
+        >>> print(torchhd.dot_similarity(estimates.unsqueeze(-2), domains).squeeze(-2))
+        MAP([[112.,  80., 136., 106., 106.],
+            [ 98., 102., 100., 110.,  74.],
+            [116.,  94., 104., 112.,  82.]])
+        >>> # Create the combined symbol
+        >>> s = X[0].bind(Y[1]).bind(Z[3])
+        MAP([[100.,   8.,   8.,  -2.,  -2.],
+            [-18.,  70.,  44.,  -6.,  14.],
+            [  8.,   2.,   4., 100.,  -2.]])
+        >>> # resonator step
+        >>> estimates = torchhd.resonator(s, estimates, domains)
+        >>> # look at similarity of new estimates with the domain
+        >>> print(torchhd.dot_similarity(estimates.unsqueeze(-2), domains).squeeze(-2))
+        MAP([[100.,   8.,   8.,  -2.,  -2.],
+            [  4., 100.,  14.,   0., -16.],
+            [  8.,   2.,   4., 100.,  -2.]])
+
+    """
+    input = as_vsa_model(input)
+    estimates = as_vsa_model(estimates)
+    domains = as_vsa_model(domains)
+
+    if not isinstance(input, MAP):
+        raise ValueError(
+            f"Resonator currently only supports Multiply-Add-Permute (MAP) VSA model, provided: {input.__class__.__name__}"
+        )
+
+    if not isinstance(estimates, MAP):
+        raise ValueError(
+            f"Resonator currently only supports Multiply-Add-Permute (MAP) VSA model, provided: {estimates.__class__.__name__}"
+        )
+
+    if not isinstance(domains, MAP):
+        raise ValueError(
+            f"Resonator currently only supports Multiply-Add-Permute (MAP) VSA model, provided: {domains.__class__.__name__}"
+        )
+
+    n = estimates.size(-2)
+
+    # Get binding inverse of the estimates
+    inv_estimates = estimates.inverse()
+
+    # Roll over the number of estimates to align each row with the other symbols
+    # Example: for factorizing x, y, z the stacked matrix has the following estimates:
+    # [[z, y],
+    #  [x, z],
+    #  [y, x]]
+    rolled = []
+    for i in range(1, n):
+        rolled.append(inv_estimates.roll(i, -2))
+
+    inv_estimates = torch.stack(rolled, dim=-2)
+
+    # First bind all the other estimates together: z * y, x * z, y * z
+    inv_others = multibind(inv_estimates)
+    # Then unbind all other estimates from the input: s * (x * y), s * (x * z), s * (y * z)
+    new_estimates = bind(input.unsqueeze(-2), inv_others)
+
+    similarity = dot_similarity(new_estimates.unsqueeze(-2), domains)
+    output = dot_similarity(similarity, domains.mT).squeeze(-2)
+
+    # normalize the output vector with a non-linearity
+    return output.sign()
+
+
 def map_range(
     input: Tensor,
     in_min: float,
diff --git a/torchhd/memory.py b/torchhd/memory.py
@@ -0,0 +1,126 @@
+from typing import Optional
+import torch
+from torch import Tensor
+import torch.nn as nn
+from scipy.stats import binom
+
+import torchhd.functional as functional
+
+
+class SDM(nn.Module):
+    r"""`Sparse Distributed Memory <https://redwood.berkeley.edu/wp-content/uploads/2020/08/KanervaP_SDMrelated_models1993.pdf>`_
+
+    The Sparse Distributed Memory (SDM) is specified by its (typically random) addresses and its content.
+
+    Args:
+        num_addresses (int): The number of addresses in he memory.
+        address_dim (int): The dimensionality of the address vectors.
+        content_dim (int): The dimensionality of the content vectors.
+        p (float, optional): The expected fraction of memory address that will contain any value. Default: ``0.000368``.
+        kappa (int, optional): The maximum count for each memory cell, values are clipped between [-kappa, kappa]. Default: no clipping.
+        dtype (``torch.dtype``, optional): the desired data type of returned tensor. Default: if ``None`` depends on VSA_Model.
+        device (``torch.device``, optional):  the desired device of returned tensor. Default: if ``None``, uses the current device for the default tensor type (see torch.set_default_tensor_type()). ``device`` will be the CPU for CPU tensor types and the current CUDA device for CUDA tensor types.
+        requires_grad (bool, optional): If autograd should record operations on the returned tensor. Default: ``False``.
+
+    Shapes:
+        - Addresses: :math:`(n, d)`
+        - Content: :math:`(n, d)`
+
+    Examples::
+        >>> address = torchhd.random_hv(6, 512)
+        >>> sdm = torchhd.memory.SDM(100000, 512, 512)
+        >>> # use as associative memory
+        >>> sdm.write(address, address)
+        >>> read = sdm.read(address)
+        >>> torchhd.cos_similarity(read, address)
+        MAP([[ 1.0000,  0.0156, -0.0039, -0.0742,  0.0000, -0.0195],
+             [ 0.0156,  1.0000, -0.0352, -0.0586,  0.0000, -0.0039],
+             [-0.0039, -0.0352,  1.0000,  0.0156,  0.0820, -0.0234],
+             [-0.0742, -0.0586,  0.0156,  1.0000, -0.0039,  0.0000],
+             [ 0.0000,  0.0000,  0.0820, -0.0039,  1.0000,  0.0195],
+             [-0.0195, -0.0039, -0.0234,  0.0000,  0.0195,  1.0000]])
+    """
+
+    addresses: Tensor
+    content: Tensor
+    threshold: int
+    kappa: Optional[int]
+
+    def __init__(
+        self,
+        num_addresses: int,
+        address_dim: int,
+        content_dim: int,
+        p: float = 0.000368,
+        kappa: Optional[int] = None,
+        dtype=None,
+        device=None,
+        requires_grad=False,
+    ) -> None:
+        super().__init__()
+
+        radius = int(binom.ppf(p, address_dim, 0.5))
+        self.threshold = address_dim - 2 * radius
+        self.kappa = kappa
+
+        addresses = functional.random_hv(
+            num_addresses, address_dim, dtype=dtype, device=device
+        )
+        self.addresses = nn.Parameter(addresses, requires_grad)
+
+        content = functional.empty_hv(num_addresses, content_dim, device=device)
+        self.content = nn.Parameter(content, requires_grad)
+
+    def read(self, address: Tensor) -> Tensor:
+        r"""Read value from Sparse Distributed Memory at address.
+
+        Args:
+            address (Tensor): The address vector for the memory lookup.
+
+        Shapes:
+            - Address: :math:`(*, d)`
+
+        """
+
+        if address.dim() == 1:
+            address.unsqueeze_(0)
+
+        similarity = functional.dot_similarity(address, self.addresses)
+        is_active = similarity >= self.threshold
+        print(is_active.sum(-1))
+
+        # sparse matrix-vector multiplication
+        r_indices, c_indices = is_active.nonzero().T
+        read = torch.zeros_like(address)
+        read.index_add_(0, r_indices, self.content[c_indices])
+        return read.sign()
+
+    def write(self, address: Tensor, value: Tensor) -> Tensor:
+        r"""Write value to Sparse Distributed Memory at address.
+
+        Args:
+            address (Tensor): The address vector for the write to memory.
+            value (Tensor): The value vector written to memory.
+
+        Shapes:
+            - Address: :math:`(*, d)`
+            - Value: :math:`(*, d)`
+
+        """
+
+        if address.dim() == 1:
+            address.unsqueeze_(0)
+
+        if value.dim() == 1:
+            value.unsqueeze_(0)
+
+        similarity = functional.dot_similarity(address, self.addresses)
+        is_active = similarity >= self.threshold
+        print(is_active.sum(-1))
+
+        # sparse outer product and addition
+        v_indices, c_indices = is_active.nonzero().T
+        self.content.index_add_(0, c_indices, value[v_indices])
+
+        if self.kappa is not None:
+            self.content.clamp_(-self.kappa, self.kappa)

Original file line number	Diff line number	Diff line change
`@@ -10,4 +10,3 @@ torchhd.models`
`10`	`10`	`:template: class.rst`
`11`	`11`
`12`	`12`	`Centroid`
`13`		`-`
-Original file line number
+Diff line change
@@ @@ -1,6 +1,7 @@ @@
 torch
 torchvision
 pandas
 +scipy
 requests
 tqdm
 numpy