tinygrab/test/test_randomness.py

import math
import unittest
import numpy as np
import torch
from tinygrad.tensor import Tensor
import tinygrad.nn as nn
from tinygrad.helpers import dtypes
from functools import partial


# https://gist.github.com/devries/11405101
def ksprob(a):
    fac, total, termbf = 2.0, 0.0, 0.0
    a2 = -2.0 * a * a
    for j in range(1, 101):
        term = fac * math.exp(a2 * j * j)
        total += term
        if math.fabs(term) <= 0.001 * termbf or math.fabs(term) <= 1e-8 * total:
            return total
        fac = -fac
        termbf = math.fabs(term)
    return 1.0


def kstest(l1, l2):
    n1, n2 = len(l1), len(l2)
    l1.sort()
    l2.sort()
    j1, j2, d, fn1, fn2 = 0, 0, 0.0, 0.0, 0.0
    while j1 < n1 and j2 < n2:
        d1, d2 = l1[j1], l2[j2]
        if d1 <= d2:
            fn1 = (float(j1) + 1.0) / float(n1)
            j1 += 1
        if d2 <= d1:
            fn2 = (float(j2) + 1.0) / float(n2)
            j2 += 1
        dtemp = math.fabs(fn2 - fn1)
        if dtemp > d:
            d = dtemp
    ne = float(n1 * n2) / float(n1 + n2)
    nesq = math.sqrt(ne)
    prob = ksprob((nesq + 0.12 + 0.11 / nesq) * d)
    return prob


def equal_distribution(
    tiny_func, torch_func=None, numpy_func=None, shape=(20, 23), alpha=0.05
):
    Tensor.manual_seed(1337)
    torch.manual_seed(1337)
    np.random.seed(1337)
    assert not (
        torch_func is None and numpy_func is None
    ), "no function to compare with"
    x = tiny_func(*shape).numpy().flatten()
    if numpy_func is not None:
        y = numpy_func(shape).flatten()
    if torch_func is not None:
        z = torch_func(shape).numpy().flatten()
    return (numpy_func is None or kstest(x, y) >= alpha) and (
        torch_func is None or kstest(x, z) >= alpha
    )


def normal_test(func, shape=(20, 23), alpha=0.05):
    return equal_distribution(
        func, numpy_func=lambda x: np.random.randn(*x), shape=shape, alpha=alpha
    )


class TestRandomness(unittest.TestCase):
    def test_rand(self):
        self.assertFalse(normal_test(Tensor.rand))
        self.assertTrue(
            equal_distribution(Tensor.rand, torch.rand, lambda x: np.random.rand(*x))
        )

    def test_randn(self):
        self.assertTrue(normal_test(Tensor.randn))
        self.assertTrue(
            equal_distribution(Tensor.randn, torch.randn, lambda x: np.random.randn(*x))
        )

    def test_normal(self):
        self.assertTrue(normal_test(Tensor.normal))
        self.assertTrue(
            equal_distribution(
                Tensor.normal,
                lambda x: torch.nn.init.normal_(torch.empty(x), mean=0, std=1),
                lambda x: np.random.normal(loc=0, scale=1, size=x),
            )
        )

    def test_uniform(self):
        self.assertFalse(normal_test(Tensor.uniform))
        self.assertTrue(
            equal_distribution(
                Tensor.uniform,
                lambda x: torch.nn.init.uniform_(torch.empty(x)),
                lambda x: np.random.uniform(size=x),
            )
        )
        self.assertTrue(
            equal_distribution(
                partial(Tensor.uniform, low=-100, high=100, dtype=dtypes.int32),
                numpy_func=lambda x: np.random.randint(low=-100, high=100, size=x),
            )
        )

    def test_scaled_uniform(self):
        self.assertFalse(normal_test(Tensor.scaled_uniform))
        self.assertTrue(
            equal_distribution(
                Tensor.scaled_uniform,
                lambda x: torch.nn.init.uniform_(torch.empty(x), a=-1, b=1)
                / math.sqrt(math.prod(x)),
                lambda x: np.random.uniform(-1, 1, size=x) / math.sqrt(math.prod(x)),
            )
        )

    def test_glorot_uniform(self):
        self.assertFalse(normal_test(Tensor.glorot_uniform))
        self.assertTrue(
            equal_distribution(
                Tensor.glorot_uniform,
                lambda x: torch.nn.init.xavier_uniform_(torch.empty(x)),
                lambda x: np.random.uniform(-1, 1, size=x)
                * math.sqrt(6 / (x[0] + math.prod(x[1:]))),
            )
        )

    def test_kaiming_uniform(self):
        Tensor.manual_seed(1337)
        torch.manual_seed(1337)
        np.random.seed(1337)
        for shape in [(128, 64, 3, 3), (20, 24)]:
            self.assertTrue(
                equal_distribution(
                    Tensor.kaiming_uniform,
                    lambda x: torch.nn.init.kaiming_uniform_(torch.empty(x)),
                    shape=shape,
                )
            )

    def test_kaiming_normal(self):
        Tensor.manual_seed(1337)
        torch.manual_seed(1337)
        np.random.seed(1337)
        for shape in [(128, 64, 3, 3), (20, 24)]:
            self.assertTrue(
                equal_distribution(
                    Tensor.kaiming_normal,
                    lambda x: torch.nn.init.kaiming_normal_(torch.empty(x)),
                    shape=shape,
                )
            )

    def test_multinomial(self):
        self.assertRaises(
            AssertionError, lambda: Tensor(2).multinomial(1, replacement=False)
        )
        self.assertRaises(
            AssertionError, lambda: Tensor([1, 9]).multinomial(0, replacement=False)
        )

        def _check_with_torch(w, num_samples, replacement):
            tiny_res = Tensor(w).multinomial(num_samples, replacement=replacement)
            torch_res = torch.tensor(w).multinomial(
                num_samples, replacement=replacement
            )
            self.assertEqual(tiny_res.shape, torch_res.shape)
            if torch_res.ndim == 1:
                tiny_res = tiny_res.unsqueeze(0)
                torch_res = torch_res.unsqueeze(0)
            for i in range(torch_res.shape[0]):
                self.assertTrue(
                    equal_distribution(lambda *_: tiny_res[i], lambda _: torch_res[i])
                )

        _check_with_torch(w=[0.231, 0.0, 1.0, 0.5], num_samples=2000, replacement=True)
        _check_with_torch(
            w=[[0.2, 0.8]], num_samples=2000, replacement=True
        )  # 2D but only 1 row
        _check_with_torch(
            w=[[0.453, 0.0, 1.0, 0.81], [0.1, 0.8, 0.0, 0.1]],
            num_samples=2000,
            replacement=True,
        )
        # no-replacement isn't supported, unless taking only one sample
        w = [0.1, 0.9]
        self.assertRaises(
            AssertionError, lambda: Tensor(w).multinomial(100, replacement=False)
        )
        tiny_samples = [
            Tensor(w).multinomial(1, replacement=False).numpy().item()
            for _ in range(1000)
        ]
        torch_samples = [
            torch.tensor(w).multinomial(1, replacement=False).item()
            for _ in range(1000)
        ]
        self.assertTrue(
            equal_distribution(
                lambda *_: Tensor(tiny_samples), lambda _: torch.tensor(torch_samples)
            )
        )

    def test_multinomial_counterexample(self):
        tiny_res = Tensor([0.3, 0.6, 0.1]).multinomial(2000, replacement=True)
        torch_res = torch.tensor([0.3, 0.6, 0.1]).multinomial(2000, replacement=True)
        self.assertTrue(equal_distribution(lambda *_: tiny_res, lambda _: torch_res))
        torch_res = torch.tensor([0.2, 0.7, 0.1]).multinomial(2000, replacement=True)
        self.assertFalse(equal_distribution(lambda *_: tiny_res, lambda _: torch_res))

    def test_conv2d_init(self):
        params = (128, 256, (3, 3))
        assert equal_distribution(
            lambda *_: nn.Conv2d(*params).weight,
            lambda _: torch.nn.Conv2d(*params).weight.detach(),
        )
        assert equal_distribution(
            lambda *_: nn.Conv2d(*params).bias,
            lambda _: torch.nn.Conv2d(*params).bias.detach(),
        )

    def test_linear_init(self):
        params = (64, 64)
        assert equal_distribution(
            lambda *_: nn.Linear(*params).weight,
            lambda _: torch.nn.Linear(*params).weight.detach(),
        )
        assert equal_distribution(
            lambda *_: nn.Linear(*params).bias,
            lambda _: torch.nn.Linear(*params).bias.detach(),
        )

    def test_bn_init(self):
        params = (64,)
        assert equal_distribution(
            lambda *_: nn.BatchNorm2d(*params).weight,
            lambda _: torch.nn.BatchNorm2d(*params).weight.detach(),
        )
        assert equal_distribution(
            lambda *_: nn.BatchNorm2d(*params).bias,
            lambda _: torch.nn.BatchNorm2d(*params).bias.detach(),
        )


if __name__ == "__main__":
    unittest.main()