Skip to content

Gradient Checking¤

Verify analytic gradients from backward() against numerical central differences.

from minitorch import Tensor, check_gradient

a = Tensor([1.0, 2.0, 3.0], requires_grad=True)
b = Tensor([4.0, 5.0, 6.0], requires_grad=True)
check_gradient(lambda: (a * b).sum(), [a, b])

check_gradient ¤

check_gradient(f, inputs, eps=1e-05, atol=0.0001, rtol=0.001)

Compare analytic gradients from backward() against numerical ones.

Raises AssertionError if any gradient is off beyond the tolerance.

Source code in minitorch/grad_check.py 
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
def check_gradient(f, inputs, eps=1e-5, atol=1e-4, rtol=1e-3):
    """Compare analytic gradients from backward() against numerical ones.

    Raises AssertionError if any gradient is off beyond the tolerance.
    """
    for inp in inputs:
        inp.zero_grad()
    loss = f()
    loss.backward()
    analytic = [inp.grad.copy() for inp in inputs]

    numerical = numerical_gradient(f, inputs, eps)

    for i, (a, n) in enumerate(zip(analytic, numerical)):
        if not np.allclose(a, n, atol=atol, rtol=rtol):
            max_diff = np.max(np.abs(a - n))
            raise AssertionError(
                f"Gradient check failed for input {i}: max diff = {max_diff}\n"
                f"Analytic:\n{a}\nNumerical:\n{n}"
            )
    return True

numerical_gradient ¤

numerical_gradient(f, inputs, eps=1e-05)

Estimate gradients of f w.r.t. each input by central differences.

Source code in minitorch/grad_check.py 
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
def numerical_gradient(f, inputs, eps=1e-5):
    """Estimate gradients of `f` w.r.t. each input by central differences."""
    grads = []
    # temporarily convert all inputs to float64 for precision
    orig_data = [inp.data.copy() for inp in inputs]
    for inp in inputs:
        inp.data = inp.data.astype(np.float64)
    # also stash float64 copies for perturbation
    f64_data = [inp.data.copy() for inp in inputs]

    for k, inp in enumerate(inputs):
        grad = np.zeros(inp.data.shape, dtype=np.float64)
        it = np.nditer(f64_data[k], flags=['multi_index'])
        while not it.finished:
            idx = it.multi_index
            old_val = f64_data[k][idx]

            inp.data = f64_data[k].copy()
            inp.data[idx] = old_val + eps
            loss_plus = float(f().data)

            inp.data = f64_data[k].copy()
            inp.data[idx] = old_val - eps
            loss_minus = float(f().data)

            grad[idx] = (loss_plus - loss_minus) / (2 * eps)
            it.iternext()
        inp.data = f64_data[k].copy()
        grads.append(grad.astype(np.float32))

    # restore original float32 data
    for inp, od in zip(inputs, orig_data):
        inp.data = od
    return grads