"""Lesson 7 check: the residual connection, hyper-connections (HC) and why unconstrained HC
becomes unstable when many layers are stacked.

Toy: hidden size C = 4, n = 2 streams, the token "day" from lesson 3, a stand-in layer function F.
The HC equations and the gain measure follow the mHC paper (arXiv 2512.24880), sections 1 and 3.

Every number printed here appears in lesson-07-hyper-connections.md.
Run:  python3 lesson07_check.py
"""
import torch

torch.set_default_dtype(torch.float64)           # 60 products in a row: use double precision
torch.set_printoptions(precision=4, sci_mode=False, linewidth=120)


def r4(x):
    return [round(v, 4) for v in torch.as_tensor(x).flatten().tolist()]


# ---------------------------------------------------------------- 0. the toy
C, n = 4, 2
x_day = torch.tensor([2., 1., 0., 0.])           # hidden state of "day" (lessons 1 and 3)
W_F = torch.tensor([[0.,  0.,  0.,  0.],         # stand-in layer: F(h) = 0.5 x [h2, h3, h4, 0]
                    [0.5, 0.,  0.,  0.],
                    [0.,  0.5, 0.,  0.],
                    [0.,  0.,  0.5, 0.]])


def F(h):                                        # (.., 4) x (4, 4) -> (.., 4)
    return h @ W_F


# The three HC mappings of the walkthrough. All values are chosen for easy arithmetic.
H_pre = torch.tensor([[0.5, 0.5]])               # (1, n): reads the streams into the layer input
H_post = torch.tensor([[1.0, 0.5]])              # (1, n): writes the layer output to the streams
M = torch.tensor([[1.1, 0.2],                    # (n, n): H_res, every entry within 0.2 of identity
                  [0.1, 0.9]])
I2 = torch.eye(n)
# The fixed mappings the mHC paper uses when a mapping is disabled (caption of its table 1)
H_pre_fix = torch.full((1, n), 1.0 / n)
H_post_fix = torch.ones(1, n)


# The two functions of the listing in section 10
def hc_layer(x, H_pre, H_post, H_res):
    h = H_pre @ x                                # A  (1, n) x (n, C) -> (1, C)
    out = F(h)                                   # B  (1, C)
    return H_res @ x + H_post.T @ out            # C  (n, n) x (n, C) + (n, 1) x (1, C)


def gains(P):
    fwd = P.abs().sum(dim=1).max().item()        # D  largest absolute row sum
    bwd = P.abs().sum(dim=0).max().item()        # E  largest absolute column sum
    return fwd, bwd


# ---------------------------------------------------------------- 1. the residual connection
print("1. The residual connection x_(l+1) = x_l + F(x_l) on 'day', 4 layers")
print("F(h) = 0.5 x [h2, h3, h4, 0]; for 'day' F gives", F(x_day).tolist(),
      "(the step D of lessons 5 and 6 for this token)")
x = x_day.clone()
outs = []
for l in range(4):
    f = F(x)
    outs.append(f)
    print(f"  layer {l + 1}: x = {x.tolist()}  F(x) = {f.tolist()}  x + F(x) = {(x + f).tolist()}")
    x = x + f
x4_res = x
print("x_4 =", x4_res.tolist())
print("unrolled: x_0 + sum of the 4 layer outputs =", x_day.tolist(), "+", torch.stack(outs).sum(0).tolist(),
      "=", (x_day + torch.stack(outs).sum(0)).tolist())
print("coefficient of x_0 in x_4: 1 x 1 x 1 x 1 =", 1 ** 4)
x = x_day.clone()
for l in range(60):
    x = x + F(x)
print("60 layers: x_60 =", x.tolist(), "(the first number grew by 60 x 0.5 =", 60 * 0.5, ")")
print("without the '+', x_(l+1) = F(x_l): after layer 1", F(x_day).tolist(), " after layer 2",
      F(F(x_day)).tolist())

# ---------------------------------------------------------------- 2. attempt 1: a larger hidden size
print("\n2. Attempt 1: make the hidden size n times larger")
for name, d, k in (("toy", 4, 2), ("27B model of the mHC paper", 2560, 4), ("V4-Pro", 7168, 4)):
    print(f"  {name}: d = {d}, one d x d weight matrix = {d * d:,}; with d = {d * k}: {(d * k) ** 2:,}"
          f" ({(d * k) ** 2 // (d * d)} times)")

# ---------------------------------------------------------------- 3. one HC layer by hand
print("\n3. One hyper-connection layer: x_(l+1) = H_res x_l + H_post^T F(H_pre x_l)")
x0 = torch.stack([x_day, x_day])                 # (n, C): both streams start as copies (our choice)
print("x_0 (2, 4):\n", x0)
h = H_pre @ x0
print("step 1, read:  H_pre (1, 2) x x_0 (2, 4) -> h (1, 4) =", h.tolist())
out = F(h)
print("step 2, layer: F(h) (1, 4) =", out.tolist())
wb = H_post.T @ out
print("step 3, write: H_post^T (2, 1) x F(h) (1, 4) -> (2, 4) =", wb.tolist())
mix = M @ x0
print("step 4, mix:   H_res (2, 2) x x_0 (2, 4) -> (2, 4) =", [r4(r) for r in mix])
print("   row sums of H_res:", r4(M.sum(1)), " column sums:", r4(M.sum(0)))
x1 = mix + wb
print("step 5, add:   x_1 (2, 4) =", [r4(r) for r in x1])
assert torch.allclose(x1, hc_layer(x0, H_pre, H_post, M))
h2 = H_pre @ x1
print("layer 2 reads: H_pre x x_1 =", r4(h2), " F =", r4(F(h2)))

# ---------------------------------------------------------------- 4. four layers
print("\n4. Four layers")


def run_hc(x, L, H_pre, H_post, H_res):
    for _ in range(L):
        x = hc_layer(x, H_pre, H_post, H_res)
    return x


xa = run_hc(x0, 4, H_pre_fix, H_post_fix, I2)
print("(a) fixed mappings H_pre = [0.5, 0.5], H_post = [1, 1], H_res = I: x_4 =", [r4(r) for r in xa],
      " equal to the plain residual stack in both streams:", torch.allclose(xa, x4_res.expand(2, 4)))
xa2 = run_hc(x0, 4, H_pre, H_post, I2)
print("    walkthrough H_pre, H_post with H_res = I: x_4 =", [r4(r) for r in xa2])
xb = x0
for l in range(4):
    xb = hc_layer(xb, H_pre, H_post, M)
    print(f"(b) H_res = M, after layer {l + 1}:", [r4(r) for r in xb])
M2 = M @ M
M4 = M2 @ M2
print("M x M =", [r4(r) for r in M2])
print("M^4 = (M x M) x (M x M) =", [r4(r) for r in M4])
print("M^4 row sums", r4(M4.sum(1)), " column sums", r4(M4.sum(0)))
print("I^4 =", torch.matrix_power(I2, 4).tolist())
print("first term of the unrolled form, M^4 x x_0 =", [r4(r) for r in M4 @ x0])
# the unrolled form (eq. 4 of the mHC paper) reproduces x_4
xs, terms = [x0], []
for l in range(4):
    xs.append(hc_layer(xs[-1], H_pre, H_post, M))
unrolled = M4 @ x0
for i in range(4):
    unrolled = unrolled + torch.matrix_power(M, 3 - i) @ H_post.T @ F(H_pre @ xs[i])
print("unrolled sum equals the layer-by-layer result:", torch.allclose(unrolled, xs[4]))

# ---------------------------------------------------------------- 5. the gain of the product
print("\n5. Amax Gain Magnitude of the product of the residual mappings")
print("forward = largest absolute row sum, backward = largest absolute column sum")
for name, P in (("I^4", torch.matrix_power(I2, 4)), ("M^4", M4)):
    f, b = gains(P)
    print(f"  {name}: forward {f:.4f}  backward {b:.4f}")
# what the two numbers mean, on the product alone (F switched off)
xin = torch.ones(n, C, requires_grad=True)       # every stream carries the same vector of ones
y = xin
for _ in range(4):
    y = M @ y
print("  forward check: streams of ones through 4 mixings ->", r4(y[:, 0]), "(the row sums of M^4)")
y.sum().backward()
print("  backward check: a gradient of ones on x_4 arrives at x_0 as", r4(xin.grad[:, 0]),
      "(the column sums of M^4, by autograd)")

print("\n   depth   identity      M: forward      backward     0.95 I + 0.05: forward")
S = torch.tensor([[0.90, 0.05], [0.05, 0.90]])   # every row and column sums to 0.95
curve = {}
for L in (1, 2, 4, 8, 16, 30, 60):
    fi, _ = gains(torch.matrix_power(I2, L))
    fm, bm = gains(torch.matrix_power(M, L))
    fs, _ = gains(torch.matrix_power(S, L))
    curve[L] = (fm, bm, fs)
    print(f"   {L:>5}   {fi:.4f}   {fm:>14,.4f}   {bm:>12,.4f}   {fs:.4f}")
ev = torch.linalg.eigvals(M).real
print("eigenvalues of M:", r4(ev), "; the larger one to the power 60:", f"{ev.max().item() ** 60:,.1f}")
print("gain per layer between depth 30 and 60:", round((curve[60][0] / curve[30][0]) ** (1 / 30), 4))

# the whole toy forward pass, with F, for 60 layers
x60a = run_hc(x0, 60, H_pre_fix, H_post_fix, I2)
xb = run_hc(x0, 60, H_pre, H_post, M)
print("toy forward pass with F, 60 layers: (a) fixed mappings x_60 =", [r4(r) for r in x60a])
print("                                    (b) H_res = M     x_60 =", [[f"{v:,.0f}" for v in r] for r in xb.tolist()])

# ---------------------------------------------------------------- 6. a different matrix in every layer
print("\n6. A different residual mapping in every layer: H_res = I + eps x N(0, 1), 60 layers, 1,000 runs")
for eps in (0.05, 0.1, 0.2):
    g = torch.Generator().manual_seed(7)
    fw = []
    for run in range(1000):
        P = torch.eye(n)
        for l in range(60):
            P = (torch.eye(n) + eps * torch.randn(n, n, generator=g)) @ P
        fw.append(gains(P)[0])
    fw = torch.tensor(fw)
    q = torch.quantile(fw, torch.tensor([0.05, 0.5, 0.95]))
    print(f"  eps = {eps}: forward gain at depth 60: 5th percentile {q[0]:.3f}  median {q[1]:.3f}  "
          f"95th percentile {q[2]:.2f}  largest {fw.max():,.1f}  smallest {fw.min():.4f}")
print("  (the deviations have mean zero, so no layer pushes in a fixed direction; M pushes the same way"
      " in all 60 layers)")

# ---------------------------------------------------------------- 7. the reported figure for scale
print("\n7. The mHC paper reports peaks of 3000 for the composite mapping of HC in its 27B model")
print("27B model: 30 layers; attention and FFN each count as a layer in its figure 3: 30 x 2 =", 30 * 2)
print("a constant gain g per layer with g^60 = 3000: g =", round(3000 ** (1 / 60), 4))
print("mHC paper, section 5.4: about 1.6 for mHC; 3000 / 1.6 =", round(3000 / 1.6))

# ---------------------------------------------------------------- 8. cost
print("\n8. What HC adds")


def cost(name, C, n, units=None):
    params = (n + 2) * C + n * n + 2 * n + 3     # theta_pre, theta_post (1, C); theta_res (n, C);
    #                                              b_pre, b_post (1, n); b_res (n, n); three alphas
    rd_res, wr_res = 2 * C, C
    rd_hc = (5 * n + 1) * C + n * n + 2 * n
    wr_hc = (3 * n + 1) * C + n * n + 2 * n
    print(f"  {name}: C = {C}, n = {n}")
    print(f"    residual stream per token: {C:,} -> {n * C:,} numbers")
    print(f"    entries of the three mappings per layer: n^2 + 2n = {n * n + 2 * n}")
    print(f"    parameters that generate them (eq. 5): (n + 2) C + n^2 + 2n + 3 = {params:,}")
    if units:
        print(f"    x {units} hyper-connections = {params * units:,}")
    print(f"    memory access per token: read {rd_res:,} -> {rd_hc:,} ({rd_hc / rd_res:.1f} times), "
          f"write {wr_res:,} -> {wr_hc:,} ({wr_hc / wr_res:.1f} times)")
    return params


cost("toy", 4, 2, 4)
p27 = cost("27B model of the mHC paper", 2560, 4, 60)
print("    share of 27.0B parameters:", f"{p27 * 60 / 27.0e9:.6%}")
cost("V4-Pro sizes (V4 uses mHC, lesson 8)", 7168, 4)
cost("V4-Flash sizes", 4096, 4)
print("  toy walkthrough uses static mappings only:", n * n + 2 * n, "numbers per layer")
print("  27B benchmark scores, baseline -> HC (mHC paper, table 4): BBH 43.8 -> 48.9, GSM8K 46.7 -> 53.2,"
      " MMLU 59.0 -> 63.0")
