NumPy Sample Questions — Solutions

Using only: [Link], [Link], [Link], [Link], [Link], [Link]

Q1. TF-IDF from scratch

import numpy as np

corpus = [
"the cat sat on the mat",
"the dog sat on the log",
"the cat and the dog are friends"
]

# Build vocabulary
words = sorted(set(w for s in corpus for w in [Link]()))
vocab = {w: i for i, w in enumerate(words)}
V, D = len(vocab), len(corpus)

# Term Frequency
TF = [Link]((D, V))
for i, sent in enumerate(corpus):
tokens = [Link]()
for w in tokens:
TF[i, vocab[w]] += 1
TF[i] /= len(tokens)

# Inverse Document Frequency

df = [Link](TF > 0, axis=0)
IDF = [Link]((D + 1) / (df + 1)) + 1

TFIDF = TF * IDF
print("Vocabulary:", words)
print("TF-IDF Matrix:")
print([Link](TFIDF, 4))

Q2. Bag of Words from scratch

import numpy as np

corpus = ["I love cats", "I love dogs", "Dogs and cats are friends"]

# Vocabulary
words = sorted(set([Link]() for s in corpus for w in [Link]()))
vocab = {w: i for i, w in enumerate(words)}

# BoW vectors
BoW = [Link]((len(corpus), len(vocab)), dtype=int)
for i, sent in enumerate(corpus):
for w in [Link]().split():
BoW[i, vocab[w]] += 1

print("Vocabulary:", words)
print("BoW Matrix:")
print(BoW)

Q3. Cosine Similarity

import numpy as np

def cosine_similarity(a, b):

return [Link](a, b) / ([Link](a) * [Link](b) + 1e-10)
query = [Link]([1, 0, 1, 0, 1], dtype=float)
sentences = [
[Link]([1, 0, 1, 0, 0], dtype=float),
[Link]([0, 1, 0, 1, 0], dtype=float),
[Link]([1, 0, 1, 1, 1], dtype=float),
]

sims = [cosine_similarity(query, s) for s in sentences]

best = [Link](sims)
print("Similarities:", [Link](sims, 4))
print(f"Most similar sentence index: {best} (score={sims[best]:.4f})")

Q4. One-Hot Encoding

import numpy as np

sentence = "I love deep learning"

tokens = [Link]()
vocab = sorted(set(tokens))
word2idx = {w: i for i, w in enumerate(vocab)}

one_hot = [Link]((len(tokens), len(vocab)), dtype=int)

for i, w in enumerate(tokens):
one_hot[i, word2idx[w]] = 1

print("Vocabulary:", vocab)
print("One-Hot Encoding:")
print(one_hot)

Q5. Mean Word Embedding

import numpy as np

word_vectors = {
"deep": [Link]([0.1, 0.5, 0.3]),
"learning": [Link]([0.4, 0.2, 0.6]),
"is": [Link]([0.0, 0.1, 0.0]),
"fun": [Link]([0.7, 0.3, 0.8]),
}

sentence = "deep learning is fun"

vectors = [word_vectors[w] for w in [Link]() if w in word_vectors]
mean_emb = [Link](vectors, axis=0)
print("Mean Embedding:", [Link](mean_emb, 4))

Q6. Single RNN Cell Forward Pass

import numpy as np

[Link](42)
input_size, hidden_size = 3, 4

Wx = [Link](hidden_size, input_size) * 0.1

Wh = [Link](hidden_size, hidden_size) * 0.1
b = [Link](hidden_size)

x = [Link](input_size)
h = [Link](hidden_size)

h_next = [Link](Wx @ x + Wh @ h + b)
print("Input x:", [Link](x, 4))
print("Next hidden state h_next:", [Link](h_next, 4))
Q7. Multi-step RNN Forward Pass (5 time steps)
import numpy as np

[Link](0)
T, input_size, hidden_size = 5, 3, 4

Wx = [Link](hidden_size, input_size) * 0.1

Wh = [Link](hidden_size, hidden_size) * 0.1
b = [Link](hidden_size)

X = [Link](T, input_size)
h = [Link](hidden_size)

for t in range(T):
h = [Link](Wx @ X[t] + Wh @ h + b)
print(f"Step {t+1} hidden state: {[Link](h, 4)}")

Q8. Backpropagation Through Time (BPTT)

import numpy as np

[Link](1)
input_size, hidden_size = 2, 3
T = 3

Wx = [Link](hidden_size, input_size) * 0.1

Wh = [Link](hidden_size, hidden_size) * 0.1
b = [Link](hidden_size)
Wy = [Link](1, hidden_size) * 0.1

X = [Link](T, input_size)
y_true = [Link]([1.0])

# Forward
hs = [[Link](hidden_size)]
for t in range(T):
[Link]([Link](Wx @ X[t] + Wh @ hs[-1] + b))

y_pred = Wy @ hs[-1]
loss = float(0.5 * (y_pred - y_true) ** 2)

# Backward
dWx = np.zeros_like(Wx)
dWh = np.zeros_like(Wh)
db = np.zeros_like(b)

dh = (Wy.T * (y_pred - y_true)).flatten()

for t in reversed(range(T)):
dt = dh * (1 - hs[t+1] ** 2)
dWx += [Link](dt, X[t])
dWh += [Link](dt, hs[t])
db += dt
dh = Wh.T @ dt

print(f"Loss: {loss:.4f}")
print("dWx (first row):", [Link](dWx[0], 6))
print("dWh (first row):", [Link](dWh[0], 6))

Q9. Many-to-One RNN (sum output)

import numpy as np

[Link](2)
input_size, hidden_size = 1, 4

Wx = [Link](hidden_size, input_size) * 0.1

Wh = [Link](hidden_size, hidden_size) * 0.1
b = [Link](hidden_size)
Wy = [Link](1, hidden_size) * 0.1

sequence = [Link]([[1.0],[2.0],[3.0],[4.0],[5.0]])
h = [Link](hidden_size)
for x in sequence:
h = [Link](Wx @ x + Wh @ h + b)

output = Wy @ h
print(f"Input sequence sum: {[Link]()}")
print(f"RNN output (single value): {output[0]:.4f}")

Q10. Vanilla RNN Character-level Forward Pass

import numpy as np

[Link](3)
chars = ['h','e','l','o']
char2i = {c:i for i,c in enumerate(chars)}
vocab_size, hidden_size = len(chars), 5

Wx = [Link](hidden_size, vocab_size) * 0.1

Wh = [Link](hidden_size, hidden_size) * 0.1
b = [Link](hidden_size)
Wy = [Link](vocab_size, hidden_size) * 0.1

seq = "hello"
h = [Link](hidden_size)
for ch in seq:
x = [Link](vocab_size)
x[char2i[ch[0]]] = 1
h = [Link](Wx @ x + Wh @ h + b)
y = Wy @ h
print(f"Char '{ch}' -> output logits: {[Link](y, 3)}")

Q11. LSTM Cell Forward Pass

import numpy as np

[Link](4)
input_size, hidden_size = 3, 4

def sigmoid(z): return 1 / (1 + [Link](-z))

# Weight matrices [hidden+input, hidden]

W = [Link](4 * hidden_size, hidden_size + input_size) * 0.1
b = [Link](4 * hidden_size)

x = [Link](input_size)
h = [Link](hidden_size)
C = [Link](hidden_size)

combined = [Link]([h, x])

gates = W @ combined + b

f = sigmoid(gates[0*hidden_size : 1*hidden_size]) # forget

i = sigmoid(gates[1*hidden_size : 2*hidden_size]) # input
g = [Link] (gates[2*hidden_size : 3*hidden_size]) # cell candidate
o = sigmoid(gates[3*hidden_size : 4*hidden_size]) # output
C_next = f * C + i * g
h_next = o * [Link](C_next)

print("Forget gate f :", [Link](f, 4))

print("Input gate i :", [Link](i, 4))
print("Cell cand. g :", [Link](g, 4))
print("Output gate o :", [Link](o, 4))
print("Cell state C :", [Link](C_next, 4))
print("Hidden state h:", [Link](h_next, 4))

Q12. LSTM over 4 Time Steps

import numpy as np

[Link](5)
input_size, hidden_size, T = 3, 4, 4

def sigmoid(z): return 1 / (1 + [Link](-z))

W = [Link](4*hidden_size, hidden_size+input_size) * 0.1

b = [Link](4*hidden_size)
X = [Link](T, input_size)

h, C = [Link](hidden_size), [Link](hidden_size)
for t in range(T):
combined = [Link]([h, X[t]])
gates = W @ combined + b
f = sigmoid(gates[0*hidden_size:1*hidden_size])
i = sigmoid(gates[1*hidden_size:2*hidden_size])
g = [Link] (gates[2*hidden_size:3*hidden_size])
o = sigmoid(gates[3*hidden_size:4*hidden_size])
C = f * C + i * g
h = o * [Link](C)
print(f"t={t+1} C: {[Link](C,3)} h: {[Link](h,3)}")

Q13. All Four LSTM Gates with Manual Weights

import numpy as np

[Link](6)
input_size, hidden_size = 2, 3

def sigmoid(z): return 1 / (1 + [Link](-z))

Wf = [Link](hidden_size, hidden_size+input_size) * 0.1

Wi = [Link](hidden_size, hidden_size+input_size) * 0.1
Wg = [Link](hidden_size, hidden_size+input_size) * 0.1
Wo = [Link](hidden_size, hidden_size+input_size) * 0.1
bf, bi, bg, bo = ([Link](hidden_size) for _ in range(4))

x = [Link]([0.5, -0.3])
h = [Link](hidden_size)
C = [Link](hidden_size)
z = [Link]([h, x])

f = sigmoid(Wf @ z + bf)
i = sigmoid(Wi @ z + bi)
g = [Link] (Wg @ z + bg)
o = sigmoid(Wo @ z + bo)
C = f * C + i * g
h = o * [Link](C)

print("Forget gate:", [Link](f, 4))

print("Input gate:", [Link](i, 4))
print("Cell cand. :", [Link](g, 4))
print("Output gate:", [Link](o, 4))
print("C_next :", [Link](C, 4))
print("h_next :", [Link](h, 4))

Q14. Forget Gate Effect on Cell State

import numpy as np

C_prev = [Link]([1.0, 2.0, 3.0])

# Forget gate = 0: memory fully erased

f_zero = [Link](3)
C_new_0 = f_zero * C_prev
print("f=0 (forget all) -> C:", C_new_0)

# Forget gate = 1: memory fully retained

f_one = [Link](3)
C_new_1 = f_one * C_prev
print("f=1 (remember all)-> C:", C_new_1)

Q15. Two-step LSTM + Softmax Output

import numpy as np

[Link](7)
input_size, hidden_size, output_size = 3, 4, 5

def sigmoid(z): return 1 / (1 + [Link](-z))

def softmax(z):
e = [Link](z - [Link]())
return e / [Link]()

W = [Link](4*hidden_size, hidden_size+input_size) * 0.1

b = [Link](4*hidden_size)
Wy = [Link](output_size, hidden_size) * 0.1
X = [[Link](input_size) for _ in range(2)]

h, C = [Link](hidden_size), [Link](hidden_size)
for t, x in enumerate(X):
z = [Link]([h, x])
g = W @ z + b
f = sigmoid(g[0*hidden_size:1*hidden_size])
i = sigmoid(g[1*hidden_size:2*hidden_size])
cg= [Link] (g[2*hidden_size:3*hidden_size])
o = sigmoid(g[3*hidden_size:4*hidden_size])
C = f * C + i * cg
h = o * [Link](C)

probs = softmax(Wy @ h)
print("Output probabilities:", [Link](probs, 4))
print("Predicted class:", [Link](probs))

Q16. GRU Cell Forward Pass

import numpy as np

[Link](8)
input_size, hidden_size = 3, 4

def sigmoid(z): return 1 / (1 + [Link](-z))

Wr = [Link](hidden_size, hidden_size+input_size) * 0.1

Wz = [Link](hidden_size, hidden_size+input_size) * 0.1
Wn = [Link](hidden_size, hidden_size+input_size) * 0.1
br, bz, bn = ([Link](hidden_size) for _ in range(3))

x = [Link](input_size)
h = [Link](hidden_size)
z_cat = [Link]([h, x])

r = sigmoid(Wr @ z_cat + br) # reset gate

z = sigmoid(Wz @ z_cat + bz) # update gate
n = [Link](Wn @ [Link]([r*h, x]) + bn) # candidate
h_next = (1 - z) * n + z * h

print("Reset gate r :", [Link](r, 4))

print("Update gate z :", [Link](z, 4))
print("Candidate n :", [Link](n, 4))
print("New hidden h :", [Link](h_next, 4))

Q18. GRU over 5 Time Steps

import numpy as np

[Link](9)
input_size, hidden_size, T = 3, 4, 5

def sigmoid(z): return 1 / (1 + [Link](-z))

Wr = [Link](hidden_size, hidden_size+input_size) * 0.1

Wz = [Link](hidden_size, hidden_size+input_size) * 0.1
Wn = [Link](hidden_size, hidden_size+input_size) * 0.1
br, bz, bn = ([Link](hidden_size) for _ in range(3))
X = [Link](T, input_size)

h = [Link](hidden_size)
for t in range(T):
z_cat = [Link]([h, X[t]])
r = sigmoid(Wr @ z_cat + br)
z = sigmoid(Wz @ z_cat + bz)
n = [Link](Wn @ [Link]([r*h, X[t]]) + bn)
h = (1 - z) * n + z * h
print(f"Step {t+1} h: {[Link](h, 4)}")

Q19. GRU vs LSTM Parameter Count

import numpy as np

hidden_size, input_size = 128, 64

# GRU: 3 gates, each has W[h,h+i] + b[h]

gru_params = 3 * (hidden_size*(hidden_size+input_size) + hidden_size)

# LSTM: 4 gates, each has W[h,h+i] + b[h]

lstm_params = 4 * (hidden_size*(hidden_size+input_size) + hidden_size)

print(f"GRU parameters: {gru_params:,}")

print(f"LSTM parameters: {lstm_params:,}")
print(f"LSTM/GRU ratio : {lstm_params/gru_params:.4f}")

Q20. Reset Gate Effect in GRU

import numpy as np

[Link](10)
hidden_size, input_size = 3, 2
Wn = [Link](hidden_size, hidden_size+input_size) * 0.1
bn = [Link](hidden_size)

x = [Link]([0.5, -0.3])
h = [Link]([1.0, -1.0, 0.5])

# r = 0: previous hidden state ignored

r0 = [Link](hidden_size)
n0 = [Link](Wn @ [Link]([r0*h, x]) + bn)

# r = 1: previous hidden state fully used

r1 = [Link](hidden_size)
n1 = [Link](Wn @ [Link]([r1*h, x]) + bn)

print("Candidate h_tilde with r=0:", [Link](n0, 4))

print("Candidate h_tilde with r=1:", [Link](n1, 4))

Q21. Multi-step GRU + Sigmoid Binary Classification

import numpy as np

[Link](11)
input_size, hidden_size, T = 3, 4, 5

def sigmoid(z): return 1 / (1 + [Link](-z))

Wr = [Link](hidden_size, hidden_size+input_size) * 0.1

Wz = [Link](hidden_size, hidden_size+input_size) * 0.1
Wn = [Link](hidden_size, hidden_size+input_size) * 0.1
Wy = [Link](1, hidden_size) * 0.1
X = [Link](T, input_size)
h = [Link](hidden_size)

for t in range(T):
z_cat = [Link]([h, X[t]])
r = sigmoid(Wr @ z_cat)
z = sigmoid(Wz @ z_cat)
n = [Link](Wn @ [Link]([r*h, X[t]]))
h = (1 - z) * n + z * h

prob = sigmoid(Wy @ h)[0]

print(f"Binary classification probability: {prob:.4f}")
print(f"Predicted class: {int(prob >= 0.5)}")

Q22. Simple Encoder-Decoder (RNN)

import numpy as np

[Link](12)
input_size, hidden_size = 3, 4

def sigmoid(z): return 1 / (1 + [Link](-z))

Wx_enc = [Link](hidden_size, input_size) * 0.1

Wh_enc = [Link](hidden_size, hidden_size) * 0.1

Wx_dec = [Link](hidden_size, hidden_size) * 0.1

Wh_dec = [Link](hidden_size, hidden_size) * 0.1
Wy_dec = [Link](input_size, hidden_size) * 0.1

X = [Link](3, input_size) # source sequence len=3

# Encode
h = [Link](hidden_size)
for x in X:
h = [Link](Wx_enc @ x + Wh_enc @ h)
context = h
print("Context vector:", [Link](context, 4))

# Decode (reconstruct 3 steps)

h = context
for t in range(3):
h = [Link](Wx_dec @ h + Wh_dec @ h)
out = Wy_dec @ h
print(f"Decoder step {t+1} output: {[Link](out, 4)}")

Q23. Seq2Seq: Encoder Final State -> Decoder Init

import numpy as np

[Link](13)
input_size, hidden_size = 3, 4

Wx = [Link](hidden_size, input_size) * 0.1

Wh = [Link](hidden_size, hidden_size) * 0.1

X_enc = [Link](4, input_size)

X_dec = [Link](3, input_size)

# Encoder
h = [Link](hidden_size)
for x in X_enc:
h = [Link](Wx @ x + Wh @ h)
h_dec = h # pass final encoder state to decoder

print("Encoder final h:", [Link](h_dec, 4))

# Decoder
for t, x in enumerate(X_dec):
h_dec = [Link](Wx @ x + Wh @ h_dec)
print(f"Decoder step {t+1}: {[Link](h_dec, 4)}")

Q24. Character-level Encoder: 'hello' -> context vector

import numpy as np

[Link](14)
chars = sorted(set("hello"))
c2i = {c: i for i, c in enumerate(chars)}
vocab_size, hidden_size = len(chars), 6

Wx = [Link](hidden_size, vocab_size) * 0.1

Wh = [Link](hidden_size, hidden_size) * 0.1

h = [Link](hidden_size)
for ch in "hello":
x = [Link](vocab_size)
x[c2i[ch]] = 1
h = [Link](Wx @ x + Wh @ h)

print("Context vector for 'hello':", [Link](h, 4))

Q25. Teacher Forcing in Decoder

import numpy as np

[Link](15)
input_size, hidden_size = 3, 4

Wx = [Link](hidden_size, input_size) * 0.1

Wh = [Link](hidden_size, hidden_size) * 0.1
Wy = [Link](input_size, hidden_size) * 0.1

true_outputs = [Link](4, input_size) # ground-truth targets

h = [Link](hidden_size) # context from encoder

print("Teacher Forcing Decoder:")

for t in range(len(true_outputs)-1):
teacher_input = true_outputs[t] # true previous output
h = [Link](Wx @ teacher_input + Wh @ h)
y_pred = Wy @ h
print(f"Step {t+1}: pred={[Link](y_pred, 3)}")

Q26. 2-step Decoder from Context Vector

import numpy as np

[Link](16)
hidden_size, output_size = 4, 3

Wh = [Link](hidden_size, hidden_size) * 0.1

Wy = [Link](output_size, hidden_size) * 0.1
b = [Link](hidden_size)

context = [Link](hidden_size)
h = context

for step in range(2):

h = [Link](Wh @ h + b)
token = Wy @ h
print(f"Step {step+1} output token: {[Link](token, 4)}")

Q27. Bahdanau (Additive) Attention

import numpy as np

[Link](17)
enc_steps, enc_dim, dec_dim, attn_dim = 5, 4, 4, 6

def softmax(z):
e = [Link](z - [Link]())
return e / [Link]()

Wa = [Link](attn_dim, enc_dim) * 0.1

Ua = [Link](attn_dim, dec_dim) * 0.1
va = [Link](attn_dim) * 0.1

encoder_states = [Link](enc_steps, enc_dim)

s = [Link](dec_dim) # decoder state

scores = [Link]([va @ [Link](Wa @ h + Ua @ s) for h in encoder_states])

weights = softmax(scores)
context = weights @ encoder_states

print("Attention weights:", [Link](weights, 4))

print("Context vector :", [Link](context, 4))

Q28. Luong (Dot-product) Attention

import numpy as np
[Link](18)
enc_steps, dim = 5, 4

def softmax(z):
e = [Link](z - [Link]())
return e / [Link]()

encoder_states = [Link](enc_steps, dim)

s = [Link](dim) # decoder hidden state

scores = encoder_states @ s # dot product

weights = softmax(scores)
context = weights @ encoder_states

print("Luong attention weights:", [Link](weights, 4))

print("Context vector :", [Link](context, 4))

Q29. Attention Context Vector (4 encoder states)

import numpy as np

[Link](19)
enc_steps, dim = 4, 6

def softmax(z):
e = [Link](z - [Link]())
return e / [Link]()

encoder_states = [Link](enc_steps, dim)

s = [Link](dim)

scores = encoder_states @ s
weights = softmax(scores)
context = weights @ encoder_states

for i, (w, sc) in enumerate(zip(weights, scores)):

print(f"Encoder step {i+1}: score={sc:.4f} weight={w:.4f}")
print("Context vector:", [Link](context, 4))

Q30. Softmax Attention + Text Bar Chart

import numpy as np

[Link](20)
enc_steps = 6

def softmax(z):
e = [Link](z - [Link]())
return e / [Link]()

scores = [Link](enc_steps)
weights = softmax(scores)

print("Attention weight bar chart:")

for i, w in enumerate(weights):
bar = "#" * int(w * 40)
print(f" enc[{i+1}] {w:.4f} |{bar}")

Q31. Self-Attention on 3 Word Vectors

import numpy as np

[Link](21)
seq_len, d_model, d_k = 3, 8, 4

def softmax(z, axis=-1):

e = [Link](z - [Link](axis=axis, keepdims=True))
return e / [Link](axis=axis, keepdims=True)

X = [Link](seq_len, d_model)
Wq = [Link](d_model, d_k) * 0.1
Wk = [Link](d_model, d_k) * 0.1
Wv = [Link](d_model, d_k) * 0.1

Q = X @ Wq
K = X @ Wk
V = X @ Wv

scores = Q @ K.T / [Link](d_k)

weights = softmax(scores)
output = weights @ V

print("Attention weights:")
print([Link](weights, 4))
print("Self-attention output:")
print([Link](output, 4))

Q32. Scaled Dot-Product Attention

import numpy as np

[Link](22)
seq_len, d_k = 4, 8

def softmax(z, axis=-1):

e = [Link](z - [Link](axis=axis, keepdims=True))
return e / [Link](axis=axis, keepdims=True)

Q = [Link](seq_len, d_k)
K = [Link](seq_len, d_k)
V = [Link](seq_len, d_k)

scores = Q @ K.T / [Link](d_k) # scale by sqrt(d_k)

weights = softmax(scores)
output = weights @ V

print("Scaled attention weights:")

print([Link](weights, 4))
print("Output:")
print([Link](output, 4))

Q33. Positional Encoding (sine/cosine)

import numpy as np

def positional_encoding(max_pos, d_model):

PE = [Link]((max_pos, d_model))
pos = [Link](max_pos)[:, None]
div = [Link]([Link](0, d_model, 2) * (-[Link](10000.0) / d_model))
PE[:, 0::2] = [Link](pos * div)
PE[:, 1::2] = [Link](pos * div)
return PE

PE = positional_encoding(max_pos=5, d_model=8)
print("Positional Encoding (positions 0-4, embedding size 8):")
print([Link](PE, 4))
Q34. Single Attention Head
import numpy as np

[Link](23)
seq_len, d_model, d_k = 3, 8, 4

def softmax(z, axis=-1):

e = [Link](z - [Link](axis=axis, keepdims=True))
return e / [Link](axis=axis, keepdims=True)

X = [Link](seq_len, d_model)
Wq = [Link](d_model, d_k) * 0.1
Wk = [Link](d_model, d_k) * 0.1
Wv = [Link](d_model, d_k) * 0.1

Q = X @ Wq
K = X @ Wk
V = X @ Wv

scores = Q @ K.T / [Link](d_k)

attn = softmax(scores)
output = attn @ V

print("Attention head output shape:", [Link])

print("Output:")
print([Link](output, 4))

Q35. Multi-Head Attention (2 heads)

import numpy as np

[Link](24)
seq_len, d_model, num_heads = 4, 8, 2
d_k = d_model // num_heads # 4

def softmax(z, axis=-1):

e = [Link](z - [Link](axis=axis, keepdims=True))
return e / [Link](axis=axis, keepdims=True)

X = [Link](seq_len, d_model)
Wo = [Link](d_model, d_model) * 0.1

head_outputs = []
for h in range(num_heads):
Wq = [Link](d_model, d_k) * 0.1
Wk = [Link](d_model, d_k) * 0.1
Wv = [Link](d_model, d_k) * 0.1
Q, K, V = X @ Wq, X @ Wk, X @ Wv
out = softmax(Q @ K.T / [Link](d_k)) @ V
head_outputs.append(out)
print(f"Head {h+1} output shape: {[Link]}")

concat = [Link](head_outputs, axis=-1) # (seq_len, d_model)

MHA = concat @ Wo
print("Multi-Head Attention output shape:", [Link])
print([Link](MHA, 4))

Q36. Transformer Encoder Block

import numpy as np

[Link](25)
seq_len, d_model, num_heads, d_ff = 4, 8, 2, 16
d_k = d_model // num_heads

def softmax(z, axis=-1):

e = [Link](z - [Link](axis=axis, keepdims=True))
return e / [Link](axis=axis, keepdims=True)

def layer_norm(X, eps=1e-6):

mu = [Link](axis=-1, keepdims=True)
std = [Link](axis=-1, keepdims=True)
return (X - mu) / (std + eps)

X = [Link](seq_len, d_model)

# --- Multi-Head Self-Attention ---

Wo = [Link](d_model, d_model) * 0.1
heads = []
for _ in range(num_heads):
Wq = [Link](d_model, d_k) * 0.1
Wk = [Link](d_model, d_k) * 0.1
Wv = [Link](d_model, d_k) * 0.1
Q, K, V = X @ Wq, X @ Wk, X @ Wv
[Link](softmax(Q @ K.T / [Link](d_k)) @ V)

attn_out = [Link](heads, axis=-1) @ Wo

X = layer_norm(X + attn_out) # residual + norm

# --- Feed-Forward Network ---

W1 = [Link](d_model, d_ff) * 0.1
b1 = [Link](d_ff)
W2 = [Link](d_ff, d_model) * 0.1
b2 = [Link](d_model)

ff_out = [Link](0, X @ W1 + b1) @ W2 + b2 # ReLU

X = layer_norm(X + ff_out) # residual + norm

print("Transformer Encoder Block output shape:", [Link])

print([Link](X, 4))