Neural Networks & GenAI Intuition

import math

# ── A single artificial neuron ─────────────────────────────────────────
# This is the core building block of every neural network.
# In practice, frameworks like PyTorch or TensorFlow handle this
# internally — but understanding it from scratch is essential.

def neuron_forward(inputs: list, weights: list, bias: float) -> float:
    """
    Computes the output of a single neuron.
    
    inputs  : list of input values [x1, x2, x3, ...]
    weights : list of weights      [w1, w2, w3, ...]  (same length as inputs)
    bias    : single float value
    returns : the neuron's output after activation
    """

    # Step 1: Compute the weighted sum  z = Σ(xi * wi) + b
    z = sum(x * w for x, w in zip(inputs, weights)) + bias

    # Step 2: Apply the activation function
    # Here we use ReLU (Rectified Linear Unit) — explained in the next section
    # ReLU: output = max(0, z)  — fires if z > 0, silent if z ≤ 0
    output = max(0.0, z)

    return output


# ── Let's trace through the diagram example exactly ──────────────────
inputs  = [0.8, 0.3, 1.0]   # x1, x2, x3
weights = [0.5, 1.2, -0.7]  # w1, w2, w3
bias    = 0.1

z = (inputs[0] * weights[0]  # 0.8 * 0.5  = 0.40
   + inputs[1] * weights[1]  # 0.3 * 1.2  = 0.36
   + inputs[2] * weights[2]  # 1.0 * -0.7 = -0.70
   + bias)                       #              + 0.10
                                 # z = 0.16

output = neuron_forward(inputs, weights, bias)
print(f"Pre-activation z: {z:.4f}")       # 0.1600
print(f"Neuron output:   {output:.4f}")   # 0.1600 (ReLU: max(0, 0.16) = 0.16)

# If z had been negative, say z = -0.3:
# output = max(0.0, -0.3) = 0.0  ← neuron is "silent" (doesn't fire)

import math

# ── Activation functions implemented from scratch ──────────────────────
# These are the exact functions used inside neural networks.
# In PyTorch: torch.relu(), torch.sigmoid(), torch.softmax() etc.

def relu(z: float) -> float:
    # Fires if positive, silent if zero or negative
    return max(0.0, z)

def sigmoid(z: float) -> float:
    # Squishes any value into (0.0, 1.0)  →  interpretable as probability
    return 1.0 / (1.0 + math.exp(-z))

def tanh(z: float) -> float:
    # Squishes any value into (-1.0, 1.0)  →  zero-centred
    return math.tanh(z)

def softmax(scores: list) -> list:
    # Converts a list of raw scores into probabilities that sum to 1.0
    exps = [math.exp(s) for s in scores]
    total = sum(exps)
    return [e / total for e in exps]


# ── See what each does with a few values ──────────────────────────────
test_values = [-2.0, -0.5, 0.0, 0.5, 2.0]
print("z      | ReLU  | Sigmoid | Tanh")
print("-" * 38)
for z in test_values:
    print(f"{z:+.1f}  | {relu(z):.3f} | {sigmoid(z):.3f}   | {tanh(z):.3f}")

# Output:
# z      | ReLU  | Sigmoid | Tanh
# --------------------------------------
# -2.0  | 0.000 | 0.119   | -0.964
# -0.5  | 0.000 | 0.378   | -0.462
#  0.0  | 0.000 | 0.500   |  0.000
# +0.5  | 0.500 | 0.622   |  0.462
# +2.0  | 2.000 | 0.880   |  0.964
#
# Key observation: ReLU is either 0 or z (hard gate)
# Sigmoid is smooth 0→1 (probability-like)
# Tanh is smooth -1→1 (zero-centred)

# ── Softmax for multi-class output ────────────────────────────────────
raw_scores = [2.0, 1.0, 0.1]   # raw logits for 3 classes: cat / dog / bird
probs = softmax(raw_scores)
print(f"\nRaw scores: {raw_scores}")
print(f"Softmax →   cat:{probs[0]:.3f}  dog:{probs[1]:.3f}  bird:{probs[2]:.3f}")
print(f"Sum of probs: {sum(probs):.3f}")  # always 1.000

# Output:
# Raw scores: [2.0, 1.0, 0.1]
# Softmax →   cat:0.659  dog:0.242  bird:0.099
# Sum of probs: 1.000
#
# Interpretation: this network is 65.9% confident it's a cat.
# Testing implication: the threshold you set to declare "cat" (e.g., >0.5)
# directly affects precision vs recall — a testing decision, not just a
# model decision. CT-AI Ch.3 covers this in the confusion matrix section.

import math

# ── Activation functions (defined earlier) ────────────────────────────
def relu(z):    return max(0.0, z)
def sigmoid(z): return 1.0 / (1.0 + math.exp(-z))


# ════════════════════════════════════════════════════════════════════════
#  NETWORK ARCHITECTURE:
#
#   Input Layer:  x = [x1, x2]
#                      x1 = word_count (e.g., 0.4 after normalisation)
#                      x2 = num_links  (e.g., 0.9 after normalisation)
#
#   Hidden Layer: 2 neurons (h1, h2) with ReLU activation
#
#   Output Layer: 1 neuron (y) with Sigmoid activation
#                 output = probability of spam (0 to 1)
# ════════════════════════════════════════════════════════════════════════

# ── INPUTS (normalised feature values) ────────────────────────────────
x1 = 0.4   # word_count: moderate length email
x2 = 0.9   # num_links:  lots of links → suspicious

# ── LEARNED WEIGHTS (pretend training set these) ───────────────────────
# Hidden layer weights: each hidden neuron gets 2 inputs (x1, x2)
w_h1_x1, w_h1_x2, b_h1 = 0.8,  1.5,  -0.3   # hidden neuron 1
w_h2_x1, w_h2_x2, b_h2 = -0.5, 1.2,   0.1   # hidden neuron 2

# Output layer weights: 1 output neuron gets 2 inputs (h1, h2)
w_y_h1, w_y_h2, b_y = 1.1, 0.9, -0.5


# ════════════════════════════════════════════════════════════════════════
#  FORWARD PASS — compute layer by layer
# ════════════════════════════════════════════════════════════════════════

# ── Hidden Layer: compute each neuron's pre-activation (z) and output ──
z_h1 = w_h1_x1 * x1 + w_h1_x2 * x2 + b_h1
# z_h1 = 0.8*0.4 + 1.5*0.9 + (-0.3)
#       = 0.32   + 1.35    - 0.30
#       = 1.37
h1 = relu(z_h1)   # relu(1.37) = 1.37   (positive → passes through)

z_h2 = w_h2_x1 * x1 + w_h2_x2 * x2 + b_h2
# z_h2 = -0.5*0.4 + 1.2*0.9 + 0.1
#       = -0.20   + 1.08    + 0.10
#       = 0.98
h2 = relu(z_h2)   # relu(0.98) = 0.98   (positive → passes through)

print(f"Hidden neuron 1: z={z_h1:.3f} → after ReLU: {h1:.3f}")  # 1.370 → 1.370
print(f"Hidden neuron 2: z={z_h2:.3f} → after ReLU: {h2:.3f}")  # 0.980 → 0.980


# ── Output Layer: one neuron combines hidden layer outputs ─────────────
z_y = w_y_h1 * h1 + w_y_h2 * h2 + b_y
# z_y = 1.1*1.37 + 0.9*0.98 + (-0.5)
#      = 1.507   + 0.882    - 0.500
#      = 1.889
y = sigmoid(z_y)  # sigmoid(1.889) ≈ 0.868

print(f"Output neuron:   z={z_y:.3f} → after Sigmoid: {y:.3f}")   # 1.889 → 0.868
print()
print(f"─── PREDICTION ────────────────────────────")
print(f"Spam probability: {y:.1%}")      # 86.8%
print(f"Decision (threshold 0.5): {'SPAM' if y > 0.5 else 'NOT SPAM'}")  # SPAM

# ─────────────────────────────────────────────────────────────────────
# The network says: this email has an 86.8% probability of being spam.
# Given lots of links (x2=0.9) — the weights in h1 and h2 that emphasise
# the link count drove this prediction. That makes intuitive sense.
#
# TESTING NOTE: What if the threshold was 0.7 instead of 0.5?
# Then 0.868 > 0.7 → still SPAM.
# But what if y = 0.55? At threshold 0.5 → SPAM. At 0.7 → NOT SPAM.
# Threshold selection is a TESTING DECISION that affects Precision/Recall.
# ─────────────────────────────────────────────────────────────────────

import torch
import torch.nn as nn

# ── Define the same tiny network using PyTorch ─────────────────────────
class SpamClassifier(nn.Module):
    def __init__(self):
        super().__init__()
        # nn.Linear(in_features, out_features) creates a fully-connected layer
        # with weights and biases automatically — no manual w_h1_x1 etc.
        self.hidden = nn.Linear(2, 2)  # 2 inputs → 2 hidden neurons
        self.output = nn.Linear(2, 1)  # 2 hidden → 1 output neuron

    def forward(self, x):
        # This IS the forward pass — called automatically during training
        h = torch.relu(self.hidden(x))       # hidden layer + ReLU
        y = torch.sigmoid(self.output(h))    # output layer + Sigmoid
        return y


model     = SpamClassifier()
loss_fn   = nn.BCELoss()              # Binary Cross-Entropy — for spam (0/1) labels
optimiser = torch.optim.SGD(          # SGD = Stochastic Gradient Descent
    model.parameters(),                 # all weights and biases to update
    lr=0.01                            # learning rate α — hyperparameter
)

# ── Simulated training batch (4 emails) ───────────────────────────────
X_train = torch.tensor([
    [0.4, 0.9],   # email 1: moderate words, many links → spam
    [0.8, 0.1],   # email 2: many words, few links   → not spam
    [0.2, 0.8],   # email 3: few words, many links   → spam
    [0.9, 0.2],   # email 4: many words, few links   → not spam
])
y_train = torch.tensor([[1.0], [0.0], [1.0], [0.0]])  # labels


# ── Training loop — 5 epochs for illustration ─────────────────────────
for epoch in range(5):

    # ① FORWARD PASS — get predictions
    predictions = model(X_train)

    # ② COMPUTE LOSS — how wrong are we?
    loss = loss_fn(predictions, y_train)

    # ③ BACKWARD PASS — backpropagation computes gradients for all weights
    optimiser.zero_grad()   # clear gradients from previous step (don't accumulate)
    loss.backward()         # ← this ONE LINE does all the backpropagation math

    # ④ UPDATE WEIGHTS — gradient descent nudges all weights
    optimiser.step()        # ← applies: w = w - α * gradient(w) for all weights

    print(f"Epoch {epoch+1}: loss = {loss.item():.4f}")

# Typical output (loss should decrease each epoch):
# Epoch 1: loss = 0.7124   ← starting random, very wrong
# Epoch 2: loss = 0.6891
# Epoch 3: loss = 0.6673
# Epoch 4: loss = 0.6468
# Epoch 5: loss = 0.6276   ← improving, but needs many more epochs
#
# After 500-1000 epochs with real data, loss would drop much further.
# loss.backward() is doing ALL the calculus automatically.
# This is what makes frameworks like PyTorch and TensorFlow powerful.

import torch
import torch.nn as nn

# ── Network definition ────────────────────────────────────────────────
class SpamNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.layer1 = nn.Linear(2, 8)   # 2 inputs  → 8 hidden neurons
        self.layer2 = nn.Linear(8, 4)   # 8 hidden  → 4 hidden neurons
        self.out    = nn.Linear(4, 1)   # 4 hidden  → 1 output (spam probability)

    def forward(self, x):
        x = torch.relu(self.layer1(x))   # hidden layer 1: weighted sum + ReLU
        x = torch.relu(self.layer2(x))   # hidden layer 2: weighted sum + ReLU
        x = torch.sigmoid(self.out(x))   # output: sigmoid → probability
        return x

model     = SpamNet()
loss_fn   = nn.BCELoss()
optimiser = torch.optim.Adam(model.parameters(), lr=0.001)
# Adam is a smarter gradient descent — adapts learning rate per weight


# ── Synthetic training data (20 emails) ───────────────────────────────
torch.manual_seed(42)  # for reproducibility in this demo
X = torch.rand(20, 2)  # 20 emails × 2 features (word_count, link_count)
# Spam rule: link count (X[:,1]) > 0.6 → spam (label = 1)
y = (X[:, 1] > 0.6).float().unsqueeze(1)


# ── PHASE 1: TRAINING ─────────────────────────────────────────────────
model.train()  # sets model to training mode (enables dropout, batchnorm etc.)

for epoch in range(200):
    predictions = model(X)
    loss        = loss_fn(predictions, y)
    optimiser.zero_grad()
    loss.backward()
    optimiser.step()
    if (epoch + 1) % 50 == 0:
        print(f"Epoch {epoch+1:3d}: loss = {loss.item():.4f}")

# Output (loss decreasing is what we want):
# Epoch  50: loss = 0.5832
# Epoch 100: loss = 0.4601
# Epoch 150: loss = 0.3244
# Epoch 200: loss = 0.2019  ← converging


# ── PHASE 2: EVALUATION ───────────────────────────────────────────────
model.eval()  # sets model to evaluation mode
with torch.no_grad():  # no gradients needed for evaluation — saves memory
    preds     = model(X)
    predicted = (preds > 0.5).float()      # apply classification threshold
    correct   = (predicted == y).sum().item()
    accuracy  = correct / len(y) * 100
    print(f"\nTraining accuracy: {accuracy:.1f}%")

# IMPORTANT: This is training accuracy, NOT test accuracy.
# Training accuracy always looks good because the model has seen this data.
# For honest evaluation you NEED a separate test dataset the model has
# never seen. CT-AI Ch.3 and Ch.5 cover why this split is critical.


# ── PHASE 3: INFERENCE (production use) ───────────────────────────────
model.eval()
with torch.no_grad():
    new_emails = torch.tensor([
        [0.3, 0.85],   # short email, many links → likely spam
        [0.9, 0.05],   # long email, few links  → likely not spam
        [0.5, 0.55],   # borderline — close to 0.5 threshold!
    ])
    spam_probs = model(new_emails)
    for i, (email, prob) in enumerate(zip(new_emails, spam_probs)):
        label = "SPAM" if prob > 0.5 else "NOT SPAM"
        print(f"Email {i+1} [links={email[1]:.2f}] → P(spam)={prob.item():.3f} → {label}")

# The borderline email (P=~0.55) shows why threshold testing matters:
# threshold=0.50 → SPAM
# threshold=0.60 → NOT SPAM
# Same model, different threshold, different decision.
# A tester must verify the threshold is set appropriately for the use case.

# ── What a tokeniser does (conceptual) ────────────────────────────────
# Real tokenisers use BPE (Byte-Pair Encoding) or SentencePiece algorithms.
# Here we illustrate the concept with a tiny vocabulary.

# Step 1: Vocabulary — every possible token maps to an ID
vocabulary = {
    "Testing": 23456,
    "AI":      7890,
    "is":      318,
    "chall":   3164,
    "enging":  3181,
    # ... 50,000+ more tokens in a real LLM vocabulary
}

# Step 2: Tokenise the input text
input_text   = "Testing AI is challenging"
token_ids    = [23456, 7890, 318, 3164, 3181]  # from the tokeniser
print(f"Input: '{input_text}'")
print(f"Token IDs: {token_ids}")

# Step 3: Embedding — each token ID becomes a vector of numbers
# In GPT-4, each token becomes a vector of ~12,288 numbers.
# For illustration, we use 4-dimensional embeddings:
embedding_table = {
    23456: [0.12, -0.45, 0.78, 0.33],  # "Testing" embedding
    7890:  [0.88,  0.21, 0.09, -0.64],  # "AI" embedding
    318:   [0.01, -0.02, 0.03, -0.01],  # "is" embedding (simple stop word)
    # ... 50,000+ more embedding vectors, all learned during training
}

embeddings = [embedding_table[tid] for tid in token_ids if tid in embedding_table]
print(f"\nEmbeddings (4-dim for illustration):")
for token, emb in zip(["Testing", "AI", "is"], embeddings):
    print(f"  '{token}' → {emb}")

# These vectors are what actually enters the Transformer layers.
# The model processes ALL token embeddings simultaneously (unlike older
# sequential RNNs), and learns to relate them via the attention mechanism.

# ── Calling an LLM API — real-world testing context ───────────────────
# This is how you'd interact with an LLM in practice.
# Key inputs a tester must control and document:
#   - model (version!) 
#   - system prompt
#   - user prompt
#   - temperature
#   - max_tokens

import anthropic  # or: from openai import OpenAI

client = anthropic.Anthropic()  # reads ANTHROPIC_API_KEY from environment

def call_llm(prompt: str, temperature: float) -> str:
    """Call the LLM and return its text response."""
    response = client.messages.create(
        model="claude-sonnet-4-6",          # always pin a specific model version!
        max_tokens=100,                    # limit output length
        temperature=temperature,            # the randomness knob (0.0 – 1.0)
        messages=[
            {"role": "user", "content": prompt}
        ]
    )
    return response.content[0].text


# ── Demo: same prompt, different temperatures ──────────────────────────
prompt = "Complete this sentence: The capital of France is"

print("Testing non-determinism — same prompt, three runs at temperature=1.0:")
for run in range(3):
    output = call_llm(prompt, temperature=1.0)
    print(f"  Run {run+1}: {output.strip()}")
# At temp=1.0, outputs might vary in wording/length but all say "Paris"
# For creative prompts, the variation would be far more dramatic

print("\nDeterministic run at temperature=0.0:")
for run in range(3):
    output = call_llm(prompt, temperature=0.0)
    print(f"  Run {run+1}: {output.strip()}")
# At temp=0.0, all three runs produce identical output


# ── TESTER INSIGHT ─────────────────────────────────────────────────────
# For reproducible tests: set temperature=0.0
# For testing the full distribution of model behaviour: use temp > 0 and
# run many samples to characterise output variability
# 
# CT-AI Ch.4 key point: you cannot use a single LLM response as a 
# definitive pass/fail. You need to assess outputs statistically or use
# boundary criteria (output must be within acceptable range, not identical).


# ── Testing the "input explosion" problem ─────────────────────────────
# CT-AI Ch.4 notes the input space of an LLM is enormous:
# system prompt × user prompt × temperature × model version × context history

test_cases = [
    {"system": "You are a geography expert.",     "user": "Capital of France?"},
    {"system": "Answer only in Spanish.",          "user": "Capital of France?"},
    {"system": "Be extremely brief.",             "user": "Capital of France?"},
    {"system": "You are a chef. Discuss food.",    "user": "Capital of France?"},
]
# Same user prompt → wildly different outputs depending on system prompt.
# This illustrates why black-box coverage strategies (CT-AI Ch.4) for LLMs
# must account for the system prompt as a test dimension.