Jupyter Notebooks for ML

📚 What is Jupyter?

# ── Installation ──────────────────────────────────────────────────────────────
# Install JupyterLab (recommended)
# pip install jupyterlab

# Install classic Notebook
# pip install notebook

# Install with conda (includes many data science packages)
# conda install -c conda-forge jupyterlab

# Launch JupyterLab
# jupyter lab
# jupyter lab --port=8889 --no-browser    # specific port, no auto-open

# Launch classic Notebook
# jupyter notebook

# ── IPython features available in notebooks ────────────────────────────────────
# Tab completion: type pandas. then Tab → see all methods
# ? after a function: np.random.randn?   → show docstring
# ?? after a function: np.random.randn?? → show source code
# Shift+Tab inside function parens → show signature

import numpy as np
import pandas as pd

# These work in any Jupyter cell:
# display() renders rich HTML for DataFrames
df = pd.DataFrame({'a': [1,2,3], 'b': [4.1, 5.2, 6.3]})
# display(df)      # renders as styled HTML table
# df               # last expression in cell auto-displays

# Multiple displays per cell:
# from IPython.display import display
# display(df.head(3))
# display(df.describe())

⚡ Essential Workflow

📉 Effective ML Workflows

# ── nbstripout: strip outputs before every git commit ─────────────────────────
# pip install nbstripout
# nbstripout --install     # installs git filter for current repo
# nbstripout --install --global   # for all repos

# ── jupytext: sync notebook to .py percent format ────────────────────────────
# pip install jupytext
# jupytext --to py:percent my_analysis.ipynb   # one-time conversion
# jupytext --sync my_analysis.ipynb            # sync paired files
# In notebook metadata, add: "jupytext": {"formats": "ipynb,py:percent"}
# Now both files stay in sync on save

# ── papermill: parameterised notebook execution ──────────────────────────────
# pip install papermill
# Run a notebook with parameters from command line:
# papermill template_notebook.ipynb output_run_42.ipynb \
#   -p LEARNING_RATE 0.001 \
#   -p N_EPOCHS 50 \
#   -p RANDOM_SEED 42

# In the notebook, mark the parameters cell with the tag "parameters":
# (Add tag via View > Cell Toolbar > Tags in classic Notebook)
# LEARNING_RATE = 0.01  # default value — overridden by papermill
# N_EPOCHS = 100
# RANDOM_SEED = 0

# ── Example: well-structured notebook imports cell ────────────────────────────
import warnings
warnings.filterwarnings('ignore')

import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns

# Reproducibility
RANDOM_SEED = 42
np.random.seed(RANDOM_SEED)

# Display settings
pd.set_option('display.max_columns', 50)
pd.set_option('display.float_format', '{:.4f}'.format)
plt.rcParams.update({
    'figure.figsize': (12, 5),
    'axes.grid': True,
    'grid.alpha': 0.3,
})

# Paths
DATA_DIR   = '../data'
MODELS_DIR = '../models'
FIGURES_DIR = '../figures'

print(f"numpy  {np.__version__}")
print(f"pandas {pd.__version__}")
print(f"matplotlib {matplotlib.__version__}")

🖼 Visualisation in Notebooks

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
# %matplotlib inline   # put this in a cell at notebook start

# Generate synthetic ML results data
rng = np.random.default_rng(42)
n = 300
X = rng.standard_normal((n, 2))
y = (X[:, 0] + X[:, 1] > 0).astype(int)
probs = 1 / (1 + np.exp(-(X[:, 0] + X[:, 1]))) + rng.normal(0, 0.1, n)
probs = np.clip(probs, 0, 1)

# ── 1. Training/Validation Loss Curve ────────────────────────────────────────
epochs = np.arange(1, 51)
train_loss = 1.5 * np.exp(-0.08 * epochs) + rng.normal(0, 0.02, 50)
val_loss   = 1.5 * np.exp(-0.06 * epochs) + rng.normal(0, 0.04, 50) + 0.1

fig, ax = plt.subplots(figsize=(8, 4))
ax.plot(epochs, train_loss, label='Train loss', color='#00d9ff', linewidth=2)
ax.plot(epochs, val_loss,   label='Val loss',   color='#8b5cf6', linewidth=2, linestyle='--')
ax.axvline(np.argmin(val_loss)+1, color='#f97316', linestyle=':', label=f'Best epoch={np.argmin(val_loss)+1}')
ax.set_xlabel('Epoch'); ax.set_ylabel('Loss')
ax.set_title('Training and Validation Loss', fontweight='bold')
ax.legend(); ax.grid(True, alpha=0.3)
plt.tight_layout()
# plt.savefig('loss_curve.png', dpi=150, bbox_inches='tight')
plt.show()

# ── 2. Confusion Matrix ───────────────────────────────────────────────────────
from sklearn.metrics import confusion_matrix
y_pred = (probs > 0.5).astype(int)
cm = confusion_matrix(y, y_pred)
fig, ax = plt.subplots(figsize=(5, 4))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
            xticklabels=['Pred 0','Pred 1'], yticklabels=['True 0','True 1'],
            ax=ax, linewidths=0.5)
ax.set_title('Confusion Matrix', fontweight='bold')
plt.tight_layout(); plt.show()

# ── 3. Feature Importance Bar Chart ──────────────────────────────────────────
feature_names = [f'feat_{i}' for i in range(10)]
importances   = np.abs(rng.standard_normal(10))
importances  /= importances.sum()
order = np.argsort(importances)[::-1]

fig, ax = plt.subplots(figsize=(9, 4))
ax.barh(np.array(feature_names)[order], importances[order], color='#0066ff', alpha=0.8)
ax.set_xlabel('Importance'); ax.set_title('Feature Importances', fontweight='bold')
ax.invert_yaxis()
plt.tight_layout(); plt.show()

# ── 4. Distribution by Class ─────────────────────────────────────────────────
df = pd.DataFrame({'value': X[:, 0], 'label': y.astype(str)})
fig, ax = plt.subplots(figsize=(8, 4))
for label, color in [('0', '#00d9ff'), ('1', '#f97316')]:
    sns.kdeplot(df[df.label==label]['value'], label=f'Class {label}',
                ax=ax, fill=True, alpha=0.3, color=color)
ax.set_xlabel('Feature value'); ax.set_ylabel('Density')
ax.set_title('Feature Distribution by Class', fontweight='bold')
ax.legend(); plt.tight_layout(); plt.show()

# ── 5. Correlation Heatmap ────────────────────────────────────────────────────
feature_data = pd.DataFrame(
    rng.standard_normal((200, 8)),
    columns=[f'F{i}' for i in range(8)]
)
feature_data['F3'] = feature_data['F0'] * 0.8 + rng.normal(0, 0.2, 200)

corr = feature_data.corr()
mask = np.triu(np.ones_like(corr, dtype=bool))  # upper triangle mask
fig, ax = plt.subplots(figsize=(8, 6))
sns.heatmap(corr, mask=mask, annot=True, fmt='.2f', cmap='coolwarm',
            center=0, vmin=-1, vmax=1, ax=ax, linewidths=0.5)
ax.set_title('Feature Correlation Heatmap', fontweight='bold')
plt.tight_layout(); plt.show()

🚀 Performance & Remote Notebooks

# ── Profiling a slow function ──────────────────────────────────────────────────
# In a notebook cell:
# %load_ext line_profiler
# %load_ext memory_profiler

import numpy as np

def slow_pairwise_distance(X):
    """Naive O(n^2) pairwise distances — intentionally slow."""
    n = len(X)
    dist = np.zeros((n, n))
    for i in range(n):
        for j in range(i+1, n):
            diff = X[i] - X[j]
            dist[i, j] = dist[j, i] = np.sqrt((diff**2).sum())
    return dist

def fast_pairwise_distance(X):
    """Vectorised using broadcasting — fast."""
    diff = X[:, np.newaxis, :] - X[np.newaxis, :, :]   # (n,n,d)
    return np.sqrt((diff**2).sum(axis=2))

X_small = np.random.randn(200, 10)

# In notebook: %timeit slow_pairwise_distance(X_small)
# In notebook: %timeit fast_pairwise_distance(X_small)
# Expected speedup: ~100x for n=200

# Line profiler (requires %load_ext line_profiler):
# %lprun -f slow_pairwise_distance slow_pairwise_distance(X_small)

# ── GPU check cell (put at top of GPU notebooks) ──────────────────────────────
import torch
if torch.cuda.is_available():
    gpu = torch.cuda.get_device_properties(0)
    print(f"GPU: {gpu.name}")
    print(f"VRAM: {gpu.total_memory / 1e9:.1f} GB")
    print(f"CUDA version: {torch.version.cuda}")
elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():
    print("Apple Silicon MPS available")
else:
    print("No GPU — running on CPU")

# ── Google Colab: mount Google Drive ─────────────────────────────────────────
# from google.colab import drive
# drive.mount('/content/drive')
# df = pd.read_csv('/content/drive/MyDrive/my_dataset.csv')

# ── Install packages in Colab ─────────────────────────────────────────────────
# !pip install -q transformers accelerate bitsandbytes
# !pip install -q 'xformers<0.0.27'  # specific version

# ── Colab: check GPU quota remaining ─────────────────────────────────────────
# !nvidia-smi
# from google.colab import runtime
# runtime.unassign()   # release GPU when done