{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cell-000",
   "metadata": {},
   "source": [
    "# Lesson 8A — Integration, Delivery & Monitoring\n",
    "\n",
    "**Companion notebook** for the Lesson 8A slides. The pieces of the platform meet here:\n",
    "a model **artifact** that can prove what it is, a **promotion rule** that decides when a\n",
    "new model may replace the champion, **drift monitoring** on the quotes coming in, and a\n",
    "CI/CD workflow that builds and smoke-tests the image before anything ships."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-001",
   "metadata": {},
   "outputs": [],
   "source": [
    "def check(label, got, expected, tol=1e-6):\n",
    "    \"\"\"Compare your answer with the expected value; prints ✅ or ❌ (never raises).\"\"\"\n",
    "    import math\n",
    "    try:\n",
    "        if isinstance(expected, float) or isinstance(got, float):\n",
    "            ok = math.isclose(float(got), float(expected), rel_tol=tol, abs_tol=tol)\n",
    "        elif isinstance(expected, (list, tuple)) and expected and isinstance(expected[0], str):\n",
    "            ok = list(got) == list(expected)\n",
    "        elif hasattr(expected, \"__len__\") and not isinstance(expected, str):\n",
    "            import numpy as np\n",
    "            ok = np.allclose(np.asarray(got, dtype=float), np.asarray(expected, dtype=float), rtol=tol, atol=tol)\n",
    "        else:\n",
    "            ok = got == expected\n",
    "    except Exception:\n",
    "        ok = False\n",
    "    print((\"✅ \" if ok else \"❌ \") + f\"{label}: got {got!r}, expected {expected!r}\")\n",
    "import hashlib\n",
    "import json\n",
    "import shutil\n",
    "import tempfile\n",
    "from pathlib import Path\n",
    "\n",
    "import joblib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import yaml\n",
    "from pydantic import ValidationError\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.linear_model import GammaRegressor, PoissonRegressor\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import FunctionTransformer, OneHotEncoder\n",
    "\n",
    "MOTOR = Path(\"../data/motor\")\n",
    "EDGES = [16, 21, 25, 30, 40, 50, 65, 75, 121]\n",
    "LABELS = [\"17-20\", \"21-24\", \"25-29\", \"30-39\", \"40-49\", \"50-64\", \"65-74\", \"75+\"]\n",
    "\n",
    "\n",
    "def log_miles(m):\n",
    "    return np.log(m / 12_000)\n",
    "\n",
    "\n",
    "def glm(columns, family):\n",
    "    cats = [c for c in (\"vehicle_style\", \"age_band\") if c in columns]\n",
    "    steps = [(\"cat\", OneHotEncoder(drop=[{\"vehicle_style\": \"Sedan\", \"age_band\": \"40-49\"}[c] for c in cats],\n",
    "                                   handle_unknown=\"ignore\", sparse_output=False), cats),\n",
    "             (\"urban\", \"passthrough\", [\"urban\"])]\n",
    "    if \"annual_mileage\" in columns:\n",
    "        steps.append((\"miles\", FunctionTransformer(log_miles), [\"annual_mileage\"]))\n",
    "    return Pipeline([(\"prep\", ColumnTransformer(steps)),\n",
    "                     (\"glm\", family(alpha=1e-4, solver=\"newton-cholesky\", tol=1e-8, max_iter=1000))])\n",
    "\n",
    "\n",
    "class PurePremiumModel:\n",
    "    \"\"\"Frequency x severity GLMs, balanced to the training data (Course 3).\"\"\"\n",
    "\n",
    "    def fit(self, frame):\n",
    "        f = frame.assign(age_band=pd.cut(frame[\"driver_age\"], EDGES, right=False, labels=LABELS).astype(str))\n",
    "        self.freq = glm([\"vehicle_style\", \"age_band\", \"urban\", \"annual_mileage\"], PoissonRegressor)\n",
    "        self.freq.fit(f, f[\"n_liab\"] / f[\"exposure\"], glm__sample_weight=f[\"exposure\"])\n",
    "        c = f[f[\"n_liab\"] > 0]\n",
    "        self.sev = glm([\"vehicle_style\", \"urban\"], GammaRegressor)\n",
    "        self.sev.fit(c, c[\"inc_liab\"] / c[\"n_liab\"], glm__sample_weight=c[\"n_liab\"])\n",
    "        self.fs = f[\"n_liab\"].sum() / (self.freq.predict(f) * f[\"exposure\"]).sum()\n",
    "        self.ss = c[\"inc_liab\"].sum() / (self.sev.predict(c) * c[\"n_liab\"]).sum()\n",
    "        return self\n",
    "\n",
    "    def pure_premium(self, frame):\n",
    "        f = frame.assign(age_band=pd.cut(frame[\"driver_age\"], EDGES, right=False, labels=LABELS).astype(str))\n",
    "        return self.freq.predict(f) * self.fs * self.sev.predict(f) * self.ss\n",
    "\n",
    "\n",
    "def gini(losses, predicted, exposure):\n",
    "    order = np.argsort(predicted)\n",
    "    cum_e = np.r_[0, np.cumsum(exposure[order]) / exposure.sum()]\n",
    "    cum_l = np.r_[0, np.cumsum(losses[order]) / losses.sum()]\n",
    "    return float(1 - 2 * np.trapezoid(cum_l, cum_e))\n",
    "\n",
    "\n",
    "frame = pd.read_parquet(MOTOR / \"model_frame.parquet\")\n",
    "train, hold = frame[frame[\"policy_year\"] < 2025], frame[frame[\"policy_year\"] == 2025]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-002",
   "metadata": {},
   "source": [
    "---\n",
    "## Your turn 1 — validate, then build the artifact\n",
    "\n",
    "1. Fit on 2021–2024 and score the 2025 holdout: actual ÷ expected losses and the Gini.\n",
    "2. Refit on **all** years (the production model), `joblib.dump` it to\n",
    "   `ARTIFACT / \"model.joblib\"`, and write `ARTIFACT / \"manifest.json\"` with the file's\n",
    "   SHA-256, `sklearn_version`, the holdout metrics, `trained_rows`, and\n",
    "   `version = \"glm-\" + sha256[:10]`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-003",
   "metadata": {},
   "outputs": [],
   "source": [
    "ARTIFACT = Path(tempfile.mkdtemp()) / \"champion\"\n",
    "ae = g = manifest = None\n",
    "...  # your code here\n",
    "check(\"holdout A/E\", None if ae is None else round(ae, 4), 1.0143)\n",
    "check(\"holdout Gini\", None if g is None else round(g, 4), 0.2753)\n",
    "check(\"version comes from the checksum\",\n",
    "      None if manifest is None else manifest[\"version\"] == \"glm-\" + manifest[\"sha256\"][:10], True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-004",
   "metadata": {},
   "source": [
    "## Your turn 2 — load only what the manifest vouches for\n",
    "\n",
    "Write `load_verified(folder)`: recompute the SHA-256 and raise `RuntimeError` if it\n",
    "differs from the manifest, and also if the major.minor scikit-learn version differs;\n",
    "only then `joblib.load`. Test it on the artifact, a copy with one extra byte, and a copy\n",
    "whose manifest claims scikit-learn 0.24."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-005",
   "metadata": {},
   "outputs": [],
   "source": [
    "outcomes = None\n",
    "...  # your code here\n",
    "check(\"genuine, tampered, old version\", outcomes, [\"loaded\", \"refused\", \"refused\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-006",
   "metadata": {},
   "source": [
    "## Your turn 3 — a promotion rule\n",
    "\n",
    "A challenger replaces the champion only if **all** hold on the same holdout:\n",
    "- its A/E is within 5% of 1 (0.95 to 1.05)\n",
    "- its Gini beats the champion's by at least 0.005\n",
    "- it passed its tests (`tests_passed` is True)\n",
    "\n",
    "Write `promote(champion, challenger) -> bool` over dicts like the ones below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-007",
   "metadata": {},
   "outputs": [],
   "source": [
    "champion = {\"ae\": 1.014, \"gini\": 0.275, \"tests_passed\": True}\n",
    "CASES = [\n",
    "    {\"ae\": 1.002, \"gini\": 0.291, \"tests_passed\": True},    # better\n",
    "    {\"ae\": 1.043, \"gini\": 0.235, \"tests_passed\": True},    # the Course 3 GBM\n",
    "    {\"ae\": 0.930, \"gini\": 0.300, \"tests_passed\": True},    # sharper, but under-predicts 7%\n",
    "    {\"ae\": 1.000, \"gini\": 0.278, \"tests_passed\": True},    # not enough better\n",
    "    {\"ae\": 1.000, \"gini\": 0.320, \"tests_passed\": False},   # failing tests\n",
    "]\n",
    "promote = None\n",
    "...  # your code here\n",
    "check(\"decisions\", None if promote is None else [promote(champion, c) for c in CASES],\n",
    "      [True, False, False, False, False])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-008",
   "metadata": {},
   "source": [
    "## Your turn 4 — is the business still quoting the risks the model knows?\n",
    "\n",
    "The Population Stability Index compares the share of quotes in each bin with the share\n",
    "in a reference population:\n",
    "\n",
    "PSI = Σ (current − reference) × ln(current ÷ reference)\n",
    "\n",
    "(floor both shares at 0.0001 so an empty bin doesn't divide by zero). Below 0.10 is\n",
    "stable, 0.10–0.25 watch, above 0.25 investigate.\n",
    "\n",
    "Write `psi(reference, current)` for two Series of shares, and `drift(reference_df,\n",
    "current_df)` returning `{feature: psi}` for the four rating variables binned as the\n",
    "manual bins them (`BINS`; `None` means categorical). Compare the 2025 book with (a) the\n",
    "valid new-business quote requests and (b) the same week after a marketing campaign\n",
    "brought in 400 young sports-car drivers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-009",
   "metadata": {},
   "outputs": [],
   "source": [
    "BINS = {\"driver_age\": [16, 25, 30, 65, 75, 121], \"annual_mileage\": [0, 7_500, 15_000, float(\"inf\")],\n",
    "        \"vehicle_style\": None, \"urban\": None}\n",
    "book = pd.read_csv(MOTOR / \"book_2025.csv\")\n",
    "requests = pd.read_csv(MOTOR / \"quote_requests.csv\")\n",
    "ok = (requests[\"driver_age\"].between(16, 110) & requests[\"annual_mileage\"].gt(0) & requests[\"vehicle_value\"].gt(0)\n",
    "      & requests[\"vehicle_style\"].isin([\"Sedan\", \"SUV\", \"Hatchback\", \"Pickup\", \"Minivan\", \"Sports\"]))\n",
    "new_business = requests[ok].copy()\n",
    "campaign = new_business.head(400).assign(driver_age=[19.0 + i % 5 for i in range(400)], vehicle_style=\"Sports\")\n",
    "after_campaign = pd.concat([campaign, new_business.tail(600)])\n",
    "print(len(new_business), \"new-business requests pass these checks\")\n",
    "\n",
    "psi = drift = None\n",
    "...  # your code here\n",
    "d1 = None if drift is None else drift(book, new_business)\n",
    "d2 = None if drift is None else drift(book, after_campaign)\n",
    "check(\"new business: largest PSI is below 0.10\", None if d1 is None else max(d1.values()) < 0.10, True)\n",
    "check(\"campaign: driver age PSI\", None if d2 is None else d2[\"driver_age\"], 0.6674)\n",
    "check(\"campaign: vehicle style PSI\", None if d2 is None else d2[\"vehicle_style\"], 0.9045)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-010",
   "metadata": {},
   "source": [
    "## Your turn 5 — continuous delivery\n",
    "\n",
    "Write `WORKFLOW` with two jobs:\n",
    "- `checks` (lint, types, tests: as in Lesson 2)\n",
    "- `images`, which **needs** `checks`, builds the API image with\n",
    "  `docker/build-push-action@v7` (`target: api`), then runs it and `curl`s `/health` as a\n",
    "  smoke test\n",
    "\n",
    "`cd_ok` verifies the shape."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-011",
   "metadata": {},
   "outputs": [],
   "source": [
    "WORKFLOW = \"\"\n",
    "...  # your code here\n",
    "\n",
    "\n",
    "def cd_ok(text):\n",
    "    jobs = (yaml.safe_load(text) or {}).get(\"jobs\", {})\n",
    "    images = jobs.get(\"images\", {})\n",
    "    steps = images.get(\"steps\", [])\n",
    "    build = [s for s in steps if s.get(\"uses\", \"\").startswith(\"docker/build-push-action\")]\n",
    "    return (\"checks\" in jobs and images.get(\"needs\") in (\"checks\", [\"checks\"])\n",
    "            and bool(build) and build[0].get(\"with\", {}).get(\"target\") == \"api\"\n",
    "            and any(\"/health\" in s.get(\"run\", \"\") for s in steps))\n",
    "\n",
    "\n",
    "check(\"delivery workflow\", cd_ok(WORKFLOW), True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}