{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cell-000",
   "metadata": {},
   "source": [
    "# Lesson 5 — Exploratory Data Analysis & Statistical Visualization\n",
    "\n",
    "**Companion notebook** for the Lesson 5 slides. The charts here reproduce the ones on\n",
    "the slides; the **Your turn** section asks you to build your own executive chart."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-001",
   "metadata": {},
   "outputs": [],
   "source": [
    "def check(label, got, expected, tol=1e-6):\n",
    "    \"\"\"Compare your answer with the expected value; prints ✅ or ❌ (never raises).\"\"\"\n",
    "    import math\n",
    "    try:\n",
    "        if isinstance(expected, float) or isinstance(got, float):\n",
    "            ok = math.isclose(float(got), float(expected), rel_tol=tol, abs_tol=tol)\n",
    "        elif isinstance(expected, (list, tuple)) and expected and isinstance(expected[0], str):\n",
    "            ok = list(got) == list(expected)\n",
    "        elif hasattr(expected, \"__len__\") and not isinstance(expected, str):\n",
    "            import numpy as np\n",
    "            ok = np.allclose(np.asarray(got, dtype=float), np.asarray(expected, dtype=float), rtol=tol, atol=tol)\n",
    "        else:\n",
    "            ok = got == expected\n",
    "    except Exception:\n",
    "        ok = False\n",
    "    print((\"✅ \" if ok else \"❌ \") + f\"{label}: got {got!r}, expected {expected!r}\")\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as mtick\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "TEAL, AMBER, INK, MUTED = \"#0e6b6b\", \"#c26a1b\", \"#1b2330\", \"#6b7384\"\n",
    "sns.set_theme(style=\"whitegrid\")\n",
    "\n",
    "abt = pd.read_parquet(\"../data/processed/abt.parquet\")\n",
    "claims = pd.read_csv(\"../data/motor/claims.csv\")\n",
    "claims[\"incurred\"] = claims[\"paid\"] + claims[\"outstanding\"]\n",
    "claims = claims.merge(abt[[\"vehicle_id\", \"vehicle_style\", \"age_band\"]], on=\"vehicle_id\", validate=\"many_to_one\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-002",
   "metadata": {},
   "source": [
    "## 1. Claim sizes: linear vs. log scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-003",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(11, 4))\n",
    "axes[0].hist(claims[\"incurred\"], bins=60, color=TEAL)\n",
    "axes[0].set(title=\"Linear scale\", xlabel=\"Incurred per claim\")\n",
    "axes[1].hist(claims[\"incurred\"], bins=np.logspace(0, 5.1, 60), color=AMBER)\n",
    "axes[1].set_xscale(\"log\")\n",
    "axes[1].set(title=\"Log scale\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-004",
   "metadata": {},
   "source": [
    "## 2. Comparing distributions: KDE by coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-005",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(9, 4))\n",
    "sns.kdeplot(data=claims, x=\"incurred\", hue=\"coverage\", log_scale=True, common_norm=False, fill=True,\n",
    "            alpha=0.25, ax=ax)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-006",
   "metadata": {},
   "source": [
    "## 3. Discrete counts: bars, not histograms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-007",
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = abt[\"n_claims\"].astype(int).clip(upper=3).value_counts().sort_index()\n",
    "fig, ax = plt.subplots(figsize=(6, 3.5))\n",
    "bars = ax.bar([\"0\", \"1\", \"2\", \"3+\"], counts.values, color=TEAL)\n",
    "ax.bar_label(bars, labels=[f\"{v:,}\" for v in counts.values])\n",
    "ax.set_yscale(\"log\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-008",
   "metadata": {},
   "source": [
    "## 4. Non-linear trends: frequency by driver age, with exposure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-009",
   "metadata": {},
   "outputs": [],
   "source": [
    "abt[\"age_2y\"] = (abt[\"youngest_driver_age\"] // 2 * 2).clip(17, 85)\n",
    "by_age = abt.groupby(\"age_2y\").agg(exposure=(\"exposure\", \"sum\"), claims=(\"n_claims\", \"sum\"))\n",
    "by_age[\"frequency\"] = by_age[\"claims\"] / by_age[\"exposure\"]\n",
    "by_age = by_age[by_age[\"exposure\"] >= 50]\n",
    "fig, ax = plt.subplots(figsize=(10, 4))\n",
    "ax2 = ax.twinx()\n",
    "ax2.bar(by_age.index, by_age[\"exposure\"], width=1.6, color=MUTED, alpha=0.25)\n",
    "ax2.grid(False)\n",
    "ax.plot(by_age.index, by_age[\"frequency\"], marker=\"o\", color=AMBER)\n",
    "ax.set_zorder(ax2.get_zorder() + 1)\n",
    "ax.patch.set_visible(False)\n",
    "ax.set(title=\"Claim frequency by youngest driver age\", xlabel=\"Age\", ylabel=\"Frequency\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-010",
   "metadata": {},
   "source": [
    "## 5. What correlation misses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-011",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(abt[[\"youngest_driver_age\", \"n_claims\"]].corr(method=\"spearman\").round(3))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-012",
   "metadata": {},
   "source": [
    "A correlation near zero, yet the chart above shows a strong U-shaped effect.\n",
    "\n",
    "---\n",
    "## Your turn 1 — a styling helper\n",
    "\n",
    "Write `style_axes(ax, title, source)` that: sets a **left-aligned** bold title, adds\n",
    "the source note below the axes in small gray text, and hides the top and right\n",
    "spines. Use it on every chart you make from now on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-013",
   "metadata": {},
   "outputs": [],
   "source": [
    "def style_axes(ax, title: str, source: str):\n",
    "    ...  # your code here\n",
    "    return ax\n",
    "\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(6, 3.5))\n",
    "ax.bar([\"A\", \"B\"], [1, 2], color=TEAL)\n",
    "style_axes(ax, \"A headline title\", \"Source: example\")\n",
    "check(\"title is left-aligned\", ax.get_title(loc=\"left\"), \"A headline title\")\n",
    "check(\"top spine hidden\", ax.spines[\"top\"].get_visible(), False)\n",
    "plt.close(fig)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cell-014",
   "metadata": {},
   "source": [
    "## Your turn 2 — an executive chart\n",
    "\n",
    "Pure premium by age band as bars, with a currency axis, loss-ratio labels on the bars,\n",
    "and a headline title stating the finding. First compute `seg` (index `age_band`,\n",
    "columns `pure_premium` and `loss_ratio`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cell-015",
   "metadata": {},
   "outputs": [],
   "source": [
    "seg = None\n",
    "...  # your code here\n",
    "check(\"pure premium 17-24\", None if seg is None else round(seg.loc[\"17-24\", \"pure_premium\"]), 1699)\n",
    "\n",
    "if seg is not None:\n",
    "    fig, ax = plt.subplots(figsize=(7, 3.8))\n",
    "    bars = ax.bar(seg.index.astype(str), seg[\"pure_premium\"],\n",
    "                  color=[AMBER if lr > 0.76 else TEAL for lr in seg[\"loss_ratio\"]])\n",
    "    ax.bar_label(bars, labels=[f\"LR {lr:.0%}\" for lr in seg[\"loss_ratio\"]], padding=3)\n",
    "    ax.yaxis.set_major_formatter(mtick.StrMethodFormatter(\"${x:,.0f}\"))\n",
    "    ax.set_ylim(0, seg[\"pure_premium\"].max() * 1.15)\n",
    "    style_axes(ax, \"Young drivers cost 50% more per car year\", \"Source: motor ABT, incurred at 31 Dec 2025.\")\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}