API Overview and Data Completeness

# --- COLLAPSIBLE SETUP BLOCK ---
# Standard imports plus the project helpers. We extend sys.path so the book can
# import book_utils whether it is executed from the repo root or the chapters dir.
import pandas as pd
import numpy as np
import plotly.express as px
import sys
from pathlib import Path

cwd = Path.cwd().resolve()
sys.path[:0] = [str(cwd), str(cwd.parent)]

from book_utils import api_get, fetch_all, congress_range, searchable_table

pd.set_option("display.max_columns", 80)
pd.set_option("display.max_colwidth", 120)

# Pull top-level totals and show them as a tidy key/value frame.
stats, _ = api_get("stats")
pd.Series(stats).drop("bills_by_congress").to_frame("value")

# Reference table of the endpoints each chapter relies on.
endpoint_notes = pd.DataFrame(
    [
        {"endpoint": "/stats", "use": "Database totals and bills by congress"},
        {"endpoint": "/congresses?include_stats=true", "use": "Congress timeline, members, committees, bill counts"},
        {"endpoint": "/documents", "use": "Bills with filters for congress, chamber, scope, date, author, search"},
        {"endpoint": "/search/documents", "use": "Topic search across bill text and author names"},
        {"endpoint": "/people", "use": "Legislator roster by chamber and congress"},
        {"endpoint": "/people/{id}/documents", "use": "Bills authored by a legislator"},
        {"endpoint": "/people/{id}?include_congresses=true", "use": "Membership history for each legislator"},
    ]
)
endpoint_notes

# --- COLLAPSIBLE VOLUME BLOCK ---
# bills_by_congress gives full, accurate totals (no sampling needed here).
# We derive the Senate share so we can see exactly when Senate coverage begins.
volume = pd.DataFrame(stats["bills_by_congress"]).sort_values("congress")
volume["senate_share"] = np.where(
    volume["total"] > 0,
    volume["senate_bills"] / volume["total"],
    0.0,
)
volume.rename(
    columns={
        "congress": "Congress",
        "total": "Total bills",
        "house_bills": "House bills",
        "senate_bills": "Senate bills",
    }
)[["Congress", "Total bills", "House bills", "Senate bills", "senate_share"]].style.format(
    {"senate_share": "{:.0%}"}
)

# --- COLLAPSIBLE COMPLETENESS SAMPLING BLOCK ---
# For each congress we pull a small sample (one page = up to 100 bills) and measure
# the completion rate of key fields. Sampling keeps this responsive; the point is
# the relative pattern across congresses, not an exact per-field census.
#
# List-valued fields (authors, subjects, download_url_sources) are "present" only
# when the list is non-empty; scalar fields (date_filed) are "present" when non-null.
LIST_FIELDS = {"authors", "subjects", "download_url_sources"}


def completion_rate(frame: pd.DataFrame, field: str) -> float:
    if field not in frame.columns or len(frame) == 0:
        return 0.0
    if field in LIST_FIELDS:
        present = frame[field].map(lambda v: bool(v) if isinstance(v, list) else False)
    else:
        present = frame[field].notna()
    return float(present.mean())


fields = {
    "Authors linked": "authors",
    "Date filed": "date_filed",
    "Subjects/tags": "subjects",
    "Download URL": "download_url_sources",
}

rows = []
for congress in congress_range():
    # One page per congress keeps the whole audit to ~13 quick calls.
    sample = pd.DataFrame(fetch_all("documents", congress=congress, page_size=100, max_pages=1))
    row = {"Congress": congress, "Sampled bills": len(sample)}
    for label, field in fields.items():
        row[label] = completion_rate(sample, field)
    rows.append(row)

completeness = pd.DataFrame(rows).sort_values("Congress")
completeness.style.format({label: "{:.0%}" for label in fields}).background_gradient(
    cmap="RdYlGn", subset=list(fields), vmin=0, vmax=1
)

# --- COLLAPSIBLE HEATMAP BLOCK ---
# Reshape the completeness frame to long form and render it as a heatmap so the
# gaps are obvious at a glance (dark red = missing, green = complete).
heat = completeness.melt(
    id_vars=["Congress", "Sampled bills"],
    value_vars=list(fields),
    var_name="Field",
    value_name="Completion",
)

fig = px.density_heatmap(
    heat,
    x="Congress",
    y="Field",
    z="Completion",
    histfunc="avg",
    color_continuous_scale="RdYlGn",
    range_color=[0, 1],
    title="Field Completeness by Congress (sampled)",
    labels={"Completion": "Completion rate"},
)
fig.update_xaxes(dtick=1)
fig.update_coloraxes(colorbar_tickformat=".0%")
fig

# Render the completeness table as a client-side searchable/sortable table.
display_completeness = completeness.copy()
for label in fields:
    display_completeness[label] = (display_completeness[label] * 100).round(0).astype(int).astype(str) + "%"
searchable_table(display_completeness, caption="Field completeness by congress (searchable)")