Policy Topic Signals

# --- COLLAPSIBLE SETUP BLOCK ---
from collections import Counter

import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
import sys
from pathlib import Path

cwd = Path.cwd().resolve()
sys.path[:0] = [str(cwd), str(cwd.parent)]

from book_utils import api_get, fetch_all, congress_range, searchable_table

# --- COLLAPSIBLE TOPIC-DEFINITION BLOCK ---
# Transparent keyword groups. Each topic is a list of lowercase substrings; a bill
# matches a topic if any keyword appears anywhere in its combined text fields.
topics = {
    "Education": ["school", "education", "teacher", "student", "university", "learning"],
    "Health": ["health", "hospital", "medical", "patient", "disease", "medicine"],
    "Labor": ["labor", "worker", "employment", "wage", "job", "employee"],
    "Local Government": ["barangay", "municipality", "city", "province", "local government"],
    "Justice": ["court", "penal", "crime", "justice", "law enforcement", "correction"],
    "Economy": ["tax", "business", "trade", "industry", "investment", "enterprise"],
    "Environment": ["environment", "climate", "forest", "water", "waste", "renewable"],
    "Infrastructure": ["road", "bridge", "transport", "airport", "railway", "infrastructure"],
}

TEXT_COLS = ["title", "long_title", "congress_website_title", "congress_website_abstract"]


def build_search_text(frame: pd.DataFrame) -> pd.Series:
    """Concatenate the text fields of each bill into one lowercase search string."""
    present = [c for c in TEXT_COLS if c in frame.columns]
    return frame[present].fillna("").agg(" ".join, axis=1).str.lower()


def topic_rates(frame: pd.DataFrame) -> pd.DataFrame:
    """Return, per topic, the share of sampled bills that mention any keyword."""
    text = build_search_text(frame)
    rows = []
    for topic, keywords in topics.items():
        mask = text.map(lambda t: any(kw in t for kw in keywords))
        rows.append({"topic": topic, "share": float(mask.mean()), "bills": int(mask.sum())})
    return pd.DataFrame(rows).sort_values("share", ascending=False)

# --- COLLAPSIBLE CROSS-CONGRESS SAMPLING BLOCK ---
# For each congress, pull a sample (500 bills) and compute topic shares. We keep
# a consistent topic order (from the latest congress) so the bars line up when the
# dropdown swaps congresses.
SAMPLE_PAGES = 5  # 5 pages x 100 = up to 500 bills per congress

per_congress = {}
for congress in congress_range():
    sample = pd.DataFrame(fetch_all("documents", congress=congress, page_size=100, max_pages=SAMPLE_PAGES))
    per_congress[congress] = topic_rates(sample)

# Fix a stable topic ordering based on the most recent congress.
latest_congress = congress_range()[-1]
topic_order = per_congress[latest_congress]["topic"].tolist()


def ordered(frame: pd.DataFrame) -> pd.DataFrame:
    return frame.set_index("topic").reindex(topic_order).reset_index()


# Default trace: latest congress.
default = ordered(per_congress[latest_congress])
fig = go.Figure(
    go.Bar(
        x=default["share"],
        y=default["topic"],
        orientation="h",
        marker_color="#2b6cb0",
        text=[f"{s:.0%}" for s in default["share"]],
        textposition="outside",
    )
)

# One dropdown button per congress.
buttons = []
for congress in congress_range():
    frame = ordered(per_congress[congress])
    buttons.append(
        dict(
            label=f"{congress}th Congress",
            method="update",
            args=[
                {"x": [frame["share"].tolist()],
                 "text": [[f"{s:.0%}" for s in frame["share"]]]},
                {"title": f"Policy Topic Signals - {congress}th Congress (sampled)"},
            ],
        )
    )

fig.update_layout(
    title=f"Policy Topic Signals - {latest_congress}th Congress (sampled)",
    xaxis_title="Share of sampled bills mentioning the topic",
    xaxis_tickformat=".0%",
    yaxis={"categoryorder": "array", "categoryarray": topic_order[::-1]},
    updatemenus=[
        dict(
            buttons=buttons,
            direction="down",
            showactive=True,
            x=1.02,
            xanchor="left",
            y=1.0,
            yanchor="top",
            active=len(buttons) - 1,
        )
    ],
)
fig

# --- COLLAPSIBLE SUBJECTS BLOCK ---
# Tally the most common structured subject tags in the latest congress. Many bills
# carry no subjects at all (see the completeness chapter), so this counts only the
# tagged minority.
latest_docs = pd.DataFrame(fetch_all("documents", congress=latest_congress, page_size=100, max_pages=SAMPLE_PAGES))

subjects = Counter()
for value in latest_docs.get("subjects", pd.Series(dtype=object)).dropna():
    if isinstance(value, list):
        subjects.update(str(item).strip().title() for item in value if str(item).strip())

subjects_df = pd.DataFrame(subjects.most_common(30), columns=["subject", "bills"])
tagged = latest_docs["subjects"].map(lambda v: bool(v) if isinstance(v, list) else False).mean()
print(f"Share of sampled bills carrying at least one subject tag: {tagged:.0%}")

# Searchable/sortable subject-tag table.
if subjects_df.empty:
    print("No structured subject tags were present in this sample.")
else:
    searchable_table(subjects_df, caption="Structured subject tags (searchable)", page_length=10)