Legislator Networks, Dynasties, and Demographics

# --- COLLAPSIBLE DATA PREPARATION BLOCK ---
# Load data manipulation libraries, stats, and custom congress API utilities.
import pandas as pd
import numpy as np
import plotly.express as px
from collections import Counter
import sys
from pathlib import Path

cwd = Path.cwd().resolve()
sys.path[:0] = [str(cwd), str(cwd.parent)]

from book_utils import api_get, fetch_all, full_name, searchable_table

# --- COLLAPSIBLE DATA INGESTION BLOCK ---
# Retrieve data for the latest congress. We pull 5 pages (500 records) to keep local builds
# fast and responsive while securing enough records for demographic and social analysis.
stats, _ = api_get("stats")
latest_congress = max(row["congress"] for row in stats["bills_by_congress"])
documents = pd.DataFrame(fetch_all("documents", congress=latest_congress, max_pages=5))

# --- COLLAPSIBLE CO-AUTHOR PAIR CALCULATIONS ---
# Iterate through all documents, retrieve author arrays, and count individual
# filings and pairing counts.
coauthor_pairs = Counter()
author_bill_counts = Counter()

for _, row in documents.iterrows():
    authors = row.get("authors") or []
    author_names = [full_name(a) for a in authors if full_name(a)]
    
    # Track individual counts
    for name in author_names:
        author_bill_counts[name] += 1
        
    # Track pairs
    for i in range(len(author_names)):
        for j in range(i + 1, len(author_names)):
            pair = tuple(sorted([author_names[i], author_names[j]]))
            coauthor_pairs[pair] += 1

pairs_df = pd.DataFrame(
    [{"Author A": p[0], "Author B": p[1], "Shared Bills": count} for p, count in coauthor_pairs.items()]
).sort_values("Shared Bills", ascending=False)

pairs_df.head(10)

# --- COLLAPSIBLE CHART BUILDING BLOCK ---
# Build a horizontal bar chart of collaborating duos with client-side filters.
top_pairs = pairs_df.head(15).copy()
top_pairs["Duo"] = top_pairs["Author A"] + " & " + top_pairs["Author B"]

# Split data by intensity threshold
high_collabs = top_pairs[top_pairs["Shared Bills"] >= 3]

args_all = [{"x": [top_pairs["Shared Bills"].tolist()], "y": [top_pairs["Duo"].tolist()]},
            {"title": f"Top Collaborating Legislator Duos ({latest_congress}th Congress)"}]
args_high = [{"x": [high_collabs["Shared Bills"].tolist()], "y": [high_collabs["Duo"].tolist()]},
             {"title": f"High-Strength Collaborators (>= 3 Shared Bills)"}]

fig = px.bar(
    top_pairs,
    x="Shared Bills",
    y="Duo",
    orientation="h",
    title=f"Top Collaborating Legislator Duos ({latest_congress}th Congress)",
    labels={"Shared Bills": "Co-authored bills", "Duo": "Collaborators"}
)

fig.update_layout(
    updatemenus=[
        dict(
            active=0,
            type="buttons",
            direction="right",
            buttons=[
                dict(label="Show All Top Duos", method="update", args=args_all),
                dict(label="High-Strength Duos (>=3 bills)", method="update", args=args_high),
            ],
            x=0.5,
            xanchor="center",
            y=1.15
        )
    ]
)
fig

# --- COLLAPSIBLE SURNAME ANALYSIS BLOCK ---
# Extract unique authors, count surname occurrences, and categorize.
all_authors_meta = []
seen_author_ids = set()

for _, row in documents.iterrows():
    for author in row.get("authors") or []:
        author_id = author.get("id")
        if author_id and author_id not in seen_author_ids:
            seen_author_ids.add(author_id)
            all_authors_meta.append({
                "id": author_id,
                "first_name": author.get("first_name", ""),
                "last_name": author.get("last_name", ""),
                "full_name": full_name(author)
            })

authors_df = pd.DataFrame(all_authors_meta)

# Group by last name and count occurrences
surname_counts = authors_df["last_name"].value_counts().reset_index()
surname_counts.columns = ["last_name", "surname_frequency"]

authors_df = authors_df.merge(surname_counts, on="last_name")

# Label dynasty proxy: shared surname by 2 or more active authors in this dataset
authors_df["dynasty_proxy"] = np.where(authors_df["surname_frequency"] >= 2, "Dynastic Surname", "Single Surname")

# Merge with output numbers
author_bills_lookup = pd.DataFrame([{"full_name": name, "bills_filed": count} for name, count in author_bill_counts.items()])
authors_analysis = authors_df.merge(author_bills_lookup, left_on="full_name", right_on="full_name", how="left").fillna(0)

# Calculate sum and average outputs
dynasty_summary = authors_analysis.groupby("dynasty_proxy").agg(
    avg_bills=("bills_filed", "mean"),
    total_bills=("bills_filed", "sum")
).reset_index()
dynasty_summary

# --- COLLAPSIBLE CHART BUILDING BLOCK ---
# Single-bar comparative chart with interactive y-axis mapping toggles.
fig = px.bar(
    dynasty_summary,
    x="dynasty_proxy",
    y="avg_bills",
    color="dynasty_proxy",
    title=f"Legislative Volume comparison by Dynasty Proxy ({latest_congress}th Congress)",
    labels={"avg_bills": "Average bills filed", "dynasty_proxy": "Category"}
)

fig.update_layout(
    updatemenus=[
        dict(
            type="dropdown",
            direction="down",
            showactive=True,
            buttons=[
                dict(
                    label="Show Average Output",
                    method="update",
                    args=[{"y": [dynasty_summary["avg_bills"].tolist()]},
                          {"title": "Average Bills Filed: Dynastic vs. Single Surnames",
                           "yaxis": {"title": "Average bills filed per member"}}]
                ),
                dict(
                    label="Show Cumulative Total",
                    method="update",
                    args=[{"y": [dynasty_summary["total_bills"].tolist()]},
                          {"title": "Total Cumulative Bills Filed by Surname Category",
                           "yaxis": {"title": "Total bills filed"}}]
                )
            ],
            x=1.02,
            y=1.15
        )
    ]
)
fig

# --- COLLAPSIBLE SEARCHABLE ROSTER BLOCK ---
# Present the per-author surname/dynasty/output table as a client-side searchable
# table so readers can look up specific legislators or families by name.
roster = authors_analysis[
    ["full_name", "last_name", "surname_frequency", "dynasty_proxy", "bills_filed"]
].copy()
roster["bills_filed"] = roster["bills_filed"].astype(int)
roster = roster.sort_values(["surname_frequency", "bills_filed"], ascending=False).rename(
    columns={
        "full_name": "Legislator",
        "last_name": "Surname",
        "surname_frequency": "Shared-surname count",
        "dynasty_proxy": "Category",
        "bills_filed": "Bills filed",
    }
)
searchable_table(roster, caption="Authors by surname and output (searchable)")

# --- COLLAPSIBLE GENDER CLASSIFIER BLOCK ---
# Map first name indicators to gender and join with bill texts.
female_indicators = ["maria", "mary", "ma.", "ana", "gloria", "corazon", "pia", "loren", "risa", "grace", "imelda", "cynthia", "stella", "glenda", "shirley", "marlyn", "lucy", "rosanna"]
male_indicators = ["juan", "jose", "manuel", "ferdinand", "rodrigo", "alan", "jinggoy", "robinhood", "bong", "win", "mark", "joel", "hezel", "ramon", "vicente", "pantaleon", "arnolfo", "prospero"]

def guess_gender(first_name):
    first_name_lower = str(first_name).lower()
    if any(ind in first_name_lower for ind in female_indicators):
        return "Female"
    if any(ind in first_name_lower for ind in male_indicators):
        return "Male"
    if first_name_lower.endswith("a") or first_name_lower.endswith("ine") or first_name_lower.endswith("ette"):
        return "Female"
    return "Male"

authors_analysis["gender"] = authors_analysis["first_name"].map(guess_gender)

# Connect bills to topics
topics = {
    "Education/Health/Labor": ["school", "education", "teacher", "student", "university", "health", "hospital", "medical", "patient", "labor", "worker", "employment", "wage"],
    "Infrastructure/Economy/Justice": ["road", "bridge", "transport", "infrastructure", "tax", "business", "trade", "investment", "court", "penal", "crime", "justice"]
}

text_cols = ["title", "long_title", "congress_website_title", "congress_website_abstract"]
documents["search_text"] = documents[text_cols].fillna("").agg(" ".join, axis=1).str.lower()

def get_topics(text):
    matched = []
    for topic, keywords in topics.items():
        if any(kw in text for kw in keywords):
            matched.append(topic)
    return matched if matched else ["Other"]

documents["topics"] = documents["search_text"].map(get_topics)

# Explode documents by authors and map gender
exploded_docs = documents.explode("authors")
exploded_docs["author_id"] = exploded_docs["authors"].map(lambda x: x.get("id") if isinstance(x, dict) else None)
exploded_docs = exploded_docs.dropna(subset=["author_id"])

gender_lookup = authors_analysis.set_index("id")["gender"].to_dict()
exploded_docs["gender"] = exploded_docs["author_id"].map(gender_lookup).fillna("Male")

exploded_topics = exploded_docs.explode("topics")
gender_topic_mix = exploded_topics.groupby(["gender", "topics"]).size().reset_index(name="count")

# Convert count to percentages per gender group
gender_totals = gender_topic_mix.groupby("gender")["count"].transform("sum")
gender_topic_mix["percentage"] = gender_topic_mix["count"] / gender_totals
gender_topic_mix

# --- COLLAPSIBLE CHART BUILDING BLOCK ---
# Multi-trace comparative bar chart with interactive visibility triggers.
fig = px.bar(
    gender_topic_mix,
    x="topics",
    y="percentage",
    color="gender",
    barmode="group",
    title="Policy Topic Distribution by Gender (Heuristic Classifications)",
    labels={"percentage": "Share of authored bills", "topics": "Policy Group", "gender": "Gender"}
)
fig.update_layout(yaxis_tickformat=".0%")

fig.update_layout(
    updatemenus=[
        dict(
            type="buttons",
            direction="right",
            buttons=[
                dict(label="Show Both Focuses", method="update", args=[{"visible": [True, True]}]),
                dict(label="Male Focus Only", method="update", args=[{"visible": [False, True]}]),
                dict(label="Female Focus Only", method="update", args=[{"visible": [True, False]}]),
            ],
            x=0.5,
            xanchor="center",
            y=1.15
        )
    ]
)
fig

# --- COLLAPSIBLE SCOPE COMPILATION BLOCK ---
# Calculate the share of local vs national bills per author.
author_scope_counts = Counter()
for _, row in documents.iterrows():
    scope = str(row.get("scope") or "Unknown").title()
    for author in row.get("authors") or []:
        author_id = author.get("id")
        name = full_name(author)
        if author_id and name:
            author_scope_counts[(name, scope)] += 1

scope_rows = []
for (name, scope), count in author_scope_counts.items():
    scope_rows.append({"name": name, "scope": scope, "count": count})

scope_df = pd.DataFrame(scope_rows)
pivot_scope = scope_df.pivot(index="name", columns="scope", values="count").fillna(0)

if "Local" not in pivot_scope.columns:
    pivot_scope["Local"] = 0
if "National" not in pivot_scope.columns:
    pivot_scope["National"] = 0

pivot_scope["total"] = pivot_scope["Local"] + pivot_scope["National"]
pivot_scope["local_share"] = pivot_scope["Local"] / pivot_scope.replace(0, 1)["total"]
pivot_scope = pivot_scope.sort_values("local_share", ascending=False)

pivot_scope["focus"] = np.select(
    [
        pivot_scope["local_share"] >= 0.70,
        pivot_scope["local_share"] <= 0.10,
    ],
    ["Highly Local (District Focus)", "Highly National (Sectoral/Senate Focus)"],
    default="Balanced Focus"
)
pivot_scope.head(10)

# --- COLLAPSIBLE CHART BUILDING BLOCK ---
# Pie chart illustrating the share of local vs national focus groups.
focus_counts = pivot_scope["focus"].value_counts().reset_index()
focus_counts.columns = ["focus", "count"]

fig = px.pie(
    focus_counts,
    names="focus",
    values="count",
    title="Legislator Focus Classification (Based on Bill Scope Mix)"
)
fig