Metadata and Text Quality Audit

# --- COLLAPSIBLE DATA PREPARATION BLOCK ---
# We load the necessary packages, establish working directories, and import
# database helpers for retrieving document metadata.
import pandas as pd
import numpy as np
import plotly.express as px
import sys
from pathlib import Path

cwd = Path.cwd().resolve()
sys.path[:0] = [str(cwd), str(cwd.parent)]

from book_utils import api_get, fetch_all

# --- COLLAPSIBLE DATA INGESTION BLOCK ---
# We retrieve the latest congress number from the database summary stats
# and fetch a representative sample of bill documents for detailed analysis.
stats, _ = api_get("stats")
latest_congress = max(row["congress"] for row in stats["bills_by_congress"])

# Fetch 5 pages (500 records) to ensure highly responsive execution while maintaining
# statistical representation of the legislative session.
documents = pd.DataFrame(fetch_all("documents", congress=latest_congress, max_pages=5))
documents.shape

# --- COLLAPSIBLE AUDIT LOGIC BLOCK ---
# Define fields to audit and calculate missing vs completed records.
# We handle list-based fields (subjects, authors, downloads) by checking
# if the arrays are empty.
fields_to_audit = {
    "Title": "title",
    "Long Title": "long_title",
    "Date Filed": "date_filed",
    "Scope": "scope",
    "Subjects/Tags": "subjects",
    "Authors": "authors",
    "Download URL": "download_url_sources"
}

audit_rows = []
total_docs = len(documents)

for field_label, field_name in fields_to_audit.items():
    # Detect missing values depending on data type (lists vs scalar nulls)
    if field_name in ["subjects", "authors", "download_url_sources"]:
        missing_count = documents[field_name].map(lambda x: len(x) == 0 if isinstance(x, list) else True).sum()
    else:
        missing_count = documents[field_name].isna().sum()
        
    completion_rate = (total_docs - missing_count) / total_docs
    audit_rows.append({
        "Field": field_label,
        "Missing Count": int(missing_count),
        "Completion Rate": completion_rate
    })

audit_df = pd.DataFrame(audit_rows)
audit_df.style.format({"Completion Rate": "{:.1%}"})

# --- COLLAPSIBLE CHART BUILDING BLOCK ---
# We build a horizontal bar chart of completeness and add client-side
# filtering buttons via Plotly's updatemenus.
fig = px.bar(
    audit_df,
    x="Completion Rate",
    y="Field",
    orientation="h",
    range_x=[0, 1.05],
    title=f"Metadata Completion Rates in the {latest_congress}th Congress",
    text=audit_df["Completion Rate"].map(lambda x: f"{x:.1%}"),
    labels={"Completion Rate": "Completion rate", "Field": "Metadata field"}
)

# High and Low completeness masks for interactive client-side switching
high_mask = audit_df["Completion Rate"] >= 0.80
low_mask = audit_df["Completion Rate"] < 0.80

args_all = [{"x": [audit_df["Completion Rate"].tolist()], "y": [audit_df["Field"].tolist()], "text": [audit_df["Completion Rate"].map(lambda x: f"{x:.1%}").tolist()]}]
args_high = [{"x": [audit_df[high_mask]["Completion Rate"].tolist()], "y": [audit_df[high_mask]["Field"].tolist()], "text": [audit_df[high_mask]["Completion Rate"].map(lambda x: f"{x:.1%}").tolist()]}]
args_low = [{"x": [audit_df[low_mask]["Completion Rate"].tolist()], "y": [audit_df[low_mask]["Field"].tolist()], "text": [audit_df[low_mask]["Completion Rate"].map(lambda x: f"{x:.1%}").tolist()]}]

fig.update_layout(
    updatemenus=[
        dict(
            active=0,
            type="buttons",
            direction="right",
            buttons=[
                dict(label="Show All", method="update", args=args_all),
                dict(label="High Completeness (>=80%)", method="update", args=args_high),
                dict(label="Low Completeness (<80%)", method="update", args=args_low),
            ],
            x=0.5,
            xanchor="center",
            y=1.2,
            yanchor="top"
        )
    ]
)
fig

# --- COLLAPSIBLE VERB EXTRACTION BLOCK ---
# Extract first words from titles, clean punctuation, and count them
# separately by chamber (subtype == 'HB' vs 'SB').
def get_action_verb(title):
    if not isinstance(title, str) or not title.strip():
        return "unknown"
    words = title.strip().split()
    if not words:
        return "unknown"
    return "".join(c for c in words[0] if c.isalnum()).lower()

# Process titles and group by chamber
df_verbs = documents[["subtype", "title"]].copy()
df_verbs["action_verb"] = df_verbs["title"].fillna("").map(get_action_verb)
df_verbs = df_verbs[df_verbs["action_verb"] != "unknown"]

# Compute global counts
global_counts = df_verbs["action_verb"].value_counts().head(10)

# Compute chamber counts
house_counts = df_verbs[df_verbs["subtype"] == "HB"]["action_verb"].value_counts().reindex(global_counts.index, fill_value=0)
senate_counts = df_verbs[df_verbs["subtype"] == "SB"]["action_verb"].value_counts().reindex(global_counts.index, fill_value=0)

verbs_summary = pd.DataFrame({
    "All": global_counts,
    "House": house_counts,
    "Senate": senate_counts
}).reset_index()
verbs_summary.columns = ["Action Verb", "All", "House Bills", "Senate Bills"]

# --- COLLAPSIBLE CHART BUILDING BLOCK ---
# Render verb counts with interactive buttons to toggle between data series
fig = px.bar(
    verbs_summary,
    x="All",
    y="Action Verb",
    orientation="h",
    title=f"Top Action Verbs in Bill Titles ({latest_congress}th Congress)",
    labels={"All": "Number of bills", "Action Verb": "First word in title"}
)

fig.update_layout(
    updatemenus=[
        dict(
            type="dropdown",
            direction="down",
            showactive=True,
            buttons=[
                dict(
                    label="All Bills",
                    method="update",
                    args=[{"x": [verbs_summary["All"].tolist()], "y": [verbs_summary["Action Verb"].tolist()]},
                          {"title": f"Top Action Verbs - All Bills ({latest_congress}th Congress)"}]
                ),
                dict(
                    label="House Bills (HB) Only",
                    method="update",
                    args=[{"x": [verbs_summary["House Bills"].tolist()], "y": [verbs_summary["Action Verb"].tolist()]},
                          {"title": f"Top Action Verbs - House Bills ({latest_congress}th Congress)"}]
                ),
                dict(
                    label="Senate Bills (SB) Only",
                    method="update",
                    args=[{"x": [verbs_summary["Senate Bills"].tolist()], "y": [verbs_summary["Action Verb"].tolist()]},
                          {"title": f"Top Action Verbs - Senate Bills ({latest_congress}th Congress)"}]
                )
            ],
            x=1.02,
            y=1.15
        )
    ]
)
fig

# --- COLLAPSIBLE CLASSIFICATION BLOCK ---
# Map bills to intents based on keyword heuristics and group by chamber.
def classify_bill_intent(text):
    text = str(text).lower()
    if any(w in text for w in ["amending", "repeal", "amendment", "republic act", "ra no"]):
        return "Amendment / Repeal"
    elif any(w in text for w in ["establishing", "creating", "declaring", "converting", "granting", "charter"]):
        return "New Statute / Creation"
    else:
        return "Other / Administrative"

documents["legislative_intent"] = documents["title"].fillna("").map(classify_bill_intent)
intent_counts = documents.groupby(["subtype", "legislative_intent"]).size().reset_index(name="count")

# --- COLLAPSIBLE CHART BUILDING BLOCK ---
# Double-trace bar chart with interactive visibility toggles.
fig = px.bar(
    intent_counts,
    x="legislative_intent",
    y="count",
    color="subtype",
    barmode="group",
    title=f"Legislative Intent Mix: New Statutes vs. Amendments ({latest_congress}th Congress)",
    labels={"legislative_intent": "Legislative Intent", "count": "Count", "subtype": "Chamber"}
)

fig.update_layout(
    updatemenus=[
        dict(
            type="buttons",
            direction="right",
            buttons=[
                dict(label="Show Both Chambers", method="update", args=[{"visible": [True, True]}]),
                dict(label="House (HB) Only", method="update", args=[{"visible": [True, False]}]),
                dict(label="Senate (SB) Only", method="update", args=[{"visible": [False, True]}]),
            ],
            x=0.5,
            xanchor="center",
            y=1.15
        )
    ]
)
fig

# --- COLLAPSIBLE METRICS COMPUTATION BLOCK ---
# Calculate word length, character length, and vocabulary richness per chamber
documents["word_count"] = documents["title"].fillna("").map(lambda x: len(str(x).split()))
documents["char_count"] = documents["title"].fillna("").map(lambda x: len(str(x)))

all_words_by_chamber = {}
for chamber in ["HB", "SB"]:
    chamber_docs = documents[documents["subtype"] == chamber]
    words = []
    for title in chamber_docs["title"].dropna():
        words.extend([w.strip().lower() for w in str(title).split() if w.strip()])
    
    total_tokens = len(words)
    unique_types = len(set(words))
    ttr = unique_types / total_tokens if total_tokens > 0 else 0
    all_words_by_chamber[chamber] = {
        "Total Words (Tokens)": total_tokens,
        "Unique Words (Types)": unique_types,
        "Type-Token Ratio (TTR)": ttr
    }

pd.DataFrame(all_words_by_chamber).T.style.format({"Type-Token Ratio (TTR)": "{:.3f}"})

# --- COLLAPSIBLE CHART BUILDING BLOCK ---
# Render box plots comparing word counts across chambers
px.box(
    documents,
    x="subtype",
    y="word_count",
    color="subtype",
    title="Distribution of Bill Title Word Counts by Chamber",
    labels={"subtype": "Chamber", "word_count": "Word count per title"}
)