178 lines
6.9 KiB
Python
178 lines
6.9 KiB
Python
# streamlit run analysis/viz/comment_streamlit2.py
|
|
from datetime import datetime as dt
|
|
from pathlib import Path
|
|
import pandas as pd
|
|
import plotly.graph_objects as go
|
|
import plotly.express as px
|
|
import plotly.subplots as ps
|
|
import streamlit as st
|
|
|
|
workdir = Path("analysis/jobs/f452-1")
|
|
df = pd.read_csv(workdir/"review.csv")
|
|
df['date_dt'] = pd.to_datetime(df.date)
|
|
df["date_day"] = df["date_dt"].dt.date
|
|
forum = pd.read_json(workdir/"forum.jsonl", lines=True).iloc[0].to_dict()
|
|
prompt = (workdir/"prompt.txt").read_text(encoding="utf-8")
|
|
|
|
stance_colors = {'oppose':'#ffa15a', 'neutral':'#e377c2','support':'#19d3f3','unknown':'#000000'}
|
|
#stance_colors = {'oppose':'orange', 'neutral':'green','support':'blue','unknown':'gray','mixed':'violet'}
|
|
stance_order = ["oppose", "neutral", "unknown", "support"]
|
|
|
|
st.set_page_config(layout="wide")
|
|
st.title("Virginia Townhall Explorer")
|
|
st.divider()
|
|
st.subheader(forum.get('reg_title'))
|
|
st.text(forum.get('reg_desc'))
|
|
st.caption(f"Link: https://www.townhall.virginia.gov/L/Comments.cfm?GDocForumID={forum.get('forum_id')}")
|
|
|
|
st.write(f'Comments posted from {dt.strftime(min(df.date_dt),"%D")}—{dt.strftime(max(df.date_dt),"%D")}')
|
|
st.write('Data collected on _')
|
|
|
|
st.subheader("Comment Summary")
|
|
# summary
|
|
summary_left, summary_right = st.columns([1,2])
|
|
with summary_left:
|
|
# summary table
|
|
#summary_stats = df.groupby("stance").size().reindex(stance_order,fill_value=0).reset_index(name="count")
|
|
summary_stats = (
|
|
df.groupby("stance").size()
|
|
.reindex(stance_order, fill_value=0)
|
|
.reset_index(name="count")
|
|
.assign(percent=lambda d: (d["count"] / d["count"].sum()).map("{:.1%}".format))
|
|
)
|
|
|
|
st.dataframe(summary_stats, hide_index=True, width="stretch")
|
|
with summary_right:
|
|
# stance div-h
|
|
counts = df["stance"].value_counts()
|
|
stance_divh = go.Figure()
|
|
stance_divh.add_bar(y=["stance"], x=[-counts.get("oppose",0)], name="oppose", orientation="h", marker_color=stance_colors.get('oppose'), text=[counts.get("oppose",0)], textposition="inside")
|
|
stance_divh.add_bar(y=["stance"], x=[counts.get("neutral",0)], name="neutral", orientation="h", marker_color=stance_colors.get('neutral'), text=[counts.get("neutral",0)], textposition="inside")
|
|
stance_divh.add_bar(y=["stance"], x=[counts.get("unknown",0)], name="unknown", orientation="h", marker_color=stance_colors.get('unknown'), text=[counts.get("unknown",0)], textposition="inside")
|
|
stance_divh.add_bar(y=["stance"], x=[counts.get("support",0)], name="support", orientation="h", marker_color=stance_colors.get('support'), text=[counts.get("support",0)], textposition="inside")
|
|
stance_divh.update_yaxes(title_text="",showticklabels=False)
|
|
stance_divh.update_layout(barmode="relative", title="", height=180, margin=dict(l=0,r=0,t=0,b=0),xaxis_title="", yaxis_title="",legend=dict(orientation="v",y=0.12))
|
|
#legend_orientation="v")
|
|
st.plotly_chart(stance_divh,width='stretch')
|
|
|
|
# stance_time
|
|
#stance_order = ["oppose", "neutral","unknown","support"]
|
|
#daily = df.groupby(["date_day", "stance"]).size().reset_index(name="count")
|
|
#stance_time = px.bar(daily, x="date_day", y="count", color="stance", category_orders={"stance": stance_order},color_discrete_map=stance_colors,title="")
|
|
#st.plotly_chart(stance_time, width='stretch')
|
|
|
|
# Daily Comments Breakdown, 3 Tabs
|
|
daily_wide = (
|
|
df.groupby(["date_day", "stance"])
|
|
.size()
|
|
.unstack(fill_value=0)
|
|
.reindex(columns=stance_order, fill_value=0)
|
|
.sort_index()
|
|
)
|
|
|
|
daily_long = (
|
|
daily_wide.reset_index()
|
|
.melt(id_vars="date_day", var_name="stance", value_name="count")
|
|
)
|
|
|
|
cum_wide = daily_wide.cumsum()
|
|
|
|
cum_long = (
|
|
cum_wide.reset_index()
|
|
.melt(id_vars="date_day", var_name="stance", value_name="cumulative_count")
|
|
)
|
|
|
|
cum_total = cum_wide.sum(axis=1)
|
|
cum_share = cum_wide.div(cum_total.where(cum_total > 0), axis=0)
|
|
|
|
cum_share_long = (
|
|
cum_share.reset_index()
|
|
.melt(id_vars="date_day", var_name="stance", value_name="cumulative_share")
|
|
)
|
|
|
|
tab_daily, tab_area, tab_share = st.tabs([
|
|
"Daily",
|
|
"Cumulative",
|
|
"Cumulative Share",
|
|
])
|
|
|
|
with tab_daily:
|
|
fig = px.bar(
|
|
daily_long,
|
|
x="date_day",
|
|
y="count",
|
|
color="stance",
|
|
category_orders={"stance": stance_order},
|
|
color_discrete_map=stance_colors,
|
|
)
|
|
fig.update_layout(barmode="stack", height=420, legend_orientation="v")
|
|
st.plotly_chart(fig, width="stretch")
|
|
|
|
with tab_area:
|
|
fig = px.area(
|
|
cum_long,
|
|
x="date_day",
|
|
y="cumulative_count",
|
|
color="stance",
|
|
category_orders={"stance": stance_order},
|
|
color_discrete_map=stance_colors,
|
|
)
|
|
fig.update_layout(height=420, legend_orientation="v")
|
|
st.plotly_chart(fig, width="stretch")
|
|
|
|
with tab_share:
|
|
fig = px.line(
|
|
cum_share_long,
|
|
x="date_day",
|
|
y="cumulative_share",
|
|
color="stance",
|
|
category_orders={"stance": stance_order},
|
|
color_discrete_map=stance_colors,
|
|
)
|
|
fig.update_yaxes(tickformat=".0%", range=[0, 1])
|
|
fig.update_layout(height=420, legend_orientation="v")
|
|
st.plotly_chart(fig, width="stretch")
|
|
|
|
st.subheader("Comment Explorer")
|
|
|
|
# stance/tone heatmap
|
|
# TODO add raw values
|
|
# TODO OPT add button to swap between pct/tone <> pct/stance
|
|
x_order = ["unknown","oppose","mixed","neutral","support"] # includes mixed even if absent; harmless zero column
|
|
y_order = ["positive","neutral","mixed","negative","unclear"]
|
|
tab = pd.crosstab(df["tone"], df["stance"]).reindex(index=y_order, columns=x_order, fill_value=0)
|
|
pct = tab.div(tab.sum(axis=1).replace(0, pd.NA), axis=0).fillna(0)
|
|
fig = px.imshow(
|
|
pct,
|
|
x=x_order, y=y_order,
|
|
text_auto=".0%",
|
|
aspect="auto",
|
|
color_continuous_scale="Greens",
|
|
title="tone by stance, percent within tone",
|
|
)
|
|
fig.update_traces(text=tab.astype(str) + " / " + (pct*100).round(0).astype(int).astype(str) + "%")
|
|
fig.update_layout(height=420, xaxis_title="stance", yaxis_title="tone")
|
|
st.plotly_chart(fig, width='stretch')
|
|
|
|
# comment explorer
|
|
stance = st.multiselect("Filter stance", sorted(df["stance"].dropna().unique()), default=sorted(df["stance"].dropna().unique()))
|
|
q = st.text_input("Search comment text")
|
|
dff = df[df["stance"].isin(stance)]
|
|
if q:
|
|
dff = dff[dff["text"].fillna("").str.contains(q, case=False, regex=False)]
|
|
|
|
st.dataframe(dff[["comment_id", "title", "text", "stance", "stance_confidence", "tone"]], width="stretch")
|
|
st.write("Showing " + str(len(dff))+ " comments")
|
|
|
|
cid = st.selectbox("comment", dff["comment_id"].astype(str))
|
|
row = dff[dff["comment_id"].astype(str) == cid].iloc[0]
|
|
|
|
st.subheader(row["title"])
|
|
st.write(row["text"])
|
|
st.write(row["author"] + ", " + row["date"][:10])
|
|
st.markdown(f"**stance:** {row['stance']} \t|\t **confidence:** {row['stance_confidence']:.2f} \t|\t **tone:** {row['tone']}")
|
|
st.write("**analysis:** "+ row["stance_rationale"])
|
|
st.write("**model:** " + str(row["model"]))
|
|
with st.expander("Prompt", expanded=False):
|
|
st.code(prompt, language="text")
|