import html
import json
import os
from typing import List
import gradio as gr
import pandas as pd
from topic_pipeline import (
OUTPUT_DIR,
parse_notebooklm_tccm_text,
run_complete_pipeline,
write_tccm_dual_validation,
)
os.makedirs(OUTPUT_DIR, exist_ok=True)
def _exists(name: str) -> bool:
return os.path.exists(os.path.join(OUTPUT_DIR, name))
def _load_json(name: str):
with open(os.path.join(OUTPUT_DIR, name), "r", encoding="utf-8") as f:
return json.load(f)
def _download_files() -> List[str]:
names = [
"comparison.csv",
"taxonomy_map.json",
"topic_model_report.md",
"narrative.txt",
"cluster_optimization_log.csv",
"llm_council_validation.csv",
"tccm_validation.csv",
"tccm_dual_validation.csv",
"notebooklm_extraction.csv",
"compliance_checklist.csv",
"compliance_checklist.json",
"run_metadata.json",
"combined_labels.json",
]
return [os.path.join(OUTPUT_DIR, name) for name in names if _exists(name)]
def _phase_html() -> str:
phases = [
("Corpus", _exists("corpus_config.json")),
("Embeddings", _exists("combined_emb.npy")),
("Optimization", _exists("cluster_optimization_log.csv")),
("Clusters", _exists("combined_labels.json")),
("Council", _exists("llm_council_validation.csv")),
("TCCM", _exists("tccm_validation.csv")),
("Compliance", _exists("compliance_checklist.csv")),
("Report", _exists("topic_model_report.md")),
]
chips = []
for name, done in phases:
bg = "#0f766e" if done else "#334155"
mark = "OK" if done else "--"
chips.append(
f"{mark} {name}"
)
return "
" + "".join(chips) + "
"
def _cluster_table():
if not _exists("combined_labels.json"):
return []
rows = []
for s in _load_json("combined_labels.json"):
rows.append([
s.get("cluster_id"),
s.get("label"),
s.get("category"),
s.get("paper_count"),
s.get("confidence"),
s.get("agreement_score"),
"; ".join(s.get("keywords", [])[:8]),
" | ".join(s.get("top_titles", [])[:3]),
s.get("reasoning", ""),
])
return rows
def _council_table():
path = os.path.join(OUTPUT_DIR, "llm_council_validation.csv")
if not os.path.exists(path):
return []
return pd.read_csv(path).head(120)
def _council_viz_html() -> str:
path = os.path.join(OUTPUT_DIR, "llm_council_validation.csv")
if not os.path.exists(path):
return (
"Run the pipeline to activate the LLM Council "
"validation board.
"
)
df = pd.read_csv(path)
if df.empty:
return "Council validation file is empty.
"
grouped = list(df.groupby(["cluster_id", "final_label"], sort=False))[:6]
rows = []
avg_agreement = float(df["agreement_score"].mean()) if "agreement_score" in df else 0
avg_confidence = float(df["confidence"].mean()) if "confidence" in df else 0
llm_member_present = df["member"].astype(str).str.contains("LLM|Mistral", case=False, regex=True).any()
llm_status = "Mistral LLM active" if llm_member_present else "Local semantic fallback active"
for (cluster_id, final_label), group in grouped:
votes = []
for _, row in group.iterrows():
member = html.escape(str(row.get("member", "")))
label = html.escape(str(row.get("member_label", "")))
method = html.escape(str(row.get("method", "")))
votes.append(
""
"
"
f"
{member}{label}{method}
"
"
"
)
confidence = int(float(group["confidence"].iloc[0]) * 100)
agreement = int(float(group["agreement_score"].iloc[0]) * 100)
rows.append(
""
"
"
f"Cluster {html.escape(str(cluster_id))}"
f"{html.escape(str(final_label))}"
"
"
"
"
+ "".join(votes) +
"
"
"Accepted label"
f"{html.escape(str(final_label))}"
f"{confidence}% confidence | {agreement}% agreement"
"
"
"
"
"
"
)
return (
""
"
"
"
LLM Council Validation Running In-App
"
"
Three independent validators inspect each cluster label, compare votes, "
"and write the accepted label plus agreement score into the export file.
"
"
"
f"
{len(df['cluster_id'].unique())}clusters checked
"
f"
{int(avg_agreement * 100)}%avg agreement
"
f"
{int(avg_confidence * 100)}%avg confidence
"
f"
{html.escape(llm_status)}council mode
"
"
"
"
"
"
1
Keyword Extractor
"
"
"
"
2
PAJAIS Mapper
"
"
"
"
3
LLM / Semantic Judge
"
"
"
"
OK
Validated Label
"
"
"
+ "".join(rows) +
"
"
)
def _optimizer_table():
path = os.path.join(OUTPUT_DIR, "cluster_optimization_log.csv")
if not os.path.exists(path):
return []
df = pd.read_csv(path)
cols = [
c for c in [
"algorithm",
"umap_n_neighbors",
"umap_n_components",
"hdbscan_min_cluster_size",
"hdbscan_min_samples",
"n_clusters",
"noise_ratio",
"min_size",
"max_size",
"too_small",
"too_large",
"silhouette_cosine",
"score",
"optimizer_recommendation",
] if c in df.columns
]
return df[cols].head(80)
def _tccm_table():
path = os.path.join(OUTPUT_DIR, "tccm_validation.csv")
if not os.path.exists(path):
return []
return pd.read_csv(path).head(100)
def _tccm_dual_table():
path = os.path.join(OUTPUT_DIR, "tccm_dual_validation.csv")
if not os.path.exists(path):
return []
return pd.read_csv(path).head(100)
def _compliance_table():
path = os.path.join(OUTPUT_DIR, "compliance_checklist.csv")
if not os.path.exists(path):
return []
return pd.read_csv(path)
def _compliance_html() -> str:
path = os.path.join(OUTPUT_DIR, "compliance_checklist.csv")
if not os.path.exists(path):
return (
"Run the pipeline to generate the professor-requirement "
"compliance checklist.
"
)
df = pd.read_csv(path)
color_map = {
"PASS": "#0f766e",
"FAIL": "#b91c1c",
"CONFIG_REQUIRED": "#b45309",
"ENV_FALLBACK": "#b45309",
"INPUT_REQUIRED": "#b45309",
"PARTIAL": "#7c3aed",
"MANUAL_REQUIRED": "#475569",
"REVIEW": "#7c3aed",
}
rows = []
for _, row in df.iterrows():
status = str(row.get("Status", "REVIEW"))
color = color_map.get(status, "#475569")
rows.append(
""
f"
{html.escape(status)}"
f"
{html.escape(str(row.get('Requirement', '')))}"
f"
{html.escape(str(row.get('Evidence', '')))}
"
f"
{html.escape(str(row.get('File', '')))}"
"
"
)
return (
""
"
Professor Requirement Compliance Checklist
"
"
This separates completed app evidence from items that still need API secrets, "
"NotebookLM/full-text inputs, or mentor approval.
"
"
" + "".join(rows) + "
"
)
def _tccm_dual_status_html() -> str:
path = os.path.join(OUTPUT_DIR, "tccm_dual_validation.csv")
if not os.path.exists(path):
return (
"Upload NotebookLM and second-LLM extraction CSVs "
"to generate TCCM dual validation.
"
)
df = pd.read_csv(path)
status_col = "Final_TCCM_Compliance_Status"
if status_col not in df.columns:
return "TCCM dual validation is pending source uploads.
"
counts = df[status_col].value_counts().to_dict()
cards = []
for status, count in counts.items():
ok = "COMPLIANT" in str(status)
color = "#0f766e" if ok else "#b45309"
cards.append(
f"{count}"
f"{html.escape(str(status))}
"
)
return (
"TCCM Dual Validation Status
"
"
Required by email: NotebookLM extraction plus another LLM/extraction method. "
"This screen reconciles those files with regex/semantic extraction.
"
"
" + "".join(cards) + "
"
)
def _on_tccm_dual_validate(notebook_file, second_file):
notebook_path = notebook_file if isinstance(notebook_file, str) else getattr(notebook_file, "name", "")
second_path = second_file if isinstance(second_file, str) else getattr(second_file, "name", "")
write_tccm_dual_validation(notebook_path, second_path)
return _tccm_dual_status_html(), _tccm_dual_table(), _download_files()
def _on_notebooklm_paste(notebook_text):
if not str(notebook_text or "").strip():
return (
"Paste the NotebookLM table text first.
",
_tccm_dual_table(),
_download_files(),
)
notebook_path = parse_notebooklm_tccm_text(notebook_text)
write_tccm_dual_validation(notebook_path, "")
count = len(pd.read_csv(notebook_path)) if os.path.exists(notebook_path) else 0
status = (
f"NotebookLM Paste Imported
"
f"
Parsed {count} NotebookLM rows into outputs/notebooklm_extraction.csv. "
"Merged with the independent regex/semantic extractor in "
"outputs/tccm_dual_validation.csv. Upload a second-LLM CSV as well "
"for full NotebookLM + second LLM compliance.
"
+ _tccm_dual_status_html()
)
return status, _tccm_dual_table(), _download_files()
def _chart_iframe(name: str) -> str:
path = os.path.join(OUTPUT_DIR, "combined_charts", name)
if not os.path.exists(path):
return (
""
"Run the pipeline to generate this chart.
"
)
with open(path, "r", encoding="utf-8") as f:
srcdoc = f.read().replace("&", "&").replace('"', """)
return (
f""
)
def _cards_html() -> str:
if not _exists("combined_labels.json"):
return (
""
"Clusters will appear here after a complete run.
"
)
cards = []
for s in _load_json("combined_labels.json"):
evidence = html.escape(" | ".join(s.get("top_titles", [])[:3]))
label = html.escape(s.get("label", "Cluster"))
category = html.escape(s.get("category", "Unmapped"))
keywords = html.escape(", ".join(s.get("keywords", [])[:8]))
conf = int(float(s.get("confidence", 0)) * 100)
cards.append(
""
f"
{label}
"
f"
{category}
"
f"
"
f"{s.get('paper_count', 0)} papers | confidence {conf}% | agreement {s.get('agreement_score', 0)}
"
f"
{keywords}
"
f"
{evidence}
"
"
"
)
return (
"" + "".join(cards) + "
"
)
def _summary_markdown(result=None) -> str:
if result is None and not _exists("run_metadata.json"):
return (
"Upload the Scopus CSV and click **Run Complete Pipeline**. "
"The app will generate paper-level Title+Abstract+DOI embeddings, optimize "
"UMAP/HDBSCAN clustering, label 15-25 clusters through an in-app council, "
"map them to PAJAIS, and export TCCM validation files."
)
meta = result or {}
if not meta:
meta = {
"parameters": _load_json("run_metadata.json").get("selected_parameters", {}),
"embedding": _load_json("run_metadata.json").get("embedding", {}),
"clusters": _load_json("combined_labels.json") if _exists("combined_labels.json") else [],
"taxonomy": _load_json("taxonomy_map.json") if _exists("taxonomy_map.json") else {},
"config": _load_json("corpus_config.json") if _exists("corpus_config.json") else {},
}
params = meta.get("parameters", {})
emb = meta.get("embedding", {})
tax = meta.get("taxonomy", {}).get("coverage_stats", {})
cfg = meta.get("config", {})
return (
f"**Run complete.** Analysed {cfg.get('rows', 'N/A')} papers from "
f"{cfg.get('journal', 'the corpus')} ({cfg.get('year_min')} to {cfg.get('year_max')}).\n\n"
f"Selected clustering: `{params.get('algorithm')}` with "
f"`{params.get('n_clusters')}` clusters, min size `{params.get('min_size')}`, "
f"max size `{params.get('max_size')}`, noise ratio `{params.get('noise_ratio')}`.\n\n"
f"Embedding: `{emb.get('embedding_model')}`. PAJAIS mapped: "
f"`{tax.get('mapped', 0)}`; novel: `{tax.get('novel', 0)}`. "
"Download the optimizer log and council validation for the final submission appendix."
)
def _run(file_obj):
if file_obj is None:
return (
"Upload a CSV first.",
_phase_html(),
_cluster_table(),
_cards_html(),
_optimizer_table(),
_compliance_html(),
_compliance_table(),
_council_viz_html(),
_council_table(),
_tccm_table(),
_tccm_dual_status_html(),
_tccm_dual_table(),
_chart_iframe("intertopic_map.html"),
_chart_iframe("bar_chart.html"),
_chart_iframe("treemap.html"),
_download_files(),
)
filepath = file_obj if isinstance(file_obj, str) else file_obj.name
result = run_complete_pipeline(filepath)
return (
_summary_markdown(result),
_phase_html(),
_cluster_table(),
_cards_html(),
_optimizer_table(),
_compliance_html(),
_compliance_table(),
_council_viz_html(),
_council_table(),
_tccm_table(),
_tccm_dual_status_html(),
_tccm_dual_table(),
_chart_iframe("intertopic_map.html"),
_chart_iframe("bar_chart.html"),
_chart_iframe("treemap.html"),
result["deliverables"],
)
def _refresh():
return (
_summary_markdown(),
_phase_html(),
_cluster_table(),
_cards_html(),
_optimizer_table(),
_compliance_html(),
_compliance_table(),
_council_viz_html(),
_council_table(),
_tccm_table(),
_tccm_dual_status_html(),
_tccm_dual_table(),
_chart_iframe("intertopic_map.html"),
_chart_iframe("bar_chart.html"),
_chart_iframe("treemap.html"),
_download_files(),
)
CSS = """
.gradio-container { max-width: 1360px !important; }
.app-title { padding: 18px 0 8px; }
.app-title h1 { margin: 0; font-size: 30px; color: #0f172a; letter-spacing: 0; }
.app-title p { color: #475569; margin: 6px 0 0; }
.compliance-empty { padding: 24px; border: 1px dashed #94a3b8; border-radius: 8px; background: #f8fafc; color: #475569; }
.compliance-board, .tccm-status { background: #ffffff; border: 1px solid #d8dee9; border-radius: 10px; padding: 16px; }
.compliance-board h3, .tccm-status h3 { margin: 0; color: #0f172a; font-size: 20px; }
.compliance-board p, .tccm-status p { color: #475569; margin: 6px 0 14px; line-height: 1.45; }
.compliance-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 10px; }
.compliance-row { border: 1px solid #e2e8f0; border-radius: 8px; padding: 12px; background: #f8fafc; }
.compliance-row span { display: inline-block; color: white; font-size: 11px; font-weight: 800; padding: 3px 8px; border-radius: 999px; margin-bottom: 8px; }
.compliance-row strong { display: block; color: #0f172a; font-size: 14px; }
.compliance-row p { font-size: 12px; margin: 6px 0; color: #475569; }
.compliance-row small { color: #64748b; font-size: 11px; }
.tccm-status > div { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 10px; }
.tccm-card { border: 1px solid #e2e8f0; border-radius: 8px; padding: 12px; background: #f8fafc; }
.tccm-card strong { display: block; font-size: 24px; }
.tccm-card span { color: #475569; font-size: 12px; font-weight: 700; }
.council-empty { padding: 28px; border: 1px dashed #94a3b8; color: #475569; border-radius: 8px; background: #f8fafc; }
.council-board { background: #08111f; color: #e5edf7; border-radius: 10px; padding: 18px; border: 1px solid #1e3a5f; }
.council-top { display: grid; grid-template-columns: minmax(280px, 1.2fr) minmax(320px, 1fr); gap: 16px; align-items: start; }
.council-top h3 { margin: 0; font-size: 20px; letter-spacing: 0; }
.council-top p { margin: 6px 0 0; color: #9fb3c8; line-height: 1.45; }
.council-metrics { display: grid; grid-template-columns: repeat(2, minmax(140px, 1fr)); gap: 8px; }
.council-metrics div { background: #10243a; border: 1px solid #1f4568; border-radius: 8px; padding: 10px; }
.council-metrics strong { display: block; color: #5eead4; font-size: 18px; }
.council-metrics span { color: #a8bed4; font-size: 12px; }
.council-lane { display: grid; grid-template-columns: 1fr 60px 1fr 60px 1fr 60px 1fr; gap: 8px; align-items: center; margin: 20px 0; }
.pulse-node { min-height: 62px; display: grid; place-items: center; text-align: center; border: 1px solid #2563eb; background: #0f2546; border-radius: 8px; color: #bfdbfe; font-weight: 800; animation: councilGlow 1.8s ease-in-out infinite; }
.pulse-node span { display: block; font-size: 11px; font-weight: 600; color: #dbeafe; margin-top: 3px; }
.pulse-node.final { border-color: #14b8a6; background: #0f3b39; color: #99f6e4; }
.pulse-line { height: 3px; border-radius: 999px; background: linear-gradient(90deg, #2563eb, #14b8a6, #2563eb); background-size: 220% 100%; animation: councilFlow 1.1s linear infinite; }
.council-cluster { margin-top: 10px; padding: 12px; border: 1px solid #1f4568; border-radius: 8px; background: #0c1b2c; }
.cluster-head { display: flex; justify-content: space-between; gap: 12px; color: #cbd5e1; margin-bottom: 10px; }
.cluster-head span { color: #7dd3fc; font-weight: 800; }
.cluster-head strong { color: #f8fafc; }
.council-flow { display: grid; grid-template-columns: repeat(4, minmax(160px, 1fr)); gap: 8px; }
.council-vote, .final-label { border-radius: 8px; padding: 10px; background: #10243a; border: 1px solid #1f4568; min-height: 82px; }
.council-vote { display: flex; gap: 8px; align-items: flex-start; }
.vote-dot { width: 10px; height: 10px; margin-top: 4px; border-radius: 50%; background: #5eead4; box-shadow: 0 0 14px #5eead4; animation: councilBlink 1.2s ease-in-out infinite; flex: 0 0 auto; }
.council-vote strong, .final-label strong { display: block; color: #e2e8f0; font-size: 13px; }
.council-vote span, .final-label span { display: block; color: #5eead4; font-size: 12px; margin-top: 3px; }
.council-vote small, .final-label small { display: block; color: #94a3b8; font-size: 11px; margin-top: 4px; line-height: 1.25; }
.final-label { border-color: #14b8a6; background: #0d302f; }
@keyframes councilFlow { from { background-position: 0% 0; } to { background-position: 220% 0; } }
@keyframes councilGlow { 0%, 100% { box-shadow: 0 0 0 rgba(37,99,235,0.2); } 50% { box-shadow: 0 0 22px rgba(20,184,166,0.45); } }
@keyframes councilBlink { 0%, 100% { opacity: .35; transform: scale(.75); } 50% { opacity: 1; transform: scale(1.15); } }
@media (max-width: 900px) {
.council-top, .council-flow { grid-template-columns: 1fr; }
.council-lane { grid-template-columns: 1fr; }
.pulse-line { height: 18px; width: 3px; justify-self: center; }
}
"""
with gr.Blocks(title="SPJIMR Topic Modelling Submission App", css=CSS, theme=gr.themes.Soft()) as demo:
gr.HTML(
"SPJIMR Topic Modelling Submission App
"
"
Paper-level Title + Abstract + DOI vectors, UMAP/HDBSCAN optimization, "
"LLM council validation, PAJAIS mapping, and TCCM extraction appendix.
"
)
phase = gr.HTML(value=_phase_html())
with gr.Row():
csv_file = gr.File(label="Upload Scopus CSV", file_types=[".csv"], scale=3)
with gr.Column(scale=1):
run_btn = gr.Button("Run Complete Pipeline", variant="primary")
refresh_btn = gr.Button("Refresh Outputs")
summary = gr.Markdown(value=_summary_markdown())
with gr.Tabs():
with gr.Tab("Clusters"):
cluster_table = gr.Dataframe(
headers=[
"Cluster ID", "Label", "PAJAIS Category", "Papers", "Confidence",
"Agreement", "Keywords", "Top 3 Titles", "Reasoning",
],
value=_cluster_table(),
wrap=True,
interactive=False,
)
cluster_cards = gr.HTML(value=_cards_html())
with gr.Tab("Optimization"):
optimizer_table = gr.Dataframe(value=_optimizer_table(), wrap=True, interactive=False)
with gr.Tab("Compliance"):
compliance_panel = gr.HTML(value=_compliance_html())
compliance_table = gr.Dataframe(value=_compliance_table(), wrap=True, interactive=False)
with gr.Tab("Council Validation"):
council_viz = gr.HTML(value=_council_viz_html())
council_table = gr.Dataframe(value=_council_table(), wrap=True, interactive=False)
with gr.Tab("TCCM Validation"):
tccm_table = gr.Dataframe(value=_tccm_table(), wrap=True, interactive=False)
with gr.Tab("TCCM Dual Validation"):
gr.Markdown(
"Upload the **NotebookLM extraction CSV** and the **second LLM extraction CSV** "
"from full-text PDFs. Expected columns can include Title, DOI, Theory, Context, "
"Variables/Constructs, Method, and Computational Techniques."
)
notebook_paste = gr.Textbox(
label="Paste NotebookLM table output",
lines=10,
placeholder="Paste the copied NotebookLM table here, including Paper ID / Paper Citation / Study Type / DV / IV / Evidence lines...",
)
notebook_paste_btn = gr.Button("Import NotebookLM Paste", variant="secondary")
with gr.Row():
notebook_file = gr.File(label="NotebookLM extraction CSV", file_types=[".csv"])
second_llm_file = gr.File(label="Second LLM extraction CSV", file_types=[".csv"])
tccm_dual_btn = gr.Button("Validate NotebookLM + Second LLM", variant="primary")
tccm_dual_status = gr.HTML(value=_tccm_dual_status_html())
tccm_dual_table = gr.Dataframe(value=_tccm_dual_table(), wrap=True, interactive=False)
with gr.Tab("Charts"):
chart_map = gr.HTML(value=_chart_iframe("intertopic_map.html"))
chart_bar = gr.HTML(value=_chart_iframe("bar_chart.html"))
chart_tree = gr.HTML(value=_chart_iframe("treemap.html"))
with gr.Tab("Downloads"):
downloads = gr.File(value=_download_files(), label="Generated deliverables", file_count="multiple")
outputs = [
summary,
phase,
cluster_table,
cluster_cards,
optimizer_table,
compliance_panel,
compliance_table,
council_viz,
council_table,
tccm_table,
tccm_dual_status,
tccm_dual_table,
chart_map,
chart_bar,
chart_tree,
downloads,
]
run_btn.click(fn=_run, inputs=[csv_file], outputs=outputs, show_api=False, api_name=False)
refresh_btn.click(fn=_refresh, inputs=None, outputs=outputs, show_api=False, api_name=False)
tccm_dual_btn.click(
fn=_on_tccm_dual_validate,
inputs=[notebook_file, second_llm_file],
outputs=[tccm_dual_status, tccm_dual_table, downloads],
show_api=False,
api_name=False,
)
notebook_paste_btn.click(
fn=_on_notebooklm_paste,
inputs=[notebook_paste],
outputs=[tccm_dual_status, tccm_dual_table, downloads],
show_api=False,
api_name=False,
)
if __name__ == "__main__":
demo.launch(server_name="0.0.0.0", server_port=7860, ssr_mode=False)