import html import json import os from typing import List import gradio as gr import pandas as pd from topic_pipeline import ( OUTPUT_DIR, parse_notebooklm_tccm_text, run_complete_pipeline, write_tccm_dual_validation, ) os.makedirs(OUTPUT_DIR, exist_ok=True) def _exists(name: str) -> bool: return os.path.exists(os.path.join(OUTPUT_DIR, name)) def _load_json(name: str): with open(os.path.join(OUTPUT_DIR, name), "r", encoding="utf-8") as f: return json.load(f) def _download_files() -> List[str]: names = [ "comparison.csv", "taxonomy_map.json", "topic_model_report.md", "narrative.txt", "cluster_optimization_log.csv", "llm_council_validation.csv", "tccm_validation.csv", "tccm_dual_validation.csv", "notebooklm_extraction.csv", "compliance_checklist.csv", "compliance_checklist.json", "run_metadata.json", "combined_labels.json", ] return [os.path.join(OUTPUT_DIR, name) for name in names if _exists(name)] def _phase_html() -> str: phases = [ ("Corpus", _exists("corpus_config.json")), ("Embeddings", _exists("combined_emb.npy")), ("Optimization", _exists("cluster_optimization_log.csv")), ("Clusters", _exists("combined_labels.json")), ("Council", _exists("llm_council_validation.csv")), ("TCCM", _exists("tccm_validation.csv")), ("Compliance", _exists("compliance_checklist.csv")), ("Report", _exists("topic_model_report.md")), ] chips = [] for name, done in phases: bg = "#0f766e" if done else "#334155" mark = "OK" if done else "--" chips.append( f"{mark} {name}" ) return "
" + "".join(chips) + "
" def _cluster_table(): if not _exists("combined_labels.json"): return [] rows = [] for s in _load_json("combined_labels.json"): rows.append([ s.get("cluster_id"), s.get("label"), s.get("category"), s.get("paper_count"), s.get("confidence"), s.get("agreement_score"), "; ".join(s.get("keywords", [])[:8]), " | ".join(s.get("top_titles", [])[:3]), s.get("reasoning", ""), ]) return rows def _council_table(): path = os.path.join(OUTPUT_DIR, "llm_council_validation.csv") if not os.path.exists(path): return [] return pd.read_csv(path).head(120) def _council_viz_html() -> str: path = os.path.join(OUTPUT_DIR, "llm_council_validation.csv") if not os.path.exists(path): return ( "
Run the pipeline to activate the LLM Council " "validation board.
" ) df = pd.read_csv(path) if df.empty: return "
Council validation file is empty.
" grouped = list(df.groupby(["cluster_id", "final_label"], sort=False))[:6] rows = [] avg_agreement = float(df["agreement_score"].mean()) if "agreement_score" in df else 0 avg_confidence = float(df["confidence"].mean()) if "confidence" in df else 0 llm_member_present = df["member"].astype(str).str.contains("LLM|Mistral", case=False, regex=True).any() llm_status = "Mistral LLM active" if llm_member_present else "Local semantic fallback active" for (cluster_id, final_label), group in grouped: votes = [] for _, row in group.iterrows(): member = html.escape(str(row.get("member", ""))) label = html.escape(str(row.get("member_label", ""))) method = html.escape(str(row.get("method", ""))) votes.append( "
" "
" f"
{member}{label}{method}
" "
" ) confidence = int(float(group["confidence"].iloc[0]) * 100) agreement = int(float(group["agreement_score"].iloc[0]) * 100) rows.append( "
" "
" f"Cluster {html.escape(str(cluster_id))}" f"{html.escape(str(final_label))}" "
" "
" + "".join(votes) + "
" "Accepted label" f"{html.escape(str(final_label))}" f"{confidence}% confidence | {agreement}% agreement" "
" "
" "
" ) return ( "
" "
" "

LLM Council Validation Running In-App

" "

Three independent validators inspect each cluster label, compare votes, " "and write the accepted label plus agreement score into the export file.

" "
" f"
{len(df['cluster_id'].unique())}clusters checked
" f"
{int(avg_agreement * 100)}%avg agreement
" f"
{int(avg_confidence * 100)}%avg confidence
" f"
{html.escape(llm_status)}council mode
" "
" "
" "
1
Keyword Extractor
" "
" "
2
PAJAIS Mapper
" "
" "
3
LLM / Semantic Judge
" "
" "
OK
Validated Label
" "
" + "".join(rows) + "
" ) def _optimizer_table(): path = os.path.join(OUTPUT_DIR, "cluster_optimization_log.csv") if not os.path.exists(path): return [] df = pd.read_csv(path) cols = [ c for c in [ "algorithm", "umap_n_neighbors", "umap_n_components", "hdbscan_min_cluster_size", "hdbscan_min_samples", "n_clusters", "noise_ratio", "min_size", "max_size", "too_small", "too_large", "silhouette_cosine", "score", "optimizer_recommendation", ] if c in df.columns ] return df[cols].head(80) def _tccm_table(): path = os.path.join(OUTPUT_DIR, "tccm_validation.csv") if not os.path.exists(path): return [] return pd.read_csv(path).head(100) def _tccm_dual_table(): path = os.path.join(OUTPUT_DIR, "tccm_dual_validation.csv") if not os.path.exists(path): return [] return pd.read_csv(path).head(100) def _compliance_table(): path = os.path.join(OUTPUT_DIR, "compliance_checklist.csv") if not os.path.exists(path): return [] return pd.read_csv(path) def _compliance_html() -> str: path = os.path.join(OUTPUT_DIR, "compliance_checklist.csv") if not os.path.exists(path): return ( "
Run the pipeline to generate the professor-requirement " "compliance checklist.
" ) df = pd.read_csv(path) color_map = { "PASS": "#0f766e", "FAIL": "#b91c1c", "CONFIG_REQUIRED": "#b45309", "ENV_FALLBACK": "#b45309", "INPUT_REQUIRED": "#b45309", "PARTIAL": "#7c3aed", "MANUAL_REQUIRED": "#475569", "REVIEW": "#7c3aed", } rows = [] for _, row in df.iterrows(): status = str(row.get("Status", "REVIEW")) color = color_map.get(status, "#475569") rows.append( "
" f"{html.escape(status)}" f"{html.escape(str(row.get('Requirement', '')))}" f"

{html.escape(str(row.get('Evidence', '')))}

" f"{html.escape(str(row.get('File', '')))}" "
" ) return ( "
" "

Professor Requirement Compliance Checklist

" "

This separates completed app evidence from items that still need API secrets, " "NotebookLM/full-text inputs, or mentor approval.

" "
" + "".join(rows) + "
" ) def _tccm_dual_status_html() -> str: path = os.path.join(OUTPUT_DIR, "tccm_dual_validation.csv") if not os.path.exists(path): return ( "
Upload NotebookLM and second-LLM extraction CSVs " "to generate TCCM dual validation.
" ) df = pd.read_csv(path) status_col = "Final_TCCM_Compliance_Status" if status_col not in df.columns: return "
TCCM dual validation is pending source uploads.
" counts = df[status_col].value_counts().to_dict() cards = [] for status, count in counts.items(): ok = "COMPLIANT" in str(status) color = "#0f766e" if ok else "#b45309" cards.append( f"
{count}" f"{html.escape(str(status))}
" ) return ( "

TCCM Dual Validation Status

" "

Required by email: NotebookLM extraction plus another LLM/extraction method. " "This screen reconciles those files with regex/semantic extraction.

" "
" + "".join(cards) + "
" ) def _on_tccm_dual_validate(notebook_file, second_file): notebook_path = notebook_file if isinstance(notebook_file, str) else getattr(notebook_file, "name", "") second_path = second_file if isinstance(second_file, str) else getattr(second_file, "name", "") write_tccm_dual_validation(notebook_path, second_path) return _tccm_dual_status_html(), _tccm_dual_table(), _download_files() def _on_notebooklm_paste(notebook_text): if not str(notebook_text or "").strip(): return ( "
Paste the NotebookLM table text first.
", _tccm_dual_table(), _download_files(), ) notebook_path = parse_notebooklm_tccm_text(notebook_text) write_tccm_dual_validation(notebook_path, "") count = len(pd.read_csv(notebook_path)) if os.path.exists(notebook_path) else 0 status = ( f"

NotebookLM Paste Imported

" f"

Parsed {count} NotebookLM rows into outputs/notebooklm_extraction.csv. " "Merged with the independent regex/semantic extractor in " "outputs/tccm_dual_validation.csv. Upload a second-LLM CSV as well " "for full NotebookLM + second LLM compliance.

" + _tccm_dual_status_html() ) return status, _tccm_dual_table(), _download_files() def _chart_iframe(name: str) -> str: path = os.path.join(OUTPUT_DIR, "combined_charts", name) if not os.path.exists(path): return ( "
" "Run the pipeline to generate this chart.
" ) with open(path, "r", encoding="utf-8") as f: srcdoc = f.read().replace("&", "&").replace('"', """) return ( f"" ) def _cards_html() -> str: if not _exists("combined_labels.json"): return ( "
" "Clusters will appear here after a complete run.
" ) cards = [] for s in _load_json("combined_labels.json"): evidence = html.escape(" | ".join(s.get("top_titles", [])[:3])) label = html.escape(s.get("label", "Cluster")) category = html.escape(s.get("category", "Unmapped")) keywords = html.escape(", ".join(s.get("keywords", [])[:8])) conf = int(float(s.get("confidence", 0)) * 100) cards.append( "
" f"
{label}
" f"
{category}
" f"
" f"{s.get('paper_count', 0)} papers | confidence {conf}% | agreement {s.get('agreement_score', 0)}
" f"
{keywords}
" f"
{evidence}
" "
" ) return ( "
" + "".join(cards) + "
" ) def _summary_markdown(result=None) -> str: if result is None and not _exists("run_metadata.json"): return ( "Upload the Scopus CSV and click **Run Complete Pipeline**. " "The app will generate paper-level Title+Abstract+DOI embeddings, optimize " "UMAP/HDBSCAN clustering, label 15-25 clusters through an in-app council, " "map them to PAJAIS, and export TCCM validation files." ) meta = result or {} if not meta: meta = { "parameters": _load_json("run_metadata.json").get("selected_parameters", {}), "embedding": _load_json("run_metadata.json").get("embedding", {}), "clusters": _load_json("combined_labels.json") if _exists("combined_labels.json") else [], "taxonomy": _load_json("taxonomy_map.json") if _exists("taxonomy_map.json") else {}, "config": _load_json("corpus_config.json") if _exists("corpus_config.json") else {}, } params = meta.get("parameters", {}) emb = meta.get("embedding", {}) tax = meta.get("taxonomy", {}).get("coverage_stats", {}) cfg = meta.get("config", {}) return ( f"**Run complete.** Analysed {cfg.get('rows', 'N/A')} papers from " f"{cfg.get('journal', 'the corpus')} ({cfg.get('year_min')} to {cfg.get('year_max')}).\n\n" f"Selected clustering: `{params.get('algorithm')}` with " f"`{params.get('n_clusters')}` clusters, min size `{params.get('min_size')}`, " f"max size `{params.get('max_size')}`, noise ratio `{params.get('noise_ratio')}`.\n\n" f"Embedding: `{emb.get('embedding_model')}`. PAJAIS mapped: " f"`{tax.get('mapped', 0)}`; novel: `{tax.get('novel', 0)}`. " "Download the optimizer log and council validation for the final submission appendix." ) def _run(file_obj): if file_obj is None: return ( "Upload a CSV first.", _phase_html(), _cluster_table(), _cards_html(), _optimizer_table(), _compliance_html(), _compliance_table(), _council_viz_html(), _council_table(), _tccm_table(), _tccm_dual_status_html(), _tccm_dual_table(), _chart_iframe("intertopic_map.html"), _chart_iframe("bar_chart.html"), _chart_iframe("treemap.html"), _download_files(), ) filepath = file_obj if isinstance(file_obj, str) else file_obj.name result = run_complete_pipeline(filepath) return ( _summary_markdown(result), _phase_html(), _cluster_table(), _cards_html(), _optimizer_table(), _compliance_html(), _compliance_table(), _council_viz_html(), _council_table(), _tccm_table(), _tccm_dual_status_html(), _tccm_dual_table(), _chart_iframe("intertopic_map.html"), _chart_iframe("bar_chart.html"), _chart_iframe("treemap.html"), result["deliverables"], ) def _refresh(): return ( _summary_markdown(), _phase_html(), _cluster_table(), _cards_html(), _optimizer_table(), _compliance_html(), _compliance_table(), _council_viz_html(), _council_table(), _tccm_table(), _tccm_dual_status_html(), _tccm_dual_table(), _chart_iframe("intertopic_map.html"), _chart_iframe("bar_chart.html"), _chart_iframe("treemap.html"), _download_files(), ) CSS = """ .gradio-container { max-width: 1360px !important; } .app-title { padding: 18px 0 8px; } .app-title h1 { margin: 0; font-size: 30px; color: #0f172a; letter-spacing: 0; } .app-title p { color: #475569; margin: 6px 0 0; } .compliance-empty { padding: 24px; border: 1px dashed #94a3b8; border-radius: 8px; background: #f8fafc; color: #475569; } .compliance-board, .tccm-status { background: #ffffff; border: 1px solid #d8dee9; border-radius: 10px; padding: 16px; } .compliance-board h3, .tccm-status h3 { margin: 0; color: #0f172a; font-size: 20px; } .compliance-board p, .tccm-status p { color: #475569; margin: 6px 0 14px; line-height: 1.45; } .compliance-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 10px; } .compliance-row { border: 1px solid #e2e8f0; border-radius: 8px; padding: 12px; background: #f8fafc; } .compliance-row span { display: inline-block; color: white; font-size: 11px; font-weight: 800; padding: 3px 8px; border-radius: 999px; margin-bottom: 8px; } .compliance-row strong { display: block; color: #0f172a; font-size: 14px; } .compliance-row p { font-size: 12px; margin: 6px 0; color: #475569; } .compliance-row small { color: #64748b; font-size: 11px; } .tccm-status > div { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 10px; } .tccm-card { border: 1px solid #e2e8f0; border-radius: 8px; padding: 12px; background: #f8fafc; } .tccm-card strong { display: block; font-size: 24px; } .tccm-card span { color: #475569; font-size: 12px; font-weight: 700; } .council-empty { padding: 28px; border: 1px dashed #94a3b8; color: #475569; border-radius: 8px; background: #f8fafc; } .council-board { background: #08111f; color: #e5edf7; border-radius: 10px; padding: 18px; border: 1px solid #1e3a5f; } .council-top { display: grid; grid-template-columns: minmax(280px, 1.2fr) minmax(320px, 1fr); gap: 16px; align-items: start; } .council-top h3 { margin: 0; font-size: 20px; letter-spacing: 0; } .council-top p { margin: 6px 0 0; color: #9fb3c8; line-height: 1.45; } .council-metrics { display: grid; grid-template-columns: repeat(2, minmax(140px, 1fr)); gap: 8px; } .council-metrics div { background: #10243a; border: 1px solid #1f4568; border-radius: 8px; padding: 10px; } .council-metrics strong { display: block; color: #5eead4; font-size: 18px; } .council-metrics span { color: #a8bed4; font-size: 12px; } .council-lane { display: grid; grid-template-columns: 1fr 60px 1fr 60px 1fr 60px 1fr; gap: 8px; align-items: center; margin: 20px 0; } .pulse-node { min-height: 62px; display: grid; place-items: center; text-align: center; border: 1px solid #2563eb; background: #0f2546; border-radius: 8px; color: #bfdbfe; font-weight: 800; animation: councilGlow 1.8s ease-in-out infinite; } .pulse-node span { display: block; font-size: 11px; font-weight: 600; color: #dbeafe; margin-top: 3px; } .pulse-node.final { border-color: #14b8a6; background: #0f3b39; color: #99f6e4; } .pulse-line { height: 3px; border-radius: 999px; background: linear-gradient(90deg, #2563eb, #14b8a6, #2563eb); background-size: 220% 100%; animation: councilFlow 1.1s linear infinite; } .council-cluster { margin-top: 10px; padding: 12px; border: 1px solid #1f4568; border-radius: 8px; background: #0c1b2c; } .cluster-head { display: flex; justify-content: space-between; gap: 12px; color: #cbd5e1; margin-bottom: 10px; } .cluster-head span { color: #7dd3fc; font-weight: 800; } .cluster-head strong { color: #f8fafc; } .council-flow { display: grid; grid-template-columns: repeat(4, minmax(160px, 1fr)); gap: 8px; } .council-vote, .final-label { border-radius: 8px; padding: 10px; background: #10243a; border: 1px solid #1f4568; min-height: 82px; } .council-vote { display: flex; gap: 8px; align-items: flex-start; } .vote-dot { width: 10px; height: 10px; margin-top: 4px; border-radius: 50%; background: #5eead4; box-shadow: 0 0 14px #5eead4; animation: councilBlink 1.2s ease-in-out infinite; flex: 0 0 auto; } .council-vote strong, .final-label strong { display: block; color: #e2e8f0; font-size: 13px; } .council-vote span, .final-label span { display: block; color: #5eead4; font-size: 12px; margin-top: 3px; } .council-vote small, .final-label small { display: block; color: #94a3b8; font-size: 11px; margin-top: 4px; line-height: 1.25; } .final-label { border-color: #14b8a6; background: #0d302f; } @keyframes councilFlow { from { background-position: 0% 0; } to { background-position: 220% 0; } } @keyframes councilGlow { 0%, 100% { box-shadow: 0 0 0 rgba(37,99,235,0.2); } 50% { box-shadow: 0 0 22px rgba(20,184,166,0.45); } } @keyframes councilBlink { 0%, 100% { opacity: .35; transform: scale(.75); } 50% { opacity: 1; transform: scale(1.15); } } @media (max-width: 900px) { .council-top, .council-flow { grid-template-columns: 1fr; } .council-lane { grid-template-columns: 1fr; } .pulse-line { height: 18px; width: 3px; justify-self: center; } } """ with gr.Blocks(title="SPJIMR Topic Modelling Submission App", css=CSS, theme=gr.themes.Soft()) as demo: gr.HTML( "

SPJIMR Topic Modelling Submission App

" "

Paper-level Title + Abstract + DOI vectors, UMAP/HDBSCAN optimization, " "LLM council validation, PAJAIS mapping, and TCCM extraction appendix.

" ) phase = gr.HTML(value=_phase_html()) with gr.Row(): csv_file = gr.File(label="Upload Scopus CSV", file_types=[".csv"], scale=3) with gr.Column(scale=1): run_btn = gr.Button("Run Complete Pipeline", variant="primary") refresh_btn = gr.Button("Refresh Outputs") summary = gr.Markdown(value=_summary_markdown()) with gr.Tabs(): with gr.Tab("Clusters"): cluster_table = gr.Dataframe( headers=[ "Cluster ID", "Label", "PAJAIS Category", "Papers", "Confidence", "Agreement", "Keywords", "Top 3 Titles", "Reasoning", ], value=_cluster_table(), wrap=True, interactive=False, ) cluster_cards = gr.HTML(value=_cards_html()) with gr.Tab("Optimization"): optimizer_table = gr.Dataframe(value=_optimizer_table(), wrap=True, interactive=False) with gr.Tab("Compliance"): compliance_panel = gr.HTML(value=_compliance_html()) compliance_table = gr.Dataframe(value=_compliance_table(), wrap=True, interactive=False) with gr.Tab("Council Validation"): council_viz = gr.HTML(value=_council_viz_html()) council_table = gr.Dataframe(value=_council_table(), wrap=True, interactive=False) with gr.Tab("TCCM Validation"): tccm_table = gr.Dataframe(value=_tccm_table(), wrap=True, interactive=False) with gr.Tab("TCCM Dual Validation"): gr.Markdown( "Upload the **NotebookLM extraction CSV** and the **second LLM extraction CSV** " "from full-text PDFs. Expected columns can include Title, DOI, Theory, Context, " "Variables/Constructs, Method, and Computational Techniques." ) notebook_paste = gr.Textbox( label="Paste NotebookLM table output", lines=10, placeholder="Paste the copied NotebookLM table here, including Paper ID / Paper Citation / Study Type / DV / IV / Evidence lines...", ) notebook_paste_btn = gr.Button("Import NotebookLM Paste", variant="secondary") with gr.Row(): notebook_file = gr.File(label="NotebookLM extraction CSV", file_types=[".csv"]) second_llm_file = gr.File(label="Second LLM extraction CSV", file_types=[".csv"]) tccm_dual_btn = gr.Button("Validate NotebookLM + Second LLM", variant="primary") tccm_dual_status = gr.HTML(value=_tccm_dual_status_html()) tccm_dual_table = gr.Dataframe(value=_tccm_dual_table(), wrap=True, interactive=False) with gr.Tab("Charts"): chart_map = gr.HTML(value=_chart_iframe("intertopic_map.html")) chart_bar = gr.HTML(value=_chart_iframe("bar_chart.html")) chart_tree = gr.HTML(value=_chart_iframe("treemap.html")) with gr.Tab("Downloads"): downloads = gr.File(value=_download_files(), label="Generated deliverables", file_count="multiple") outputs = [ summary, phase, cluster_table, cluster_cards, optimizer_table, compliance_panel, compliance_table, council_viz, council_table, tccm_table, tccm_dual_status, tccm_dual_table, chart_map, chart_bar, chart_tree, downloads, ] run_btn.click(fn=_run, inputs=[csv_file], outputs=outputs, show_api=False, api_name=False) refresh_btn.click(fn=_refresh, inputs=None, outputs=outputs, show_api=False, api_name=False) tccm_dual_btn.click( fn=_on_tccm_dual_validate, inputs=[notebook_file, second_llm_file], outputs=[tccm_dual_status, tccm_dual_table, downloads], show_api=False, api_name=False, ) notebook_paste_btn.click( fn=_on_notebooklm_paste, inputs=[notebook_paste], outputs=[tccm_dual_status, tccm_dual_table, downloads], show_api=False, api_name=False, ) if __name__ == "__main__": demo.launch(server_name="0.0.0.0", server_port=7860, ssr_mode=False)