from __future__ import annotations
import os, re, html, pathlib, math, json, sys, traceback, statistics, warnings, io
from datetime import datetime
from typing import List, Dict, Tuple, Optional

import tkinter as tk
from tkinter import ttk, filedialog, messagebox

import numpy as np
import pandas as pd
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

warnings.filterwarnings("ignore", category=RuntimeWarning)

try:
    import ftfy
except Exception:
    ftfy = None
try:
    from wordcloud import WordCloud
except Exception:
    WordCloud = None

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from scipy.stats import ttest_ind, ttest_rel
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.cluster import KMeans


try:
    import statsmodels.formula.api as smf
except Exception:
    smf = None


from collections import Counter


import numpy as np
import pandas as pd
from typing import Tuple

def _excel_col_to_idx(col_letter: str) -> int:
    col_letter = str(col_letter).strip().upper()
    n = 0
    for ch in col_letter:
        if 'A' <= ch <= 'Z':
            n = n * 26 + (ord(ch) - ord('A') + 1)
    return n - 1  

def _pick_one_hot_category(df: pd.DataFrame, start_col: str, end_col: str,
                           truthy=("1","x","✓","true","yes","si","sì")) -> Tuple[pd.Series, pd.Series]:
    tset = set(s.lower() for s in truthy)
    c0, c1 = _excel_col_to_idx(start_col), _excel_col_to_idx(end_col)
    cols = df.columns.tolist()
    subcols = [c for i, c in enumerate(cols) if c0 <= i <= c1]

    cat_vals, warns = [], []
    for _, row in df.iterrows():
        hits = []
        for c in subcols:
            val = str(row.get(c, "")).strip().lower()
            if val in tset:
                hits.append(c)
        if len(hits) == 1:
            cat_vals.append(hits[0])     
            warns.append(np.nan)
        elif len(hits) == 0:
            cat_vals.append(np.nan)
            warns.append(f"NONE_SELECTED:{start_col}-{end_col}")
        else:
            cat_vals.append(np.nan)
            warns.append(f"MULTI_SELECTED:{start_col}-{end_col}:{'|'.join(hits)}")
    return pd.Series(cat_vals, index=df.index), pd.Series(warns, index=df.index)

def _resolve_etw_flag(df: pd.DataFrame, start_col: str, end_col: str,
                      true_vals=("si","sì","yes","true","1","x","✓"),
                      false_vals=("no","false","0")) -> Tuple[pd.Series, pd.Series]:
    tset, fset = set(s.lower() for s in true_vals), set(s.lower() for s in false_vals)
    c0, c1 = _excel_col_to_idx(start_col), _excel_col_to_idx(end_col)
    cols = df.columns.tolist()
    subcols = [c for i, c in enumerate(cols) if c0 <= i <= c1]

    flags, warns = [], []
    for _, row in df.iterrows():
        tr_hit = any(str(row.get(c, "")).strip().lower() in tset for c in subcols)
        fl_hit = any(str(row.get(c, "")).strip().lower() in fset for c in subcols)
        if tr_hit and not fl_hit:
            flags.append(True);  warns.append(np.nan)
        elif fl_hit and not tr_hit:
            flags.append(False); warns.append(np.nan)
        elif not tr_hit and not fl_hit:
            flags.append(np.nan); warns.append("ETW_NONE_SELECTED")
        else:
            flags.append(np.nan); warns.append("ETW_BOTH_TRUE_FALSE")
    return pd.Series(flags, index=df.index), pd.Series(warns, index=df.index)


import re as _re_inj, math as _math_inj
_WORD_RE_INJ = _re_inj.compile(r"[A-Za-zÀ-ÖØ-öø-ÿ']+")

def _simple_tokenize(text: str):
    return _WORD_RE_INJ.findall(str(text).lower())

def _score_arousal_dominance(text, aro_lex, dom_lex, boosters, negators, window=2, max_token_contrib=3.0):
    
    toks = _simple_tokenize(text)
    n = len(toks)
    tot_a = 0.0; cnt_a = 0
    tot_d = 0.0; cnt_d = 0
    for i, w in enumerate(toks):
        a = aro_lex.get(w) if aro_lex else None
        d = dom_lex.get(w) if dom_lex else None
        if a is None and d is None:
            continue
        L = max(0, i-window); R = min(n, i+1+window)
        ctx = toks[L:i] + toks[i+1:R]
        boost = 1.0
        if boosters:
            for b in ctx:
                if b in boosters:
                    try: boost *= float(boosters[b])
                    except Exception: pass
        
        if a is not None:
            contrib_a = max(-max_token_contrib, min(max_token_contrib, a * boost))
            tot_a += contrib_a; cnt_a += 1
        if d is not None:
            contrib_d = max(-max_token_contrib, min(max_token_contrib, d * boost))
            tot_d += contrib_d; cnt_d += 1
    arousal = (tot_a / cnt_a) if cnt_a else float("nan")
    dominance = (tot_d / cnt_d) if cnt_d else float("nan")
    return (float(arousal), float(dominance))

def _quadrant(valence, arousal, qcfg):
    
    if arousal is None or (isinstance(arousal, float) and _math_inj.isnan(arousal)):
        return "Neutral/MissingArousal"
    vpos = float(qcfg.get("val_pos", 0.05)); vneg = float(qcfg.get("val_neg", -0.05)); ah = float(qcfg.get("arousal_high", 0.25))
    if valence >= vpos and arousal >= ah: return "Positive-HighArousal"
    if valence >= vpos and arousal <  ah: return "Positive-LowArousal"
    if valence <= vneg and arousal >= ah: return "Negative-HighArousal"
    if valence <= vneg and arousal <  ah: return "Negative-LowArousal"
    return "Neutral/Mixed"



def _norm_time_label(x: str) -> str:
    s = str(x).strip().lower()
    if s in {"pre","ingresso","baseline","t0","0","before"}: return "pre"
    if s in {"post","uscita","followup","t1","1","after"}:   return "post"
    return s

def _is_treated(s: str) -> int:
    ss = str(s).strip().lower()
    return int("twin" in ss or "sper" in ss or "treat" in ss or "etwinning" in ss)

def _welch_t_pvalue(x_pre, x_post):
    
    import numpy as np
    x_pre = np.asarray(x_pre, float)
    x_post = np.asarray(x_post, float)
    x_pre = x_pre[~np.isnan(x_pre)]
    x_post = x_post[~np.isnan(x_post)]
    if len(x_pre) < 2 or len(x_post) < 2:
        return float("nan")
    m1, m2 = x_pre.mean(), x_post.mean()
    v1, v2 = x_pre.var(ddof=1), x_post.var(ddof=1)
    n1, n2 = len(x_pre), len(x_post)
    se = math.sqrt(v1/n1 + v2/n2)
    if se == 0: return float("nan")
    t = (m2 - m1) / se
    
    df = (v1/n1 + v2/n2)**2 / ((v1**2)/((n1**2)*(n1-1)) + (v2**2)/((n2**2)*(n2-1)))
    from scipy import stats
    return 2*stats.t.sf(abs(t), df)

def _did_ols_pvalue(df_area, metric, group_col="group", time_col="time"):
    
    if smf is None:
        return (float("nan"), float("nan"))
    d = df_area[[metric, group_col, time_col]].dropna().copy()
    if d.empty: return (float("nan"), float("nan"))
    d["treated"] = d[group_col].map(_is_treated)
    d["post"]    = d[time_col].map(_norm_time_label).eq("post").astype(int)
    try:
        model = smf.ols(f"{metric} ~ treated + post + treated:post", data=d).fit(cov_type="HC3")
        coef = model.params.get("treated:post", float("nan"))
        pval = model.pvalues.get("treated:post", float("nan"))
        return (float(coef), float(pval))
    except Exception:
        return (float("nan"), float("nan"))

def _did_ols_pvalue_cluster(df_area, metric, group_col="group", time_col="time", cluster_col=None):
    
    if smf is None or cluster_col is None or cluster_col not in df_area.columns:
        return (float("nan"), float("nan"))
    d = df_area[[metric, group_col, time_col, cluster_col]].dropna().copy()
    if d.empty: 
        return (float("nan"), float("nan"))
    d["treated"] = d[group_col].map(_is_treated)
    d["post"]    = d[time_col].map(_norm_time_label).eq("post").astype(int)
    try:
        model = smf.ols(f"{metric} ~ treated + post + treated:post", data=d).fit(
            cov_type="cluster", cov_kwds={"groups": d[cluster_col]}
        )
        coef = model.params.get("treated:post", float("nan"))
        pval = model.pvalues.get("treated:post", float("nan"))
        return (float(coef), float(pval))
    except Exception:
        return (float("nan"), float("nan"))






SCHOOL_COLS: list[str] = [chr(c) for c in range(ord("J"), ord("X")+1)]  
BAN_TERMS_DEFAULT  = {"amo","tv","ex"}  

_MOJI_MAP = {"‚Äô":"’","‚Äù":"”","‚Äú":"“","â€™":"’","â€œ":"“","â€\x9d":"”","â€“":"–","â€”":"—","√®":"è","√†":"à","√≤":"ò","√©":"é","√™":"™"}
def fix_mojibake_text(s: str) -> str:
    if not isinstance(s, str):
        return s
    try:
        import ftfy as _f
        return _f.fix_text(s)
    except Exception:
        pass
    for k, v in _MOJI_MAP.items():
        if k in s: s = s.replace(k, v)
    return s


_DEF_A2I = {chr(i+65): i for i in range(26)}
def excel_col_to_index(col: str) -> int:
    col = col.strip().upper()
    idx = 0
    for ch in col:
        if 'A' <= ch <= 'Z':
            idx = idx * 26 + (_DEF_A2I[ch] + 1)
    return idx - 1


def load_questionnaire_lettered(path: str) -> tuple[pd.DataFrame, dict]:
    df = pd.read_csv(
        path,
        header=None,
        dtype=str,
        sep=None,             
        engine="python",
        keep_default_na=False 
    )
    
    def excel_col(i: int) -> str:
        out = ""
        i0 = i
        while True:
            i0, r = divmod(i0, 26)
            out = chr(r + 65) + out
            if i0 == 0:
                break
            i0 -= 1
        return out
    df.columns = [excel_col(i) for i in range(df.shape[1])]
    
    desc = df.iloc[1]
    desc = desc.apply(fix_mojibake_text)
    school_map = {c: desc[c] for c in SCHOOL_COLS if c in df.columns}
    
    df = df.apply(lambda col: col.map(fix_mojibake_text))
    return df, school_map

def ensure_dir(p: str):
    pathlib.Path(p).mkdir(parents=True, exist_ok=True)

def write_text(path: str, content: str):
    with open(path, "w", encoding="utf-8") as f:
        f.write(content)

def safe_write_csv(df: pd.DataFrame, path: str, columns: Optional[List[str]] = None):
    if df is None:
        df = pd.DataFrame(columns=columns or [])
    elif df.empty and columns is not None:
        df = pd.DataFrame(columns=columns)
    df.to_csv(path, index=False, encoding="utf-8")


_IT_STOP_BASE = set("""
 a ad al allo alla ai agli alle all' allo' anche come con col coi cosa così cosi da dal dallo dalla dai dagli dalle dal' del dello della dei degli delle di in ne nei negli nelle nell' no non o od oppure per però pero più piu poiché poiche quale quali che chi cui se si sia siano suo sua suoi sue sul sullo sulla sui sugli sulle tra fra un una uno gli il la lo i le degli delle dei
""".split())
_word_re = re.compile(r"[A-Za-zÀ-ÖØ-öø-ÿ']+")
def build_tokenizer(stop_extra: set, keep_terms: set, ban_terms: set):
    stop = (_IT_STOP_BASE | set(stop_extra)) - set(keep_terms)
    ban_terms = set(ban_terms or [])
    def _tok(text: str) -> List[str]:
        text = fix_mojibake_text(text or "")
        tokens = [t.lower() for t in _word_re.findall(text)]
        return [t for t in tokens if (len(t) > 1 and (t in keep_terms or (t not in stop and t not in ban_terms)))]  
    return _tok



_SPACY_IT = None
def _init_spacy_it():
    global _SPACY_IT
    if _SPACY_IT is not None:
        return _SPACY_IT
    try:
        import spacy  
        
        for name in ("it_core_news_sm", "it_core_news_md", "it_core_news_lg"):
            try:
                _SPACY_IT = spacy.load(name)
                break
            except Exception:
                continue
        if _SPACY_IT is None:
            _SPACY_IT = "TOKENIZER_ONLY"
    except Exception:
        _SPACY_IT = None
    return _SPACY_IT

def build_bow_tokenizer(stop_extra: set, keep_terms: set, ban_terms: set, lemmatize: bool = True):
    
    stop = (_IT_STOP_BASE | set(stop_extra)) - set(keep_terms)
    ban_terms = set(ban_terms or [])
    nlp = _init_spacy_it() if lemmatize else None

    def _tok_bow(text: str):
        text = fix_mojibake_text(text or "")
        if nlp and nlp not in (None, "TOKENIZER_ONLY"):
            doc = nlp(text)
            toks = [(t.lemma_ or t.text).lower() for t in doc if t.is_alpha]
        else:
            toks = [t.lower() for t in _word_re.findall(text)]
        out = []
        for t in toks:
            if len(t) <= 1: 
                continue
            if t in keep_terms:
                out.append(t); continue
            if t in ban_terms: 
                continue
            if t in stop: 
                continue
            out.append(t)
        return out
    return _tok_bow



def derive_stopwords_from_corpus(texts: List[str], keep_terms: set, min_doc_frac: float = 0.35, min_tf_short: int = 10) -> set:
    from collections import Counter
    tf = Counter(); df = Counter()
    N = max(1, len(texts))
    for s in texts:
        toks = [w.lower() for w in _word_re.findall(s or "")]
        toks_set = set(toks)
        for w in toks: tf[w]+=1
        for w in toks_set: df[w]+=1
    out = set()
    for w in list(tf.keys()):
        if w in keep_terms: continue
        if w.isdigit(): continue
        dfr = df[w]/N
        if dfr >= min_doc_frac or (len(w)<=3 and tf[w]>=min_tf_short):
            out.add(w)
    return out




_POS = set(" bene buono ottimo positivo felice contento soddisfatto sereno tranquillo migliorato riuscito efficace utili ".split())
_NEG = set(" male cattivo pessimo negativo triste arrabbiato frustrato ansioso stressato difficile noioso fallito inutile ".split())

def sentiment_baseline(texts: List[str], tokenizer) -> pd.Series:
    vals = []
    for t in texts:
        toks = tokenizer(t)
        pos = sum(1 for w in toks if w in _POS)
        neg = sum(1 for w in toks if w in _NEG)
        score = (pos - neg) / (len(toks) + 1e-9)
        vals.append(score)
    return pd.Series(vals)

def ttr_simple(text: str, tokenizer) -> float:
    toks = tokenizer(text)
    return len(set(toks)) / max(1, len(toks))

def mtld_std(text: str, tokenizer, ttr_threshold: float = 0.72) -> float:
    toks = tokenizer(text)
    if len(toks) < 50:
        return float('nan')
    factors = 0
    start = 0
    types = set()
    for i, tok in enumerate(toks):
        types.add(tok)
        span = i - start + 1
        if len(types) / span < ttr_threshold:
            factors += 1
            start = i + 1
            types = set()
    if i - start + 1 > 0:
        denom = (i - start + 1)
        val = len(types) / denom if denom else 0
        factors += (1 - (val) / ttr_threshold)
    if factors == 0:
        return float('nan')
    return len(toks) / factors

def sentiment_from_json(texts, tokenizer, cfg):
    
    import pandas as pd

    lx = {k.lower(): float(v) for k, v in cfg["sentiment"].get("lexicon", {}).items()}
    negators = set(t.lower() for t in cfg["sentiment"].get("negators", []))
    boosters = {k.lower(): float(v) for k, v in cfg["sentiment"].get("boosters", {}).items()}
    window = int(cfg["sentiment"].get("window", 3))
    max_token_contrib = float(cfg["sentiment"].get("max_token_contrib", 3.0))

    
    aro_lex = {k.lower(): float(v) for k, v in cfg["sentiment"].get("arousal_lexicon", {}).items()}
    dom_lex = {k.lower(): float(v) for k, v in cfg["sentiment"].get("dominance_lexicon", {}).items()}
    

    vals = []
    for s in texts:
        toks = tokenizer(str(s))  
        total = 0.0

        for i, w in enumerate(toks):
            base = float(lx.get(w, 0.0))
            if base == 0.0:
                continue

            
            left = max(0, i - window)
            right = min(len(toks), i + 1 + window)
            neigh = toks[left:i] + toks[i+1:right]

            
            if (w in negators) or any(n in negators for n in neigh):
                base = -base

            
            boost = 1.0
            for b in neigh:
                if b in boosters:
                    try:
                        boost *= float(boosters[b])
                    except Exception:
                        pass

            contrib = base * boost

            
            if contrib > 0:
                contrib = min(contrib, max_token_contrib)
            else:
                contrib = max(contrib, -max_token_contrib)

            total += contrib

        
        denom = (len(toks) + 1e-9)
        vals.append(total / denom)

    return pd.Series(vals)


def _weights_lookup(weights: Dict[str, float], token: str, default: float = 1.0) -> float:
    try:
        return float(weights.get(token, default))
    except Exception:
        return default

def _weighted_ttr(tokens: List[str], weights: Dict[str, float]) -> float:
    if not tokens:
        return float('nan')
    denom = sum(_weights_lookup(weights, t) for t in tokens) + 1e-12
    types = set(tokens)
    numer = sum(_weights_lookup(weights, t) for t in types)
    return numer / denom

def ttr_from_json(text: str, tokenizer, cfg: dict) -> float:
    method = cfg["ttr"].get("method","msttr").lower()
    win = int(cfg["ttr"].get("window", 10))
    min_tokens = int(cfg["ttr"].get("min_tokens", 5))
    weights = cfg["ttr"].get("weights", {})  
    toks = tokenizer(text)
    if len(toks) < min_tokens:
        return float('nan')
    if method == "msttr" and len(toks) >= win and win > 0:
        vals = []
        for i in range(0, len(toks) - win + 1):
            span = toks[i:i+win]
            vals.append(_weighted_ttr(span, weights))
        return float(np.mean(vals)) if vals else _weighted_ttr(toks, weights)
    else:
        return _weighted_ttr(toks, weights)

def mtld_from_json(text: str, tokenizer, cfg: dict) -> float:
    thr = float(cfg["mtld"].get("threshold", 0.72))
    min_tokens = int(cfg["mtld"].get("min_tokens", 30))
    fb = cfg["mtld"].get("short_text_fallback", "cttr").lower()
    k = float(cfg["mtld"].get("cttr_k", 2.0))
    weights = cfg["mtld"].get("weights", {})
    toks = tokenizer(text)
    n_tokens = len(toks)
    if n_tokens < min_tokens:
        if fb == "cttr":
            types = set(toks)
            types_w = sum(_weights_lookup(weights, t) for t in types)
            return types_w / math.sqrt(k * max(1, n_tokens))
        else:
            return float('nan')
    factors = 0.0
    types = set()
    types_w = 0.0
    seg_token_w = 0.0
    for i, tok in enumerate(toks):
        w = _weights_lookup(weights, tok)
        seg_token_w += w
        if tok not in types:
            types.add(tok)
            types_w += _weights_lookup(weights, tok)
        ratio = types_w / max(1e-9, seg_token_w)
        if ratio < thr:
            factors += 1.0
            types.clear()
            types_w = 0.0
            seg_token_w = 0.0
    if seg_token_w > 0.0:
        ratio = types_w / max(1e-9, seg_token_w)
        factors += (1.0 - (ratio / thr))
    if factors <= 0:
        return float('nan')
    return n_tokens / factors




SPEC_TO_AREA_DEFAULT = {
    "Gestione dell’apprendimento e lacune": "Deuteroapprendimento",
    "Pensiero riflessivo e valutazione": "Deuteroapprendimento",
    "Apprendimento dall’esperienza e opportunità": "Deuteroapprendimento",
    "Auto-consapevolezza e auto-efficacia": "Personale",
    "Auto-regolazione, resilienza e benessere": "Personale",
    "Motivazione e perseveranza": "Personale",
    "Flessibilità e adattabilità": "Personale",
    "Pensiero creativo e sistemico": "Personale",
    "Abilità comunicative": "Sociale",
    "Iniziativa e gestione dell’incertezza": "Sociale",
    "Pensiero critico e risoluzione dei problemi": "Sociale",
}

OPEN_QS = [
    {"qid":"Q1_benessere","title":"Elenca tre cose di cui hai bisogno per stare bene","excel_cols":["BE","BF","BG"],"competences":["Auto-regolazione, resilienza e benessere"]},
    {"qid":"Q2_aspirazioni","title":"Quali sono le tue aspirazioni o sogni per il futuro? Scrivine almeno due","excel_cols":["BH","BI"],"competences":["Motivazione e perseveranza","Pensiero creativo e sistemico"]},
    {"qid":"Q3_obiettivo6mesi","title":"Obiettivo personale (6 mesi) e come raggiungerlo","excel_cols":["BO"],"competences":["Gestione dell’apprendimento e lacune"]},
    {"qid":"Q4_punti_forza","title":"Identifica due punti di forza principali","excel_cols":["BP","BQ"],"competences":["Auto-consapevolezza e auto-efficacia"]},
    {"qid":"Q5_migliorare","title":"Indica due aree in cui vorresti migliorare","excel_cols":["BR","BS"],"competences":["Pensiero riflessivo e valutazione"]},
    {"qid":"Q6_supera_paura","title":"Racconta una volta in cui hai superato paura/insicurezza e come hai fatto","excel_cols":["BY"],"competences":["Flessibilità e adattabilità","Abilità comunicative"]},
    {"qid":"Q7_compiti_noiosi","title":"Cosa fai quando affronti compiti difficili/noiosi? Riesci a raggiungere l’obiettivo?","excel_cols":["BZ"],"competences":["Motivazione e perseveranza"]},
    {"qid":"Q8_stress","title":"Cosa fai per gestire lo stress?","excel_cols":["DU"],"competences":["Auto-regolazione, resilienza e benessere"]},
    {"qid":"Q9_cambiamento","title":"Descrivi come hai affrontato un cambiamento importante e cosa ti ha aiutato ad adattarti","excel_cols":["EP"],"competences":["Flessibilità e adattabilità"]},
    {"qid":"Q9b_cambiare_abitudini","title":"Sei disposto a cambiare abitudini per apprendere qualcosa di nuovo? Esempio","excel_cols":["FK"],"competences":["Apprendimento dall’esperienza e opportunità"]},
    {"qid":"Q10_iniziativa","title":"Hai preso l’iniziativa per migliorare qualcosa (famiglia/scuola/amici)? Cosa e risultato","excel_cols":["FQ"],"competences":["Iniziativa e gestione dell’incertezza","Pensiero critico e risoluzione dei problemi"]},
]




def build_tables_with_uid_and_group(path_csv: str, time_label: str) -> pd.DataFrame:
    df, school_map = load_questionnaire_lettered(path_csv)
    def first_school(row):
        for c in SCHOOL_COLS:
            if c not in row: 
                continue
            val = row[c]
            if isinstance(val, str) and val.strip() != "" and val.strip().lower() != "nan":
                return school_map.get(c, "SCONOSCIUTA")
        return "SCONOSCIUTA"
    df_data = df.iloc[2:].reset_index(drop=True).copy()
    school = df_data.apply(first_school, axis=1)
    AY_i = excel_col_to_index("AY")
    AW_i = excel_col_to_index("AW")
    AX_i = excel_col_to_index("AX")

    AY = (df_data.iloc[:, AY_i] if AY_i < df_data.shape[1] else pd.Series([np.nan]*len(df_data))).astype(str).str.strip()
    aw = (df_data.iloc[:, AW_i] if AW_i < df_data.shape[1] else pd.Series([np.nan]*len(df_data))).astype(str).str.strip().str.lower()
    ax = (df_data.iloc[:, AX_i] if AX_i < df_data.shape[1] else pd.Series([np.nan]*len(df_data))).astype(str).str.strip().str.lower()
    is_etw = aw.isin({"si","sì","sí"})
    is_ctrl = ax.eq("no")
    group = pd.Series(index=df_data.index, dtype="object")
    group[is_etw] = "etwinning"
    group[is_ctrl] = "controllo"
    uid = school + "_" + AY

    rows = []
    for q in OPEN_QS:
        cols = q["excel_cols"]
        idxs = [excel_col_to_index(c) for c in cols]
        usable = [df.columns[i] for i in idxs if 0 <= i < df.shape[1]]
        sub = df.iloc[2:][usable].reset_index(drop=True) if usable else pd.DataFrame()
        if sub.empty:
            continue
        texts = sub.apply(lambda r: "; ".join([x for x in r.astype(str).tolist() if str(x).strip() not in ("","nan")]), axis=1)
        for i, t in enumerate(texts.tolist()):
            t = (t or "").strip()
            if not t: continue
            for comp in q["competences"]:
                rows.append({
                    "UID": uid.iloc[i],
                    "group": group.iloc[i],
                    "qid": q["qid"],
                    "question": q["title"],
                    "competenza": comp,
                    "text": fix_mojibake_text(t),
                    "time": time_label
                })
    out = pd.DataFrame(rows)
    return out




def _save_top_bar(freq_dict: dict, path_png: str, title: str):
    
    items = sorted(freq_dict.items(), key=lambda x: x[1], reverse=True)[:30]
    if not items:
        plt.figure(figsize=(8,5)); plt.text(0.5,0.5,"Dati insufficienti", ha="center", va="center")
        plt.axis("off"); plt.savefig(path_png, dpi=140); plt.close(); return
    labels = [w for w,_ in items]; values = [v for _,v in items]
    plt.figure(figsize=(10,6))
    plt.barh(range(len(labels)), values)
    plt.yticks(range(len(labels)), labels)
    plt.gca().invert_yaxis()
    plt.title(title); plt.tight_layout()
    plt.savefig(path_png, dpi=140); plt.close()

def build_wordclouds(df: pd.DataFrame, tok, figs_dir: str):

    
    import os, re
    import matplotlib.pyplot as plt
    try:
        from wordcloud import WordCloud
    except Exception:
        WordCloud = None

    os.makedirs(figs_dir, exist_ok=True)

    
    
    ban_terms_g   = set(globals().get("ban_terms", set()))
    ban_phrases_g = set(globals().get("ban_phrases", set()))

    groups = [None, "etwinning", "controllo"]
    suffix = {None: "all", "etwinning": "etwinning", "controllo": "controllo"}

    def _wc_common(tokens, title, fname, width=1200, height=800):
        if not tokens:
            fig = plt.figure(figsize=(width/100, height/100), dpi=100)
            plt.axis("off"); plt.title(title)
            plt.savefig(os.path.join(figs_dir, fname), dpi=120); plt.close(fig)
            return 1
        text = " ".join(tokens)
        if WordCloud is None:
            
            from collections import Counter
            cnt = Counter(tokens)
            items = cnt.most_common(100)
            labels = [k for k, _ in items]
            vals   = [v for _, v in items]
            plt.figure(figsize=(12, 6)); plt.bar(range(len(vals)), vals)
            plt.xticks(range(len(vals)), labels, rotation=90, fontsize=8)
            plt.title(title); plt.tight_layout()
            plt.savefig(os.path.join(figs_dir, fname), dpi=160); plt.close()
            return 1
        wc = WordCloud(width=width, height=height, background_color="white",
                       prefer_horizontal=0.9, collocations=False, normalize_plurals=False)
        img = wc.generate(text)
        fig = plt.figure(figsize=(width/100, height/100), dpi=100)
        plt.imshow(img, interpolation="bilinear"); plt.axis("off"); plt.title(title)
        plt.tight_layout(pad=0)
        plt.savefig(os.path.join(figs_dir, fname), dpi=180); plt.close(fig)
        return 1

    def _clean_tokens(series):
        toks = []
        for s in series.astype(str):
            s_norm = re.sub(r"\s+", " ", s.strip().lower())
            
            if ban_phrases_g and s_norm in ban_phrases_g:
                continue
            toks.extend(tok(s_norm))
        
        if ban_terms_g:
            toks = [w for w in toks if w not in ban_terms_g]
        return toks

    n = 0
    
    for (area, when), sub in df.groupby(["area", "time"], dropna=False):
        for grp in groups:
            series = sub["text"] if grp is None else sub.loc[sub["group"] == grp, "text"]
            tokens = _clean_tokens(series)
            title  = f"Wordcloud – Area: {area} · {when} · {suffix[grp]}"
            fname  = f"wordcloud_{area}_{when}_{suffix[grp]}.png"
            n += _wc_common(tokens, title, fname)

    
    for area, sub in df.groupby("area", dropna=False):
        tokens_all = _clean_tokens(sub["text"])
        n += _wc_common(tokens_all, f"Wordcloud – Area: {area} · all", f"wordcloud_{area}_all.png")
        for grp in ["etwinning", "controllo"]:
            toks_g = _clean_tokens(sub.loc[sub["group"] == grp, "text"])
            n += _wc_common(toks_g, f"Wordcloud – Area: {area} · {grp}", f"wordcloud_{area}_{grp}.png")

    
    both_all = df["text"]
    n += _wc_common(_clean_tokens(both_all),
                    "Wordcloud – COMPETENZA (tutte le aree) · all",
                    "wordcloud_COMPETENZA_all.png")
    for grp in ["etwinning", "controllo"]:
        n += _wc_common(_clean_tokens(df.loc[df["group"] == grp, "text"]),
                        f"Wordcloud – COMPETENZA · {grp}",
                        f"wordcloud_COMPETENZA_{grp}.png")
    return n


def _safe_slug(s: str) -> str:
    return re.sub(r"[^A-Za-z0-9_.-]+", "_", str(s)).strip("_")

def plot_competence_prepost_bars(df, figs_dir,
                                 metrics=("sentiment_json","mtld_json")):
    import matplotlib.pyplot as plt
    os.makedirs(figs_dir, exist_ok=True)
    for m in metrics:
        mname = m.replace("_json", "") if isinstance(m, str) else str(m)
        if m not in df.columns: continue
        g = (df.groupby(["group","time"])[m]
               .mean().reset_index())
        if g.empty: continue
        groups = ["etwinning","controllo"]
        times  = ["pre","post"]
        vals = { (gr,tm): float(g[(g["group"]==gr)&(g["time"]==tm)][m].mean())
                 if not g[(g["group"]==gr)&(g["time"]==tm)].empty else float("nan")
                 for gr in groups for tm in times }
        x = np.arange(len(groups))
        width = 0.35
        plt.figure(figsize=(7,5))
        plt.bar(x - width/2, [vals[(gr,"pre")]  for gr in groups], width, label="pre")
        plt.bar(x + width/2, [vals[(gr,"post")] for gr in groups], width, label="post")
        plt.xticks(x, groups)
        plt.title(f"COMPETENZA – {mname} (media)")
        plt.legend()
        plt.tight_layout()
        plt.savefig(os.path.join(figs_dir, f"comp_pre_post_{_safe_slug(mname)}.png"), dpi=160)
        plt.close()

def plot_area_prepost_bars(df, figs_dir,
                           metrics=("sentiment_json","mtld_json")):
    import matplotlib.pyplot as plt
    os.makedirs(figs_dir, exist_ok=True)
    for m in metrics:
        mname = m.replace("_json", "") if isinstance(m, str) else str(m)
        if m not in df.columns: continue
        for area, sub in df.groupby("area"):
            g = (sub.groupby(["group","time"])[m]
                    .mean().reset_index())
            if g.empty: continue
            groups = ["etwinning","controllo"]
            times  = ["pre","post"]
            vals = { (gr,tm): float(g[(g["group"]==gr)&(g["time"]==tm)][m].mean())
                     if not g[(g["group"]==gr)&(g["time"]==tm)].empty else float("nan")
                     for gr in groups for tm in times }
            x = np.arange(len(groups))
            width = 0.35
            plt.figure(figsize=(7.5,5))
            plt.bar(x - width/2, [vals[(gr,"pre")]  for gr in groups], width, label="pre")
            plt.bar(x + width/2, [vals[(gr,"post")] for gr in groups], width, label="post")
            plt.xticks(x, groups)
            plt.title(f"Area: {area} – {mname} (media)")
            plt.legend()
            plt.tight_layout()
            fn = f"area_{_safe_slug(area)}_pre_post_{_safe_slug(mname)}.png"
            plt.savefig(os.path.join(figs_dir, fn), dpi=160)
            plt.close()

def plot_did_by_area(did_csv_path, figs_dir, metric="sentiment"):
    import matplotlib.pyplot as plt
    if not os.path.exists(did_csv_path): return
    d = pd.read_csv(did_csv_path)
    d = d[d["metric"].astype(str).str.contains(metric, case=False, na=False)]
    if d.empty: return
    d = d.sort_values("did", ascending=False)
    x = np.arange(d.shape[0])
    plt.figure(figsize=(max(7, d.shape[0]*0.6), 5))
    plt.bar(x, d["did"].astype(float).values)
    plt.xticks(x, d["area"].astype(str).tolist(), rotation=45, ha="right")
    plt.title(f"DID per area – {metric}")
    plt.tight_layout()
    plt.savefig(os.path.join(figs_dir, f"did_by_area_{_safe_slug(metric)}.png"), dpi=160)
    plt.close()

def plot_quadrants_competence_stacked(df, figs_dir):
    import matplotlib.pyplot as plt
    if "quadrant" not in df.columns: return
    agg = (df.groupby(["group","time","quadrant"], as_index=False)
             .size().rename(columns={"size":"n"}))
    if agg.empty: return
    agg["share"] = agg.groupby(["group","time"])["n"].transform(lambda x: (x/x.sum()).round(4))
    quadrants = sorted(agg["quadrant"].dropna().unique().tolist())
    groups    = ["etwinning","controllo"]
    times     = ["pre","post"]

    for gr in groups:
        dat = agg[agg["group"]==gr]
        pre  = dat[dat["time"]=="pre"].set_index("quadrant")["share"].reindex(quadrants).fillna(0)
        post = dat[dat["time"]=="post"].set_index("quadrant")["share"].reindex(quadrants).fillna(0)
        x = np.arange(2)
        plt.figure(figsize=(8,5))
        bottom = np.zeros(2)
        for q in quadrants:
            vals = np.array([pre.get(q,0.0), post.get(q,0.0)], dtype=float)
            plt.bar(x, vals, bottom=bottom, label=f"Q{q}")
            bottom += vals
        plt.xticks(x, ["pre","post"])
        plt.ylim(0,1)
        plt.title(f"COMPETENZA – Quote quadranti ({gr})")
        plt.legend(title="Quadrante", bbox_to_anchor=(1.02, 1), loc="upper left")
        plt.tight_layout()
        plt.savefig(os.path.join(figs_dir, f"quadrants_COMPETENZA_stacked_{_safe_slug(gr)}.png"), dpi=160)
        plt.close()

def save_result_plots(df, outd, figs_dir):
    os.makedirs(figs_dir, exist_ok=True)
    
    plot_competence_prepost_bars(df, figs_dir,
                                 metrics=tuple([c for c in ["sentiment_json","mtld_json"] if c in df.columns]))
    
    plot_area_prepost_bars(df, figs_dir,
                           metrics=tuple([c for c in ["sentiment_json","mtld_json"] if c in df.columns]))
    
    plot_did_by_area(os.path.join(outd, "did_area_metrics.csv"), figs_dir, metric="sentiment")
    plot_did_by_area(os.path.join(outd, "did_area_metrics.csv"), figs_dir, metric="mtld")
    
    plot_quadrants_competence_stacked(df, figs_dir)

def key_terms_and_collocations(df, tok, ban_terms, outd):
    made = {"key_terms":0, "colloc_bi":0, "colloc_tri":0}
    for area, sub in df.groupby("area"):
        for when in ["pre","post"]:
            docs_etw = sub[(sub["time"]==when) & (sub["group"]=="etwinning")]["text"].astype(str).tolist()
            docs_ctrl= sub[(sub["time"]==when) & (sub["group"]=="controllo")]["text"].astype(str).tolist()
            
            v = CountVectorizer(tokenizer=tok, min_df=1, token_pattern=None)
            groups = {"etwinning": docs_etw, "controllo": docs_ctrl}
            vocab_counts = {}; group_counts = {}
            for g, docs in groups.items():
                if not docs: continue
                X = v.fit_transform(docs)
                counts = np.asarray(X.sum(axis=0)).ravel()
                vocab = np.array(v.get_feature_names_out())
                dvc = pd.Series(counts, index=vocab)
                group_counts[g] = dvc
                vocab_counts.setdefault("_all", pd.Series(dtype=np.int64))
                vocab_counts["_all"] = vocab_counts["_all"].add(dvc, fill_value=0)
            cols = ["term","group","log_odds","count","area","time"]
            out_parts = []
            if group_counts:
                alpha0 = vocab_counts["_all"] + 1.0
                alpha0 = alpha0 / alpha0.sum()
                for g, dvc in group_counts.items():
                    other = sum((group_counts[k] for k in group_counts if k != g), pd.Series(dtype=np.int64))
                    other = other.reindex(alpha0.index).fillna(0)
                    dvc = dvc.reindex(alpha0.index).fillna(0)
                    alpha = alpha0 * 1000.0
                    num = dvc + alpha
                    den = other + alpha
                    log_odds = np.log((num / num.sum()) / (den / den.sum()))
                    dfk = pd.DataFrame({"term": alpha0.index, "group": g, "log_odds": log_odds.values, "count": dvc.values})
                    dfk = dfk[dfk["count"] >= 3].sort_values(["count","log_odds"], ascending=[False, False])
                    if not dfk.empty:
                        dfk["area"]=area; dfk["time"]=when
                        out_parts.append(dfk)
            if out_parts:
                out = pd.concat(out_parts, ignore_index=True)
            else:
                out = pd.DataFrame(columns=cols)
            path = os.path.join(outd, f"key_terms_{area}_{when}.csv")
            safe_write_csv(out, path, columns=cols)
            made["key_terms"] += 1

            
            docs = docs_etw + docs_ctrl
            tokens = []
            for dtx in docs:
                tokens.extend(tok(dtx))
            from collections import Counter
            uni = Counter(tokens)
            def colloc(n, min_freq):
                rows = []
                if n==2:
                    pairs = list(zip(tokens, tokens[1:]))
                    ngrams = Counter(pairs)
                else:
                    trip = list(zip(tokens, tokens[1:], tokens[2:]))
                    ngrams = Counter(trip)
                total = sum(uni.values())
                for ng, f in ngrams.items():
                    if f < min_freq: continue
                    if n==2:
                        a,b = ng
                        p = f / (total - 1 if total > 1 else 1)
                        p_a = uni[a]/total if total else 0
                        p_b = uni[b]/total if total else 0
                        denom = (p_a*p_b) if (p_a and p_b) else 1e-12
                        pmi = math.log2(p/denom + 1e-12)
                    else:
                        a,b,c = ng
                        p = f / (total - 2 if total > 2 else 1)
                        p_a = uni[a]/total if total else 0
                        p_b = uni[b]/total if total else 0
                        p_c = uni[c]/total if total else 0
                        denom = (p_a*p_b*p_c) if (p_a and p_b and p_c) else 1e-18
                        pmi = math.log2(p/denom + 1e-12)
                    rows.append({"ngram":" ".join(ng), "freq":f, "PMI":pmi})
                cols2=["ngram","freq","PMI"]
                return pd.DataFrame(rows).sort_values(["freq","PMI"], ascending=[False, False])[cols2]
            bi = colloc(2, 5)
            tri = colloc(3, 3)
            path_bi = os.path.join(outd, f"collocations_{area}_{when}_bi.csv")
            path_tri = os.path.join(outd, f"collocations_{area}_{when}_tri.csv")
            safe_write_csv(bi, path_bi, columns=["ngram","freq","PMI"])
            safe_write_csv(tri, path_tri, columns=["ngram","freq","PMI"])
            made["colloc_bi"] += 1; made["colloc_tri"] += 1
    return made

def topic_model_area(df_area: pd.DataFrame, tok, n_topics: int = None, max_features: int = 4000):
    docs = df_area["text"].astype(str).tolist()
    if len(docs) < 10:
        return None
    cv = CountVectorizer(tokenizer=tok, min_df=2, max_df=0.9, max_features=max_features, token_pattern=None)
    X = cv.fit_transform(docs)
    if X.shape[0] < 10 or X.shape[1] < 10:
        return None
    if n_topics is None:
        n_topics = min(max(4, X.shape[0] // 40), 10)
    lda = LatentDirichletAllocation(n_components=n_topics, learning_method="batch", random_state=42)
    W = lda.fit_transform(X)  
    H = lda.components_       
    vocab = np.array(cv.get_feature_names_out())
    return {"lda": lda, "W": W, "H": H, "vocab": vocab, "vectorizer": cv}

def export_topics(area: str, model: dict, df_area: pd.DataFrame, outd: str):
    lda, W, H, vocab = model["lda"], model["W"], model["H"], model["vocab"]
    rows = []
    for k in range(lda.n_components):
        comp = H[k] / (H[k].sum() + 1e-9)
        top_idx = np.argsort(comp)[::-1][:20]
        for rank, j in enumerate(top_idx, start=1):
            rows.append({"area":area, "topic":k, "rank":rank, "term": vocab[j], "weight": float(comp[j])})
    topics_terms = pd.DataFrame(rows)
    safe_write_csv(topics_terms, os.path.join(outd, f"topics_terms_area_{area}.csv"), columns=["area","topic","rank","term","weight"])
    df = df_area.reset_index(drop=True).copy()
    if "group" not in df.columns: df["group"] = None
    if "time" not in df.columns: df["time"] = None
    main_topic = np.argmax(W, axis=1)
    df["_topic"] = main_topic
    prev_rows = []
    for (g,t), sub in df.groupby(["group","time"], dropna=False):
        n = sub.shape[0]
        for k in range(lda.n_components):
            nk = int((sub["_topic"]==k).sum())
            prop = nk / n if n else 0.0
            prev_rows.append({"area":area,"group":g,"time":t,"topic":k,"n":nk,"tot":n,"prop":prop})
    topics_prev = pd.DataFrame(prev_rows)
    prev_path = os.path.join(outd, "topics_prevalence_area_group.csv")
    if os.path.exists(prev_path):
        old = pd.read_csv(prev_path)
        topics_prev = pd.concat([old, topics_prev], ignore_index=True).drop_duplicates()
    safe_write_csv(topics_prev, prev_path, columns=["area","group","time","topic","n","tot","prop"])
    return topics_prev


def export_top_responses_by_question(df_rows, cfg, out_csv_path, tokenizer=_simple_tokenize):
    
    
    qcfg = cfg.get("questions", {}) if isinstance(cfg, dict) else {}
    qid_col  = qcfg.get("question_id_col", "question_id")
    qtxt_col = qcfg.get("question_text_col", "question")
    resp_col = qcfg.get("response_text_col", "text")
    top_k    = int(qcfg.get("top_k", 20))
    do_norm  = bool(qcfg.get("normalize_responses", True))

    
    for col in (qid_col, qtxt_col, resp_col):
        if col not in df_rows.columns:
            
            import pandas as pd
            pd.DataFrame(columns=["question_id","question","response_norm","count","wordcount","rank"])              .to_csv(out_csv_path, index=False)
            return

    def _normalize_resp(s: str) -> str:
        s = str(s).strip()
        if not do_norm:
            return s
        
        s = " ".join(s.lower().split())
        
        return s

    rows = []
    
    questions_order = list(dict.fromkeys(df_rows[qid_col].tolist()))
    for qid in questions_order:
        sub = df_rows[df_rows[qid_col] == qid]
        qtext = sub[qtxt_col].iloc[0] if not sub.empty else ""
        counts = Counter(_normalize_resp(s) for s in sub[resp_col].astype(str).tolist() if str(s).strip() != "")
        if not counts:
            continue
        
        top = counts.most_common(top_k)
        rank = 1
        for resp, cnt in top:
            wc = len(tokenizer(resp))
            rows.append({
                "question_id": qid,
                "question": qtext,
                "response_norm": resp,
                "count": int(cnt),
                "wordcount": int(wc),
                "rank": rank
            })
            rank += 1

    import pandas as pd
    out = pd.DataFrame(rows, columns=["question_id","question","response_norm","count","wordcount","rank"])
    
    
    order_map = {qid:i for i, qid in enumerate(questions_order)}
    out["_qorder"] = out["question_id"].map(order_map)
    out = out.sort_values(by=["_qorder","rank"]).drop(columns=["_qorder"])
    out.to_csv(out_csv_path, index=False)


def export_area_group_pvalues(df_rows, out_csv_path, area_col="area", group_col="group", time_col="time",
                              metrics=("sentiment","mtld")):
    
    import pandas as pd
    rows = []
    d = df_rows.copy()
    d[time_col] = d[time_col].map(_norm_time_label)
    for (a,g), sub in d.groupby([area_col, group_col], dropna=False):
        for m in metrics:
            if m not in sub.columns: continue
            x_pre  = sub.loc[sub[time_col]=="pre",  m].astype(float)
            x_post = sub.loc[sub[time_col]=="post", m].astype(float)
            p = _welch_t_pvalue(x_pre, x_post)
            rows.append({"area":a, "group":g, "metric":m, "p_prepost":p})
    pd.DataFrame(rows).to_csv(out_csv_path, index=False)

def export_area_pvalues(df_rows, out_csv_path, area_col="area", time_col="time",
                        metrics=("sentiment","mtld")):
    
    import pandas as pd
    rows = []
    d = df_rows.copy()
    d[time_col] = d[time_col].map(_norm_time_label)
    for a, sub in d.groupby(area_col, dropna=False):
        for m in metrics:
            if m not in sub.columns: continue
            x_pre  = sub.loc[sub[time_col]=="pre",  m].astype(float)
            x_post = sub.loc[sub[time_col]=="post", m].astype(float)
            p = _welch_t_pvalue(x_pre, x_post)
            rows.append({"area":a, "metric":m, "p_prepost_allgroups":p})
    pd.DataFrame(rows).to_csv(out_csv_path, index=False)

    """
    DID a livello 'COMPETENZA' (tutte le aree insieme).
    Calcola per ciascuna metrica:
      - delta_etwinning (post-pre)
      - delta_controllo (post-pre)
      - did = delta_etwinning - delta_controllo
      - p_did via OLS y ~ treated + post + treated:post (SE HC3)
      - p_did_cluster_school / _class se colonne 'school' / 'class' sono disponibili
    """
    import pandas as pd
    avail_metrics = [m for m in metrics if m in df_rows.columns]
    if not avail_metrics:
        pd.DataFrame(columns=["metric","did","p_did","p_did_cluster_school","p_did_cluster_class",
                              "delta_etwinning","delta_controllo"]).to_csv(out_csv_path, index=False)
        return

    d = df_rows[[group_col, time_col] + avail_metrics].copy()
    d[time_col] = d[time_col].map(_norm_time_label)

    rows = []
    for m in avail_metrics:
        
        def _delta_for(is_treated: bool) -> float:
            kk = d[d[group_col].map(_is_treated) == int(is_treated)]
            pre  = kk.loc[kk[time_col]=="pre",  m].astype(float)
            post = kk.loc[kk[time_col]=="post", m].astype(float)
            if len(pre)==0 or len(post)==0:
                return float("nan")
            return float(post.mean() - pre.mean())

        de = _delta_for(True)
        dc = _delta_for(False)
        did = (de - dc) if (not math.isnan(de) and not math.isnan(dc)) else float("nan")

        
        coef, p = _did_ols_pvalue(d[[m, group_col, time_col]], m, group_col, time_col)

        
        p_s = float("nan")
        p_c = float("nan")
        if "school" in df_rows.columns:
            _, p_s = _did_ols_pvalue_cluster(df_rows[[m, group_col, time_col, "school"]].dropna(subset=[m, group_col, time_col]),
                                             m, group_col, time_col, cluster_col="school")
        if "class" in df_rows.columns:
            _, p_c = _did_ols_pvalue_cluster(df_rows[[m, group_col, time_col, "class"]].dropna(subset=[m, group_col, time_col]),
                                             m, group_col, time_col, cluster_col="class")

        rows.append({
            "metric": m,
            "did": did,
            "p_did": p,
            "p_did_cluster_school": p_s,
            "p_did_cluster_class": p_c,
            "delta_etwinning": de,
            "delta_controllo": dc
        })

    pd.DataFrame(rows).to_csv(out_csv_path, index=False)

    """
    Calcola, per Area e metrica, il DiD via regressione e il suo p-value HC3.
    Output colonne: area, metric, did, p_did, delta_etwinning, delta_controllo
    """
    import pandas as pd
    d = df_rows.copy()
    d[time_col] = d[time_col].map(_norm_time_label)
    rows = []
    for a, sub in d.groupby(area_col, dropna=False):
        for m in metrics:
            if m not in sub.columns: continue
            
            def _delta_for(is_treated):
                mask = sub[group_col].map(_is_treated) == int(is_treated)
                kk = sub[mask]
                pre  = kk.loc[kk[time_col]=="pre",  m].astype(float)
                post = kk.loc[kk[time_col]=="post", m].astype(float)
                if len(pre)==0 or len(post)==0:
                    return float("nan")
                return float(post.mean() - pre.mean())
            de = _delta_for(True)
            dc = _delta_for(False)
            did = (de - dc) if (not math.isnan(de) and not math.isnan(dc)) else float("nan")
            coef, p = _did_ols_pvalue(sub[[m, group_col, time_col]], m, group_col, time_col)
            coef_s, p_s = _did_ols_pvalue_cluster(sub, m, group_col, time_col, cluster_col='school')
            coef_c, p_c = _did_ols_pvalue_cluster(sub, m, group_col, time_col, cluster_col='class')
            rows.append({"area":a, "metric":m, "did":did, "p_did":p, "p_did_cluster_school":p_s, "p_did_cluster_class":p_c,
                         "delta_etwinning":de, "delta_controllo":dc})
    pd.DataFrame(rows).to_csv(out_csv_path, index=False)

def export_did_with_pvalues(
    df_rows,
    out_csv_path,
    area_col="area",
    group_col="group",
    time_col="time",
    metrics=("sentiment", "mtld")
):
    
    import pandas as pd

    d = df_rows.copy()
    d[time_col] = d[time_col].map(_norm_time_label)

    rows = []

    for a, sub in d.groupby(area_col, dropna=False):
        for m in metrics:
            if m not in sub.columns:
                continue

            
            def _delta_for(is_treated: bool) -> float:
                mask = sub[group_col].map(_is_treated) == int(is_treated)
                kk = sub[mask]
                pre = kk.loc[kk[time_col] == "pre", m].astype(float)
                post = kk.loc[kk[time_col] == "post", m].astype(float)
                if len(pre) == 0 or len(post) == 0:
                    return float("nan")
                return float(post.mean() - pre.mean())

            de = _delta_for(True)
            dc = _delta_for(False)

            if not (math.isnan(de) or math.isnan(dc)):
                did = float(de - dc)
            else:
                did = float("nan")

            
            _, p = _did_ols_pvalue(sub[[m, group_col, time_col]], m, group_col, time_col)

            
            p_s = float("nan")
            p_c = float("nan")

            if "school" in sub.columns:
                _, p_s = _did_ols_pvalue_cluster(
                    sub[[m, group_col, time_col, "school"]].dropna(subset=[m, group_col, time_col]),
                    m,
                    group_col,
                    time_col,
                    cluster_col="school",
                )

            if "class" in sub.columns:
                _, p_c = _did_ols_pvalue_cluster(
                    sub[[m, group_col, time_col, "class"]].dropna(subset=[m, group_col, time_col]),
                    m,
                    group_col,
                    time_col,
                    cluster_col="class",
                )

            rows.append(
                {
                    "area": a,
                    "metric": m,
                    "did": did,
                    "p_did": p,
                    "p_did_cluster_school": p_s,
                    "p_did_cluster_class": p_c,
                    "delta_etwinning": de,
                    "delta_controllo": dc,
                }
            )

    pd.DataFrame(rows).to_csv(out_csv_path, index=False)


def export_competence_did_with_pvalues(
    df_rows,
    out_csv_path,
    group_col="group",
    time_col="time",
    metrics=("sentiment_json", "ttr_json", "mtld_json", "arousal_json", "dominance_json"),
):
    
    import pandas as pd

    avail_metrics = [m for m in metrics if m in df_rows.columns]
    if not avail_metrics:
        pd.DataFrame(
            columns=[
                "metric",
                "did",
                "p_did",
                "p_did_cluster_school",
                "p_did_cluster_class",
                "delta_etwinning",
                "delta_controllo",
            ]
        ).to_csv(out_csv_path, index=False)
        return

    d = df_rows[[group_col, time_col] + avail_metrics].copy()
    d[time_col] = d[time_col].map(_norm_time_label)

    rows = []

    for m in avail_metrics:

        def _delta_for(is_treated: bool) -> float:
            kk = d[d[group_col].map(_is_treated) == int(is_treated)]
            pre = kk.loc[kk[time_col] == "pre", m].astype(float)
            post = kk.loc[kk[time_col] == "post", m].astype(float)
            if len(pre) == 0 or len(post) == 0:
                return float("nan")
            return float(post.mean() - pre.mean())

        de = _delta_for(True)
        dc = _delta_for(False)

        if not (math.isnan(de) or math.isnan(dc)):
            did = float(de - dc)
        else:
            did = float("nan")

        
        _, p = _did_ols_pvalue(d[[m, group_col, time_col]], m, group_col, time_col)

        
        p_s = float("nan")
        p_c = float("nan")

        if "school" in df_rows.columns:
            _, p_s = _did_ols_pvalue_cluster(
                df_rows[[m, group_col, time_col, "school"]].dropna(subset=[m, group_col, time_col]),
                m,
                group_col,
                time_col,
                cluster_col="school",
            )

        if "class" in df_rows.columns:
            _, p_c = _did_ols_pvalue_cluster(
                df_rows[[m, group_col, time_col, "class"]].dropna(subset=[m, group_col, time_col]),
                m,
                group_col,
                time_col,
                cluster_col="class",
            )

        rows.append(
            {
                "metric": m,
                "did": did,
                "p_did": p,
                "p_did_cluster_school": p_s,
                "p_did_cluster_class": p_c,
                "delta_etwinning": de,
                "delta_controllo": dc,
            }
        )

    pd.DataFrame(rows).to_csv(out_csv_path, index=False)



def export_quadrant_table(df_rows, out_csv_path, area_col="area", group_col="group", time_col="time", quad_col="quadrant"):
    
    import pandas as pd
    if quad_col not in df_rows.columns:
        pd.DataFrame(columns=[area_col,group_col,time_col]).to_csv(out_csv_path, index=False)
        return
    d = df_rows.copy()
    d[time_col] = d[time_col].map(_norm_time_label)
    counts = d.groupby([area_col,group_col,time_col,quad_col]).size().reset_index(name="n")
    totals = counts.groupby([area_col,group_col,time_col])["n"].transform("sum")
    counts["pct"] = 100.0 * counts["n"] / totals
    
    wide = counts.pivot_table(index=[area_col,group_col,time_col], columns=quad_col, values="pct", fill_value=0.0)
    wide = wide.reset_index()
    wide.columns.name = None
    wide.to_csv(out_csv_path, index=False)





TOPIC_LABEL_OVERRIDES = {
    
    "Personale": {
        8: "Family, friends & everyday wellbeing",
        2: "Aspirations & future self",
        4: "Effort, goals & demanding tasks",
        1: "Positive change & resilience",
        0: "General wellbeing & school-life balance",
        3: "Change through helping / family context",
        5: "Fear & speaking / self-confidence",
        6: "Money / play / sports (more concrete daily life)",
        9: "Listening, calm & relational skills",
        7: "Determination & ‘working hard’ frame",
    },

    
    "Sociale": {
        0: "Group initiative & shared goals",
        6: "Public speaking anxiety",
        8: "Fear → overcoming through speaking",
        2: "Trying new things / sport / social courage",
        7: "Taking initiative in interaction",
        4: "Peer conflict / peace-making / negotiating",
        5: "Courage in situations",
        9: "Everyday talk with friends/family",
        3: "School-year routine / fatigue frame",
        1: "General ‘facing fears’ narrative",
    },

    
    "Deuteroapprendimento": {
        1: "Study method + self-esteem / physical state",
        4: "School–sport–family balance / motivation",
        3: "Self-improvement through sport/school routine",
        2: "Debts/grades & ‘I must improve’",
        9: "Math/grades/year performance",
        7: "Habit change / learning to learn",
        0: "New habits / language learning / examples",
        8: "Shyness/obligation → trying to succeed",
        5: "Consistency & staying well",
        6: "Anxiety/relationships & managing school-life",
    },
}



def _build_topic_legend_from_terms(outd: str, top_n_terms: int = 8) -> pd.DataFrame:
    
    term_files = sorted(pathlib.Path(outd).glob("topics_terms_area_*.csv"))
    legend_rows = []
    for fp in term_files:
        try:
            tdf = pd.read_csv(fp, dtype={"area": str, "topic": int, "rank": int, "term": str})
        except Exception:
            continue
        required = {"area", "topic", "rank", "term"}
        if not required.issubset(set(tdf.columns)):
            continue

        tdf = tdf.sort_values(["area", "topic", "rank"], kind="mergesort")
        
        top = tdf[tdf["rank"].between(1, top_n_terms, inclusive="both")].copy()

        def _uniq_terms(s: pd.Series) -> str:
            seen = set()
            out = []
            for x in s.tolist():
                if x is None:
                    continue
                tx = str(x).strip()
                if not tx or tx.lower() == "nan":
                    continue
                if tx in seen:
                    continue
                seen.add(tx)
                out.append(tx)
            return " / ".join(out)

        g = (
            top.groupby(["area", "topic"], dropna=False)["term"]
            .apply(_uniq_terms)
            .reset_index(name="top_terms")
        )
        g["topic_label"] = g["top_terms"]
        
        try:
            overrides = TOPIC_LABEL_OVERRIDES.get(str(g.iloc[0]["area"]).strip(), {}) if ("area" in g.columns and not g.empty) else {}
        except Exception:
            overrides = {}
        if overrides:
            def _ovr_label(row):
                try:
                    t = int(row["topic"])
                except Exception:
                    return row["topic_label"]
                return overrides.get(t, row["topic_label"])
            g["topic_label"] = g.apply(_ovr_label, axis=1)

        g["topic_name"] = g.apply(lambda r: f"T{int(r['topic'])}: {r['topic_label']}", axis=1)
        legend_rows.append(g[["area", "topic", "topic_label", "topic_name", "top_terms"]])

    if legend_rows:
        legend = (
            pd.concat(legend_rows, ignore_index=True)
            .drop_duplicates(subset=["area", "topic"], keep="first")
            .sort_values(["area", "topic"], kind="mergesort")
            .reset_index(drop=True)
        )
    else:
        legend = pd.DataFrame(columns=["area", "topic", "topic_label", "topic_name", "top_terms"])

    
    safe_write_csv(
        legend[["area", "topic", "topic_name", "top_terms"]],
        os.path.join(outd, "topics_legend_area.csv"),
        columns=["area", "topic", "topic_name", "top_terms"],
    )
    return legend


def _attach_topic_labels(df: pd.DataFrame, legend: pd.DataFrame) -> pd.DataFrame:
    
    if df is None or df.empty:
        return df
    if legend is None or legend.empty:
        return df
    if "area" not in df.columns or "topic" not in df.columns:
        return df
    
    try:
        df2 = df.copy()
        df2["topic"] = pd.to_numeric(df2["topic"], errors="coerce").astype("Int64")
        leg = legend.copy()
        leg["topic"] = pd.to_numeric(leg["topic"], errors="coerce").astype("Int64")
        merged = df2.merge(
            leg[["area", "topic", "topic_label", "topic_name"]],
            on=["area", "topic"],
            how="left",
        )
        
        if "topic" in merged.columns:
            merged["topic"] = merged["topic"].astype("Int64")
        return merged
    except Exception:
        return df


def _reorder_topic_label_cols(df: pd.DataFrame) -> pd.DataFrame:
    
    if df is None or df.empty:
        return df
    cols = list(df.columns)
    if "topic" not in cols:
        return df
    for c in ["topic_name", "topic_label"]:
        if c not in cols:
            continue
        cols.remove(c)
    
    tpos = cols.index("topic") + 1
    new_cols = cols[:tpos] + [c for c in ["topic_name", "topic_label"] if c in df.columns] + cols[tpos:]
    
    new_cols = [c for c in new_cols if c in df.columns]
    return df[new_cols]


def _rewrite_all_topic_exports_with_labels(outd: str, legend: pd.DataFrame, skip: Optional[set] = None):
    
    skip = skip or set()
    for fp in sorted(pathlib.Path(outd).glob("*.csv")):
        if fp.name in skip:
            continue
        try:
            df = pd.read_csv(fp)
        except Exception:
            continue
        if ("topic" in df.columns) and ("area" in df.columns):
            df2 = _attach_topic_labels(df, legend)
            df2 = _reorder_topic_label_cols(df2)
            safe_write_csv(df2, str(fp))


def _export_themes_summary(shifts: pd.DataFrame, outd: str, top_k: int = 5):
    
    if shifts is None or shifts.empty:
        safe_write_csv(
            pd.DataFrame(columns=["area","group","direction","rank","topic","topic_name","pre_prop","post_prop","delta_prop"]),
            os.path.join(outd, "themes_summary_area_group.csv"),
            columns=["area","group","direction","rank","topic","topic_name","pre_prop","post_prop","delta_prop"],
        )
        return

    df = shifts.copy()
    
    df["delta_prop"] = pd.to_numeric(df.get("delta_prop"), errors="coerce")
    df = df.dropna(subset=["delta_prop"])

    out_rows = []
    for (area, group), sub in df.groupby(["area", "group"], dropna=False):
        sub = sub.sort_values("delta_prop", kind="mergesort")
        
        dec = sub.head(top_k).copy()
        for i, row in enumerate(dec.itertuples(index=False), start=1):
            out_rows.append({
                "area": area,
                "group": group,
                "direction": "decrease",
                "rank": i,
                "topic": int(row.topic) if pd.notna(row.topic) else row.topic,
                "topic_name": getattr(row, "topic_name", None),
                "pre_prop": getattr(row, "pre_prop", None),
                "post_prop": getattr(row, "post_prop", None),
                "delta_prop": getattr(row, "delta_prop", None),
            })
        
        inc = sub.sort_values("delta_prop", ascending=False, kind="mergesort").head(top_k).copy()
        for i, row in enumerate(inc.itertuples(index=False), start=1):
            out_rows.append({
                "area": area,
                "group": group,
                "direction": "increase",
                "rank": i,
                "topic": int(row.topic) if pd.notna(row.topic) else row.topic,
                "topic_name": getattr(row, "topic_name", None),
                "pre_prop": getattr(row, "pre_prop", None),
                "post_prop": getattr(row, "post_prop", None),
                "delta_prop": getattr(row, "delta_prop", None),
            })

    out_df = pd.DataFrame(out_rows)
    safe_write_csv(
        out_df,
        os.path.join(outd, "themes_summary_area_group.csv"),
        columns=["area","group","direction","rank","topic","topic_name","pre_prop","post_prop","delta_prop"],
    )


def export_topic_shifts(outd: str, top_n_terms: int = 8, top_k_themes: int = 5):
    p = os.path.join(outd, "topics_prevalence_area_group.csv")
    if not os.path.exists(p):
        return
    prev = pd.read_csv(p, dtype={"area":str,"group":str,"time":str,"topic":int,"n":int,"tot":int,"prop":float})
    rows = []
    for (area, group, topic), sub in prev.groupby(["area","group","topic"]):
        pre = float(sub.loc[sub["time"]=="pre", "prop"].sum()) if (sub["time"]=="pre").any() else float("nan")
        post= float(sub.loc[sub["time"]=="post","prop"].sum()) if (sub["time"]=="post").any() else float("nan")
        d = post - pre if (not math.isnan(pre) and not math.isnan(post)) else float("nan")
        rows.append({"area":area,"group":group,"topic":topic,"pre_prop":pre,"post_prop":post,"delta_prop":d})
    shifts = pd.DataFrame(rows)

    
    legend = _build_topic_legend_from_terms(outd, top_n_terms=top_n_terms)

    
    shifts = _attach_topic_labels(shifts, legend)
    shifts = _reorder_topic_label_cols(shifts)

    safe_write_csv(
        shifts,
        os.path.join(outd, "topics_shift_area_group.csv"),
        columns=["area","group","topic","topic_name","topic_label","pre_prop","post_prop","delta_prop"],
    )

    
    _rewrite_all_topic_exports_with_labels(
        outd,
        legend,
        skip={"topics_shift_area_group.csv", "topics_legend_area.csv", "themes_summary_area_group.csv"},
    )

    
    _export_themes_summary(shifts, outd, top_k=top_k_themes)


def plot_topic_heatmaps(outd: str, graphs_dir: str):
    
    import os
    import numpy as np
    import pandas as pd
    import matplotlib.pyplot as plt

    os.makedirs(graphs_dir, exist_ok=True)
    shift_path = os.path.join(outd, 'topics_shift_area_group.csv')
    if not os.path.exists(shift_path):
        return 0

    df = pd.read_csv(shift_path)
    if df.empty or 'area' not in df.columns or 'topic' not in df.columns:
        return 0

    
    df['topic'] = pd.to_numeric(df['topic'], errors='coerce')
    df = df.dropna(subset=['topic'])
    df['topic'] = df['topic'].astype(int)

    if 'topic_name' not in df.columns:
        df['topic_name'] = df['topic'].apply(lambda t: f'T{t}')

    
    def _group_key(g):
        gs = str(g).strip().lower()
        if 'twin' in gs or 'sper' in gs or 'treat' in gs:
            return (0, gs)
        if 'cont' in gs or 'ctrl' in gs or 'control' in gs:
            return (1, gs)
        return (2, gs)

    nfiles = 0
    for area, sub in df.groupby('area'):
        sub = sub.copy()

        
        topic_order = sorted(sub['topic'].unique().tolist())

        
        name_map = (
            sub[['topic','topic_name']]
            .dropna()
            .drop_duplicates('topic')
            .set_index('topic')['topic_name']
            .to_dict()
        )
        ylabels = [name_map.get(t, f'T{t}') for t in topic_order]

        groups = sorted(sub['group'].dropna().unique().tolist(), key=_group_key)
        if not groups:
            continue

        def _pivot(metric_col: str):
            pv = sub.pivot_table(index='topic', columns='group', values=metric_col, aggfunc='mean')
            pv = pv.reindex(index=topic_order, columns=groups)
            return pv

        for metric_col, tag, title in [
            ('pre_prop', 'pre', 'Topic prevalence (PRE)'),
            ('post_prop','post','Topic prevalence (POST)'),
            ('delta_prop','delta','Topic prevalence Δ (POST − PRE)'),
        ]:
            if metric_col not in sub.columns:
                continue
            mat = _pivot(metric_col)
            if mat.empty:
                continue

            data = mat.values.astype(float)

            
            h = max(4.5, 0.45 * len(topic_order))
            w = max(6.0, 1.5 + 1.3 * len(groups))
            plt.figure(figsize=(w, h))
            im = plt.imshow(data, aspect='auto')
            plt.colorbar(im, fraction=0.046, pad=0.04)
            plt.title(f"{area} — {title}")
            plt.xticks(range(len(groups)), groups, rotation=25, ha='right')
            plt.yticks(range(len(topic_order)), ylabels)

            
            for i in range(data.shape[0]):
                for j in range(data.shape[1]):
                    v = data[i, j]
                    if np.isnan(v):
                        txt = 'NA'
                    else:
                        txt = f"{v:.2f}"
                    plt.text(j, i, txt, ha='center', va='center', fontsize=8)

            plt.tight_layout()
            fn = os.path.join(graphs_dir, f"heatmap_{tag}_{_safe_slug(str(area))}.png")
            plt.savefig(fn, dpi=200)
            plt.close()
            nfiles += 1

    return nfiles

def exemplar_quotes(df, tok, outd, k_per_bucket: int = 6):
    made = 0
    for (area, when, grp), sub in df.groupby(["area","time","group"], dropna=False):
        docs = sub["text"].astype(str).tolist()
        if len(docs) < max(8, k_per_bucket):
            path = os.path.join(outd, f"exemplars_{area}_{when}_{grp or 'NA'}.csv")
            safe_write_csv(pd.DataFrame(columns=["rank","uid","qid","text","score","cluster"]), path, columns=["rank","uid","qid","text","score","cluster"])
            continue
        tfv = TfidfVectorizer(tokenizer=tok, min_df=2, max_df=0.95, token_pattern=None)
        X = tfv.fit_transform(docs)
        n_clusters = min(6, max(2, X.shape[0]//20))
        km = KMeans(n_clusters=n_clusters, n_init=10, random_state=42)
        labels = km.fit_predict(X)
        import numpy as _np
        exemplars = []
        for c in range(n_clusters):
            doc_idx = _np.where(labels==c)[0]
            if len(doc_idx)==0: continue
            centroid = X[doc_idx].mean(axis=0)
            from numpy.linalg import norm
            d = []
            for di in doc_idx:
                v = X[di]
                dist = norm((v - centroid).A)
                d.append((di, float(dist)))
            d.sort(key=lambda x: x[1])
            take = d[:max(1, k_per_bucket//n_clusters)]
            for di, dist in take:
                exemplars.append((di, dist, c))
        exemplars.sort(key=lambda x: x[1])
        rows = []
        for rank, (di, dist, c) in enumerate(exemplars[:k_per_bucket], start=1):
            rows.append({
                "rank": rank,
                "uid": sub.iloc[di]["UID"],
                "qid": sub.iloc[di]["qid"],
                "text": sub.iloc[di]["text"],
                "score": dist,
                "cluster": c
            })
        dfex = pd.DataFrame(rows, columns=["rank","uid","qid","text","score","cluster"])
        path = os.path.join(outd, f"exemplars_{area}_{when}_{grp or 'NA'}.csv")
        safe_write_csv(dfex, path, columns=["rank","uid","qid","text","score","cluster"])
        html_path = os.path.join(outd, f"exemplars_{area}_{when}_{grp or 'NA'}.html")
        buf = io.StringIO()
        buf.write("<!DOCTYPE html><meta charset='utf-8'><title>Exemplar quotes</title>")
        buf.write(f"<h1>Exemplar – Area: {html.escape(str(area))} · {html.escape(str(when))} · Gruppo: {html.escape(str(grp))}</h1>")
        for _,r in dfex.iterrows():
            buf.write("<div style='border-left:4px solid #ddd;padding:8px 12px;margin:10px 0'>")
            buf.write(f"<div style='color:#666'><b>UID:</b> {html.escape(str(r['uid']))} · <b>QID:</b> {html.escape(str(r['qid']))} · <b>Cluster:</b> {int(r['cluster'])} · <b>Score:</b> {r['score']:.3f}</div>")
            buf.write(f"<div>{html.escape(str(r['text']))}</div></div>")
        write_text(html_path, buf.getvalue()); made += 1
    return made



def export_lexicon_contributions(df_rows: pd.DataFrame, cfg: dict, out_dir: str, tok_for_lex=_simple_tokenize):
    
    sent_cfg = (cfg or {}).get("sentiment", {}) or {}
    lex = {str(k).lower(): float(v) for k, v in (sent_cfg.get("lexicon", {}) or {}).items()}
    boosters = {str(k).lower(): float(v) for k, v in (sent_cfg.get("boosters", {}) or {}).items()}
    negators = set(str(w).lower() for w in (sent_cfg.get("negators", []) or []))
    try:
        aro_lex = {str(k).lower(): float(v) for k, v in (sent_cfg.get("arousal_lexicon", {}) or {}).items()}
        dom_lex = {str(k).lower(): float(v) for k, v in (sent_cfg.get("dominance_lexicon", {}) or {}).items()}
    except Exception:
        aro_lex, dom_lex = {}, {}
    from collections import defaultdict
    contrib_sent = defaultdict(float); contrib_arou = defaultdict(float); contrib_dom  = defaultdict(float)
    for s in df_rows["text"].astype(str):
        toks = tok_for_lex(s); n = len(toks)
        for i, w in enumerate(toks):
            L = max(0, i-2); R = min(n, i+3); ctx = toks[L:i] + toks[i+1:R]
            if w in lex:
                sign = -1.0 if any(b in negators for b in ctx) else 1.0
                boost = 1.0
                for b in ctx:
                    if b in boosters:
                        try: boost *= float(boosters[b])
                        except Exception: pass
                contrib_sent[w] += sign * lex[w] * boost
            if w in aro_lex or w in dom_lex:
                boost2 = 1.0
                for b in ctx:
                    if b in boosters:
                        try: boost2 *= float(boosters[b])
                        except Exception: pass
                if w in aro_lex: contrib_arou[w] += aro_lex[w] * boost2
                if w in dom_lex: contrib_dom[w]  += dom_lex[w]  * boost2
    import pandas as pd
    def _top(dfm: pd.DataFrame, k=200):
        if dfm.empty: return dfm
        dfm["_abs"] = dfm["contribution"].abs()
        return dfm.sort_values("_abs", ascending=False).drop(columns=["_abs"]).head(k)
    s_df = pd.DataFrame([{"token":k, "contribution":v} for k,v in contrib_sent.items()])
    a_df = pd.DataFrame([{"token":k, "contribution":v} for k,v in contrib_arou.items()])
    d_df = pd.DataFrame([{"token":k, "contribution":v} for k,v in contrib_dom.items()])
    _od = out_dir
    try: os.makedirs(_od, exist_ok=True)
    except Exception: pass
    _top(s_df).to_csv(os.path.join(_od, "lexicon_contributions_sentiment.csv"), index=False, encoding="utf-8")
    _top(a_df).to_csv(os.path.join(_od, "lexicon_contributions_arousal.csv"), index=False, encoding="utf-8")
    _top(d_df).to_csv(os.path.join(_od, "lexicon_contributions_dominance.csv"), index=False, encoding="utf-8")



def export_area_descriptors(df_rows: pd.DataFrame, tok, out_csv_path: str, top_k: int = 8):
    from collections import Counter
    rows = []
    for area, sub in df_rows.groupby("area"):
        toks = []
        for s in sub["text"].astype(str):
            toks.extend(tok(s))
        cnt = Counter(toks)
        top_terms = [w for w,_ in cnt.most_common(top_k)]
        rows.append({"area": area, "descriptor_terms": "; ".join(top_terms)})
    import pandas as pd, os
    os.makedirs(os.path.dirname(out_csv_path), exist_ok=True) if os.path.dirname(out_csv_path) else None
    pd.DataFrame(rows, columns=["area","descriptor_terms"]).to_csv(out_csv_path, index=False, encoding="utf-8")




class App(tk.Tk):
    def __init__(self):
        super().__init__()
        self.title("eTwinning – Analisi qualitativa (v1.10.3 – robust wordclouds)")
        self.geometry("1180x860")
        self._build()

    def _build(self):
        f = ttk.Frame(self); f.pack(fill=tk.BOTH, expand=True, padx=12, pady=12)
        r = 0
        ttk.Label(f, text="Questionario ENTRATA (CSV)").grid(row=r, column=0, sticky="w")
        self.e_pre = ttk.Entry(f, width=92); self.e_pre.grid(row=r, column=1, sticky="we", padx=8)
        ttk.Button(f, text="Scegli", command=lambda: self._pick(self.e_pre)).grid(row=r, column=2); r+=1

        ttk.Label(f, text="Questionario USCITA (CSV)").grid(row=r, column=0, sticky="w")
        self.e_post = ttk.Entry(f, width=92); self.e_post.grid(row=r, column=1, sticky="we", padx=8)
        ttk.Button(f, text="Scegli", command=lambda: self._pick(self.e_post)).grid(row=r, column=2); r+=1

        ttk.Label(f, text="BAN list file (.txt) – parole da escludere (obbligatorio)").grid(row=r, column=0, sticky="w")
        self.e_ban = ttk.Entry(f, width=92); self.e_ban.grid(row=r, column=1, sticky="we", padx=8)
        ttk.Button(f, text="Scegli", command=lambda: self._pick(self.e_ban, "*.txt")).grid(row=r, column=2); r+=1

        ttk.Label(f, text="Config JSON (metric_weights) – OBBLIGATORIO").grid(row=r, column=0, sticky="w")
        self.e_json = ttk.Entry(f, width=92); self.e_json.grid(row=r, column=1, sticky="we", padx=8)
        ttk.Button(f, text="Scegli", command=lambda: self._pick(self.e_json, "*.json")).grid(row=r, column=2); r+=1

        ttk.Label(f, text="Cartella OUTPUT (se vuota → reports/<timestamp>)").grid(row=r, column=0, sticky="w")
        self.e_out = ttk.Entry(f, width=92); self.e_out.grid(row=r, column=1, sticky="we", padx=8)
        ttk.Button(f, text="Scegli cartella", command=self._pickdir).grid(row=r, column=2); r+=1

        self.log = tk.Text(f, height=24, wrap="word", bg="#101010", fg="#d6d6d6")
        self.log.grid(row=r, column=0, columnspan=3, sticky="nsew", pady=(10,6)); r+=1
        f.rowconfigure(r-1, weight=1); f.columnconfigure(1, weight=1)

        ttk.Button(f, text="Esegui analisi", command=self.run).grid(row=r, column=0, sticky="w", pady=6)

    def _pick(self, entry, pattern="*"):
        p = filedialog.askopenfilename(title="Seleziona file", filetypes=[("File", pattern), ("Tutti i file","*.*")])
        if p:
            entry.delete(0, tk.END); entry.insert(0, p)

    def _pickdir(self):
        d = filedialog.askdirectory(title="Cartella output")
        if d:
            self.e_out.delete(0, tk.END); self.e_out.insert(0, d)

    def _log(self, msg):
        self.log.insert(tk.END, msg+"\n"); self.log.see(tk.END); self.update_idletasks()

    def run(self):
        pre = self.e_pre.get().strip()
        post= self.e_post.get().strip()
        banf= self.e_ban.get().strip()
        jsnf= self.e_json.get().strip()
        outd= self.e_out.get().strip()

        if not pre or not post:
            messagebox.showwarning("Attenzione", "Seleziona i CSV di Entrata e Uscita."); return
        if not banf:
            messagebox.showwarning("BAN list mancante", "Seleziona un file .txt con le parole da escludere."); return
        if not jsnf:
            messagebox.showwarning("Config JSON mancante", "Seleziona il file JSON delle metriche."); return

        
        if outd:
            ensure_dir(outd)
        else:
            outd = os.path.join(os.getcwd(), "reports", datetime.now().strftime("%Y%m%d_%H%M%S"))
            ensure_dir(outd)
        graphs_dir = os.path.join(outd, "graphs"); ensure_dir(graphs_dir)
        figs_legacy_dir = os.path.join(outd, "figs"); ensure_dir(figs_legacy_dir)
        figs_dir = graphs_dir
        val_dir  = os.path.join(outd, "validation"); ensure_dir(val_dir)
        diag_dir = os.path.join(outd, "diagnostics"); ensure_dir(diag_dir)
        self._log(f"Output: {outd}")

        runlog = os.path.join(outd, "run.log")
        def logline(s):
            self._log(s)
            with open(runlog, "a", encoding="utf-8") as f:
                f.write(s+"\n")
        
        try:
            with open(jsnf, "r", encoding="utf-8") as f:
                cfg = json.load(f)
            logline(f"[OK] Config caricato: {jsnf}")
        except Exception as e:
            messagebox.showerror("Config JSON non valido", f"Errore nel leggere {jsnf}:\n{e}")
            return

        
        pair = (cfg.get("pairing") or {})
        code_col_letter = str((pair.get("code_col") or "AY")).strip().upper()

        
        df_pre_full,  school_map_pre  = load_questionnaire_lettered(pre)
        df_post_full, school_map_post = load_questionnaire_lettered(post)

        
        def _first_nonempty_in_range(row: pd.Series, start_col: str, end_col: str):
            c0 = _excel_col_to_idx(str(start_col).strip().upper())
            c1 = _excel_col_to_idx(str(end_col).strip().upper())
            cols = list(row.index)
            subcols = [c for i, c in enumerate(cols) if c0 <= i <= c1]
            hits = []
            for c in subcols:
                v = str(row.get(c, "")).strip()
                if v and v.lower() != "nan":
                    hits.append(c)
            if len(hits) == 1:
                return (hits[0], np.nan)
            elif len(hits) == 0:
                return (np.nan, f"NONE_SELECTED:{start_col}-{end_col}")
            else:
                return (np.nan, f"MULTI_SELECTED:{start_col}-{end_col}:{'|'.join(hits)}")

        
        def _normalize_identity_lettered(df_lettered: pd.DataFrame, school_map: dict, time_label: str) -> pd.DataFrame:
            
            if len(df_lettered) < 2:
                return pd.DataFrame(columns=["school","class","section","group","student_id","time","code","_warn_identity"])
            
            desc = df_lettered.iloc[1]

            def _label_map(start_col: str, end_col: str) -> dict:
                c0 = _excel_col_to_idx(start_col); c1 = _excel_col_to_idx(end_col)
                cols = list(df_lettered.columns)
                return {c: desc.get(c, c) for i, c in enumerate(cols) if c0 <= i <= c1}

            
            class_map   = _label_map("Y",  "AA")
            section_map = _label_map("AB", "AV")

            
            df = df_lettered.iloc[2:].reset_index(drop=True).copy()

            
            pairs_s = [_first_nonempty_in_range(row, "J",  "X")  for _, row in df.iterrows()]
            sc,  warn_s  = zip(*pairs_s)  if pairs_s else ([], [])

            pairs_c = [_first_nonempty_in_range(row, "Y",  "AA") for _, row in df.iterrows()]
            cl,  warn_c  = zip(*pairs_c)  if pairs_c else ([], [])

            pairs_se = [_first_nonempty_in_range(row, "AB", "AV") for _, row in df.iterrows()]
            se,  warn_se = zip(*pairs_se) if pairs_se else ([], [])
            df["class_col"]   = pd.Series(cl, index=df.index)
            df["section_col"] = pd.Series(se, index=df.index)
            df["class"]   = df["class_col"].map(  lambda c: class_map.get(c,   np.nan))
            df["section"] = df["section_col"].map(lambda c: section_map.get(c, np.nan))

            
            true_vals  = set(map(str.lower, (pair.get("etw", {}) or {}).get("true_values",  ["si","sì","yes","true","1","x","✓"])))
            false_vals = set(map(str.lower, (pair.get("etw", {}) or {}).get("false_values", ["no","false","0"])))
            def _etw_flag(row: pd.Series):
                c0 = _excel_col_to_idx("AW"); c1 = _excel_col_to_idx("AX")
                cols = list(row.index)
                subcols = [c for i, c in enumerate(cols) if c0 <= i <= c1]
                vals = [str(row.get(c, "")).strip().lower() for c in subcols]
                tr = any(v in true_vals  for v in vals)
                fl = any(v in false_vals for v in vals)
                if tr and not fl:     return True,  np.nan
                if fl and not tr:     return False, np.nan
                if not tr and not fl: return np.nan, "ETW_NONE_SELECTED"
                return np.nan, "ETW_BOTH_TRUE_FALSE"
            ef, warn_e = zip(*[_etw_flag(row) for _, row in df.iterrows()])
            df["is_etw"] = pd.Series(ef, index=df.index)

            
            gl = pair.get("group_labels", {"treated":"etwinning","control":"controllo"})
            df["group"] = np.where(df["is_etw"]==True,  gl.get("treated","etwinning"),
                          np.where(df["is_etw"]==False, gl.get("control","controllo"), np.nan))

            
            code_idx = _excel_col_to_idx(code_col_letter)
            df["code"] = df.iloc[:, code_idx].astype(str).str.strip() if 0 <= code_idx < df.shape[1] else np.nan

            
            fmt = pair.get("student_id_format", "{school}|{class}|{section}|{code}")
            df["student_id"] = df.apply(
                lambda r: fmt.format_map({
                    "school":  str(r.get("school",  "")).strip(),
                    "class":   str(r.get("class",   "")).strip(),
                    "section": str(r.get("section", "")).strip(),
                    "code":    str(r.get("code",    "")).strip(),
                }),
                axis=1
            )
            
            df["time"] = time_label
            df["_warn_identity"] = (pd.Series(warn_s).fillna("").astype(str) + "|" +
                                    pd.Series(warn_c).fillna("").astype(str) + "|" +
                                    pd.Series(warn_se).fillna("").astype(str) + "|" +
                                    pd.Series(warn_e).fillna("").astype(str))
            return df

        
        df_pre_norm  = _normalize_identity_lettered(df_pre_full,  school_map_pre,  "pre")
        df_post_norm = _normalize_identity_lettered(df_post_full, school_map_post, "post")
        
        for _d in (df_pre_norm, df_post_norm):
            for _c in ("school","class","section","code","_warn_identity","group","time","student_id"):
                if _c not in _d.columns:
                    _d[_c] = np.nan
        df = pd.concat([df_pre_norm, df_post_norm], ignore_index=True)

        logline(f"[ID] colonne create: student_id / group / time / school / class / section / code")

        
        bad = df[df["_warn_identity"].str.contains("MULTI_SELECTED|NONE_SELECTED|ETW_", na=False)]
        if len(bad) > 0:
            bad_path = os.path.join(diag_dir, "identity_warnings.csv")
            bad.to_csv(bad_path, index=False, encoding="utf-8")
            logline(f"[WARN] Incongruenze rilevate su selezioni J..X / Y..AA / AB..AV / AW..AX → {bad_path}")
        else:
            logline("[OK] Nessuna incongruenza nelle selezioni (J..X / Y..AA / AB..AV / AW..AX)")

        
        roster_cols = ["student_id","school","class","section","group","time","code"]
        try:
            roster = df[roster_cols].drop_duplicates()
            roster.to_csv(os.path.join(diag_dir, "identity_roster.csv"), index=False, encoding="utf-8")
            dup_mask = roster.duplicated(subset=["student_id"], keep=False)
            dups = roster.loc[dup_mask].sort_values("student_id")
            dups.to_csv(os.path.join(diag_dir, "identity_duplicates.csv"), index=False, encoding="utf-8")
            logline(f"[OK] Esportati roster e duplicati → identity_roster.csv / identity_duplicates.csv")
        except Exception as _e:
            logline(f"[INFO] Impossibile creare roster/duplicati: {_e}")
            
        
        def _bad_mask(df_norm: pd.DataFrame) -> pd.Series:
            core = ["school","class","section","code"]
            
            dsub = df_norm.reindex(columns=core, fill_value=np.nan)

            
            if "code" in df_norm.columns:
                code_series = df_norm["code"]
            else:
                code_series = pd.Series([""] * len(df_norm), index=df_norm.index)

            miss_core = dsub.isna().any(axis=1) | code_series.astype(str).str.strip().eq("")

            
            if "_warn_identity" in df_norm.columns:
                warn_series = df_norm["_warn_identity"].astype(str)
            else:
                warn_series = pd.Series([""] * len(df_norm), index=df_norm.index)

            warn_bad = warn_series.str.contains("MULTI_SELECTED|NONE_SELECTED|ETW_", na=False)

            return (miss_core | warn_bad)

        def _identity_stats(df_norm: pd.DataFrame) -> dict:
            bm = _bad_mask(df_norm)
            total = int(len(df_norm))
            lost  = int(bm.sum())
            ok    = int(total - lost)
            uniq_ok = int(df_norm.loc[~bm, "student_id"].nunique())
            dup_ok  = int(df_norm.loc[~bm, "student_id"].duplicated(keep=False).sum())
            return {"rows_total": total, "rows_lost": lost, "rows_identifiable": ok,
                    "unique_students_identifiable": uniq_ok, "dup_ids_within_time": dup_ok}

        pre_stats  = _identity_stats(df_pre_norm)
        post_stats = _identity_stats(df_post_norm)

        
        pre_ok_ids  = set(df_pre_norm.loc[~_bad_mask(df_pre_norm),  "student_id"].dropna().astype(str))
        post_ok_ids = set(df_post_norm.loc[~_bad_mask(df_post_norm), "student_id"].dropna().astype(str))
        matched_ids = pre_ok_ids & post_ok_ids

        summary_rows = [
            {"time":"pre",  **pre_stats},
            {"time":"post", **post_stats},
            {"time":"matched_pre_and_post", "unique_students_identifiable": len(matched_ids)}
        ]
        pd.DataFrame(summary_rows).to_csv(os.path.join(diag_dir, "identity_summary_counts.csv"),
                                          index=False, encoding="utf-8")

        
        def _reason(df_norm: pd.DataFrame) -> pd.Series:
            def r(row):
                rsn = []
                if pd.isna(row.get("school")):  rsn.append("missing_school")
                if pd.isna(row.get("class")):   rsn.append("missing_class")
                if pd.isna(row.get("section")): rsn.append("missing_section")
                if not row.get("code") or str(row.get("code")).strip()=="":
                    rsn.append("missing_code")
                w = str(row.get("_warn_identity","")).strip()
                if w: rsn.append(w)
                return "|".join(rsn) if rsn else "ok"
            return df_norm.apply(r, axis=1)

        bad_pre  = df_pre_norm.loc[_bad_mask(df_pre_norm)].copy()
        bad_post = df_post_norm.loc[_bad_mask(df_post_norm)].copy()
        if not bad_pre.empty:
            bad_pre["time"] = "pre";  bad_pre["reason"]  = _reason(bad_pre)
            bad_pre[["student_id","school","class","section","code","time","_warn_identity","reason"]]                .to_csv(os.path.join(diag_dir, "identity_unidentifiable_pre.csv"), index=False, encoding="utf-8")
        if not bad_post.empty:
            bad_post["time"] = "post"; bad_post["reason"] = _reason(bad_post)
            bad_post[["student_id","school","class","section","code","time","_warn_identity","reason"]]                .to_csv(os.path.join(diag_dir, "identity_unidentifiable_post.csv"), index=False, encoding="utf-8")

        logline("[OK] Esportato identity_summary_counts.csv (+ unidentifiable_pre/post.csv se presenti)")
        
        try:
            tbl_pre  = build_tables_with_uid_and_group(pre,  "pre")
            tbl_post = build_tables_with_uid_and_group(post, "post")
        except Exception as e:
            traceback.print_exc()
            messagebox.showerror("Errore CSV", str(e)); return
        df = pd.concat([tbl_pre, tbl_post], ignore_index=True)
        if df.empty:
            messagebox.showerror("Vuoto", "Nessuna risposta aperta trovata."); return
        df["area"] = df["competenza"].map(SPEC_TO_AREA_DEFAULT).fillna("Personale")

        
        pf = df.groupby(["area","time","group"], dropna=False).size().reset_index(name="n")
        safe_write_csv(pf, os.path.join(outd, "preflight_counts.csv"), columns=["area","time","group","n"])

        
        import re  

        ban_terms = set(BAN_TERMS_DEFAULT)
        try:
            with open(banf, "r", encoding="utf-8") as f:
                for line in f:
                    w = line.strip().lower()
                    if w:
                        
                        w = re.sub(r"\s+", " ", w)
                        ban_terms.add(w)

            
            ban_phrases = {w for w in ban_terms if " " in w}

            
            with open(os.path.join(outd, "effective_banlist.txt"), "w", encoding="utf-8") as f:
                for w in sorted(ban_terms):
                    f.write(w + "\n")
        except Exception as e:
            messagebox.showerror("BAN list", f"Errore nel leggere il file BAN: {e}")
            return
        
        
        keep_terms = set(map(str.lower, cfg.get("tokenizer", {}).get("keep_terms", [])))
        derived = derive_stopwords_from_corpus(
            df["text"].astype(str).tolist(),
            keep_terms,
            min_doc_frac=0.35,
            min_tf_short=10
        )
        tok = build_tokenizer(stop_extra=derived, keep_terms=keep_terms, ban_terms=ban_terms)
        tok_bow = build_bow_tokenizer(stop_extra=derived, keep_terms=keep_terms, ban_terms=ban_terms, lemmatize=bool(cfg.get("nlp", {}).get("lemmatize_bow", True)))

        with open(os.path.join(outd, "effective_stopwords.txt"), "w", encoding="utf-8") as f:
            for w in sorted(derived):
                f.write(w + "\n")

        
        df["sentiment_base"] = sentiment_baseline(df["text"].astype(str).tolist(), tokenizer=tok)
        df["ttr_base"]       = df["text"].astype(str).map(lambda s: ttr_simple(s, tok))
        df["mtld_base"]      = df["text"].astype(str).map(lambda s: mtld_std(s, tok))
        df["sentiment_json"] = sentiment_from_json(df["text"].astype(str).tolist(), tokenizer=tok, cfg=cfg)
        df["ttr_json"]       = df["text"].astype(str).map(lambda s: ttr_from_json(s, tok, cfg))
        df["mtld_json"]      = df["text"].astype(str).map(lambda s: mtld_from_json(s, tok, cfg))
        
        _sent = cfg.get("sentiment", {}) if isinstance(cfg, dict) else {}

        
        aro_lex = {str(k).lower(): float(v) for k, v in (_sent.get("arousal_lexicon", {}) or {}).items()}
        dom_lex = {str(k).lower(): float(v) for k, v in (_sent.get("dominance_lexicon", {}) or {}).items()}

        
        boosters = {str(k).lower(): float(v) for k, v in (_sent.get("boosters", {}) or {}).items()}
        negators = set(str(w).lower() for w in (_sent.get("negators", []) or []))
        window = int(_sent.get("window", 3))
        max_token_contrib = float(_sent.get("max_token_contrib", 3.0))

        
        if not aro_lex and not dom_lex:
            df["arousal_json"] = np.nan
            df["dominance_json"] = np.nan
        else:
            tmp_ad = df["text"].astype(str).map(
                lambda s: _score_arousal_dominance(
                    s, aro_lex, dom_lex, boosters, negators, window, max_token_contrib
                )
            )
            df[["arousal_json", "dominance_json"]] = pd.DataFrame(tmp_ad.tolist(), index=df.index)

        
        qcfg = _sent.get("quadrants", {"val_pos": 0.05, "val_neg": -0.05, "arousal_high": 0.25})
        
        if isinstance(qcfg, dict) and str(qcfg.get("mode","")).lower() == "percentile":
            import numpy as _np
            v = df["sentiment_json"].astype(float).to_numpy()
            a = df["arousal_json"].astype(float).to_numpy()
            p_pos  = float(qcfg.get("val_pos_p", 66.0))
            p_neg  = float(qcfg.get("val_neg_p", 33.0))
            p_ar_h = float(qcfg.get("arousal_high_p", 66.0))
            if _np.isfinite(v).sum() >= 5:
                try:
                    qcfg["val_pos"] = float(_np.nanpercentile(v, p_pos))
                    qcfg["val_neg"] = float(_np.nanpercentile(v, p_neg))
                except Exception:
                    pass
            if _np.isfinite(a).sum() >= 5:
                try:
                    qcfg["arousal_high"] = float(_np.nanpercentile(a, p_ar_h))
                except Exception:
                    pass
            try:
                _thp = os.path.join(outd, "quadrant_thresholds_used.json")
                with open(_thp, "w", encoding="utf-8") as _f:
                    json.dump({"used_thresholds": {"val_pos": qcfg.get("val_pos"),
                                                   "val_neg": qcfg.get("val_neg"),
                                                   "arousal_high": qcfg.get("arousal_high")},
                               "mode": "percentile",
                               "params": {"val_pos_p": p_pos, "val_neg_p": p_neg, "arousal_high_p": p_ar_h}}, _f, ensure_ascii=False, indent=2)
            except Exception:
                pass
        df["quadrant"] = [_quadrant(v, a, qcfg) for v, a in zip(df["sentiment_json"], df["arousal_json"])]

        
        df["delta_sentiment"] = df["sentiment_json"] - df["sentiment_base"]
        df["delta_ttr"]       = df["ttr_json"]       - df["ttr_base"]
        df["delta_mtld"]      = df["mtld_json"]      - df["mtld_base"]

        
        safe_write_csv(df, os.path.join(outd, "open_answers_long.csv"))
        
        try:
            export_lexicon_contributions(df, cfg, os.path.join(outd, "diagnostics"), tok_for_lex=_simple_tokenize)
        except Exception:
            pass
        
        def _nan_r(a, b):
            a = pd.Series(a, dtype="float64"); b = pd.Series(b, dtype="float64")
            m = a.notna() & b.notna()
            if m.sum() < 3: return float('nan')
            a1, b1 = a[m], b[m]
            if a1.nunique() <= 1 or b1.nunique() <= 1: return float('nan')
            return float(a1.corr(b1))
        summ = {"sentiment_r": _nan_r(df["sentiment_base"], df["sentiment_json"]),
                "ttr_r": _nan_r(df["ttr_base"], df["ttr_json"]),
                "mtld_r": _nan_r(df["mtld_base"], df["mtld_json"]),
                "n_rows": int(df.shape[0])}
        pd.DataFrame([summ]).to_csv(os.path.join(outd, "validation", "summary_overall.csv"), index=False, encoding="utf-8")
        (df[["UID","group","area","time","qid","sentiment_base","sentiment_json","delta_sentiment",
              "ttr_base","ttr_json","delta_ttr","mtld_base","mtld_json","delta_mtld","text"]]
         .to_csv(os.path.join(outd, "validation", "metrics_per_response.csv"), index=False, encoding="utf-8"))

        
        area_summary = df.groupby(["area","time"]).agg(
            sentiment=("sentiment_json","mean"),
            ttr=("ttr_json","mean"),
            mtld=("mtld_json","mean"),
            arousal=("arousal_json","mean"),
            dominance=("dominance_json","mean"),
            n=("text","count")
        ).reset_index()
        safe_write_csv(area_summary, os.path.join(outd, "area_summary.csv"))
        try:
            export_area_descriptors(df, tok_bow, os.path.join(outd, "area_descriptors.csv"))
        except Exception:
            pass
        if df["group"].notna().any():
            ag = df.dropna(subset=["group"]).groupby(["area","group","time"]).agg(
                sentiment=("sentiment_json","mean"),
                ttr=("ttr_json","mean"),
                mtld=("mtld_json","mean"),
                arousal=("arousal_json","mean"),
                dominance=("dominance_json","mean"),
                n=("text","count")
            ).reset_index()
            safe_write_csv(ag, os.path.join(outd, "area_group_summary.csv"))
            
            try:
                metrics_cols = [c for c in ["sentiment","ttr","mtld","arousal","dominance","n"] if c in ag.columns]

                
                rows = []
                for (a,g), sub in ag.groupby(["area","group"], dropna=False):
                    pre  = sub.loc[sub["time"]=="pre",  metrics_cols].head(1)
                    post = sub.loc[sub["time"]=="post", metrics_cols].head(1)
                    if pre.empty or post.empty:
                        deltas = {f"delta_{m}": np.nan for m in metrics_cols}
                    else:
                        deltas = {f"delta_{m}": float(post.iloc[0][m]) - float(pre.iloc[0][m]) for m in metrics_cols}
                    rows.append({"area": a, "group": g, **deltas})
                area_group_deltas = pd.DataFrame(rows)
                safe_write_csv(area_group_deltas, os.path.join(outd, "area_group_deltas.csv"))

                
                area_metrics = df.groupby(["area","time"]).agg(
                    sentiment=("sentiment_json","mean"),
                    ttr=("ttr_json","mean"),
                    mtld=("mtld_json","mean"),
                    arousal=("arousal_json","mean"),
                    dominance=("dominance_json","mean"),
                    n=("text","count")
                ).reset_index()
                rowsA = []
                for a, sub in area_metrics.groupby("area", dropna=False):
                    pre  = sub.loc[sub["time"]=="pre",  metrics_cols].head(1)
                    post = sub.loc[sub["time"]=="post", metrics_cols].head(1)
                    if pre.empty or post.empty:
                        deltas = {f"delta_{m}": np.nan for m in metrics_cols}
                    else:
                        deltas = {f"delta_{m}": float(post.iloc[0][m]) - float(pre.iloc[0][m]) for m in metrics_cols}
                    rowsA.append({"area": a, **deltas})
                area_deltas = pd.DataFrame(rowsA)
                safe_write_csv(area_deltas, os.path.join(outd, "area_deltas.csv"))

                
                comp = df.groupby(["group","time"]).agg(
                    sentiment=("sentiment_json","mean"),
                    ttr=("ttr_json","mean"),
                    mtld=("mtld_json","mean"),
                    arousal=("arousal_json","mean"),
                    dominance=("dominance_json","mean"),
                    n=("text","count")
                ).reset_index()
                safe_write_csv(comp, os.path.join(outd, "competence_group_summary.csv"))

                
                rowsC = []
                for g, sub in comp.groupby("group", dropna=False):
                    pre  = sub.loc[sub["time"]=="pre",  metrics_cols].head(1)
                    post = sub.loc[sub["time"]=="post", metrics_cols].head(1)
                    if pre.empty or post.empty:
                        deltas = {f"delta_{m}": np.nan for m in metrics_cols}
                    else:
                        deltas = {f"delta_{m}": float(post.iloc[0][m]) - float(pre.iloc[0][m]) for m in metrics_cols}
                    rowsC.append({"group": g, **deltas})
                comp_deltas = pd.DataFrame(rowsC)
                safe_write_csv(comp_deltas, os.path.join(outd, "competence_group_deltas.csv"))
            except Exception as _e:
                try:
                    with open(os.path.join(outd, "diagnostics", "delta_exports_error.txt"), "w", encoding="utf-8") as _f:
                        _f.write(str(_e))
                except Exception:
                    pass


        
        wc_n = build_wordclouds(df, tok_bow, figs_dir)
        self._log(f"Wordclouds create: {wc_n} file (OUTPUT/graphs/)")

        
        def build_qid_wordclouds(df: pd.DataFrame,
                                 qids: list,
                                 tok,
                                 figs_dir: str,
                                 qid_col: str = "qid",
                                 text_col: str = "text",
                                 group_col: str = "group",
                                 ban_terms: set = None,
                                 ban_phrases: set = None,
                                 topN_words: int = 500) -> int:
            
            import os, re
            import matplotlib.pyplot as plt
            from collections import Counter

            os.makedirs(figs_dir, exist_ok=True)
            made = 0

            def _save_wc(freqs: dict, out_path: str):
                nonlocal made
                if not freqs:
                    return
                if WordCloud is None:
                    
                    items = sorted(freqs.items(), key=lambda x: x[1], reverse=True)[:min(75, len(freqs))]
                    if not items:
                        return
                    labels = [k for k, _ in items]
                    vals   = [v for _, v in items]
                    plt.figure(figsize=(12, 6))
                    plt.bar(range(len(vals)), vals)
                    plt.xticks(range(len(vals)), labels, rotation=90, fontsize=8)
                    plt.tight_layout()
                    plt.savefig(out_path, dpi=160)
                    plt.close()
                    made += 1
                    return

                wc = WordCloud(width=1600, height=900, background_color="white",
                               prefer_horizontal=0.9, collocations=False,
                               normalize_plurals=False)
                wc.generate_from_frequencies(freqs)
                plt.figure(figsize=(12, 7))
                plt.imshow(wc, interpolation="bilinear")
                plt.axis("off")
                plt.tight_layout(pad=0)
                plt.savefig(out_path, dpi=180)
                plt.close()
                made += 1

            for qid in qids:
                base = df[df[qid_col].astype(str).eq(str(qid))]
                for label, gname in [("all", None), ("etwinning", "etwinning"), ("controllo", "controllo")]:
                    sub = base if gname is None else base[base[group_col].astype(str).str.lower().eq(gname)]
                    if sub.empty:
                        continue

                    tokens = []
                    for s in sub[text_col].astype(str):
                        s_norm = re.sub(r"\s+", " ", s.strip().lower())
                        
                        if ban_phrases and s_norm in ban_phrases:
                            continue
                        for t in tok(s_norm):
                            if ban_terms and t in ban_terms:
                                continue
                            if len(t) < 2:
                                continue
                            tokens.append(t)

                    if not tokens:
                        continue

                    cnt = Counter(tokens)
                    freqs = dict(cnt.most_common(topN_words))
                    out_png = os.path.join(figs_dir, f"wordcloud_{qid}_{label}.png")
                    _save_wc(freqs, out_png)

            return made

        
        qcfg = cfg.get("questions", {})

        def _pick_col(cols, candidates):
            for c in candidates:
                if c in cols:
                    return c
            return None

        qid_col   = qcfg.get("question_id_col") or _pick_col(df.columns, ["question_id","id_domanda","qid","domanda_id"])
        qtext_col = qcfg.get("question_text_col") or _pick_col(df.columns, ["question","domanda","question_text","testo_domanda"])
        rtext_col = (qcfg.get("response_text_col") 
                     or _pick_col(df.columns, ["text","risposta","response","open_answer"]) 
                     or "text")
        area_col  = qcfg.get("area_col", "area")
        group_col = qcfg.get("group_col", "group")
        time_col  = qcfg.get("time_col", "time")
        id_col    = qcfg.get("id_col", ("student_id" if "student_id" in df.columns else None))


        
        qid_list_for_wc = ["Q1_benessere", "Q2_aspirazioni", "Q4_punti_forza", "Q5_migliorare"]
        wc_qid_n = build_qid_wordclouds(
            df=df,
            qids=qid_list_for_wc,
            tok=tok_bow,
            figs_dir=figs_dir,
            qid_col=qid_col,          
            text_col=rtext_col,
            group_col=group_col,
            ban_terms=ban_terms,      
            ban_phrases=ban_phrases   
        )
        self._log(f"[OK] Wordcloud QID-specifiche create: {wc_qid_n} file")

        
        kmade = key_terms_and_collocations(df, tok_bow, ban_terms, outd)

        
        topics_all_prev = []
        topics_terms_count = 0
        for area, sub in df.groupby("area"):
            model = topic_model_area(sub, tok_bow)
            if model is None:
                safe_write_csv(pd.DataFrame(columns=["area","topic","rank","term","weight"]), os.path.join(outd, f"topics_terms_area_{area}.csv"), columns=["area","topic","rank","term","weight"])
                continue
            topics_prev = export_topics(area, model, sub, outd)
            topics_all_prev.append(topics_prev)
            topics_terms_count += 1
        if topics_all_prev:
            prev_all = pd.concat(topics_all_prev, ignore_index=True)
            prev_path = os.path.join(outd, "topics_prevalence_area_group.csv")
            if os.path.exists(prev_path):
                old = pd.read_csv(prev_path)
                prev_all = pd.concat([old, prev_all], ignore_index=True).drop_duplicates()
            safe_write_csv(prev_all, prev_path)
        else:
            safe_write_csv(pd.DataFrame(columns=["area","group","time","topic","n","tot","prop"]), os.path.join(outd,"topics_prevalence_area_group.csv"),
                           columns=["area","group","time","topic","n","tot","prop"])
        export_topic_shifts(outd)
        try:
            n_hm = plot_topic_heatmaps(outd, graphs_dir)
            if n_hm:
                self._log(f"[OK] Heatmaps topics salvate in {graphs_dir} ({n_hm} file)")
            else:
                self._log("[INFO] Nessuna heatmap topics generata (file o dati mancanti).")
        except Exception as _e_hm:
            try:
                with open(os.path.join(outd, 'diagnostics', 'heatmaps_error.txt'), 'w', encoding='utf-8') as _f:
                    _f.write(traceback.format_exc())
            except Exception:
                pass
            self._log(f"[WARN] Errore nella generazione heatmaps: {_e_hm}")

        
        ex_n = exemplar_quotes(df, tok_bow, outd, k_per_bucket=6)

        
        prog = df.groupby(["area","group","time"]).agg(sentiment=("sentiment_json","mean"), mtld=("mtld_json","mean")).reset_index()
        safe_write_csv(prog, os.path.join(outd,"progression_area_group.csv"))
        did_rows = []
        for area, sub in prog.groupby("area"):
            for metric in ["sentiment","mtld"]:
                def val(g,t):
                    v = sub[(sub["group"]==g) & (sub["time"]==t)][metric]
                    return float(v.values[0]) if len(v)==1 else float("nan")
                etw_delta = (val("etwinning","post") - val("etwinning","pre")) if all(~pd.isna([val("etwinning","post"), val("etwinning","pre")])) else float("nan")
                ctl_delta = (val("controllo","post") - val("controllo","pre")) if all(~pd.isna([val("controllo","post"), val("controllo","pre")])) else float("nan")
                did = (etw_delta - ctl_delta) if (not math.isnan(etw_delta) and not math.isnan(ctl_delta)) else float("nan")
                did_rows.append({"area":area,"metric":metric,"did":did,"delta_etwinning":etw_delta,"delta_controllo":ctl_delta})
        safe_write_csv(pd.DataFrame(did_rows), os.path.join(outd,"did_area_metrics.csv"), columns=["area","metric","did","delta_etwinning","delta_controllo"])
        try:
            export_did_with_pvalues(df, os.path.join(outd, "did_with_pvalues.csv"))
        except Exception:
            pass

        
        try:
            
            metrics_cols = [c for c in ["sentiment_json","ttr_json","mtld_json","arousal_json","dominance_json"] if c in df.columns]
            rows_comp = []
            for m in metrics_cols:
                def _val(g, t):
                    v = df[(df[group_col]==g) & (df[time_col]==t)][m].astype(float)
                    return float(v.mean()) if len(v)>0 else float("nan")
                de = (_val("etwinning","post") - _val("etwinning","pre"))
                dc = (_val("controllo","post") - _val("controllo","pre"))
                did = (de - dc) if (not math.isnan(de) and not math.isnan(dc)) else float("nan")
                rows_comp.append({"metric": m, "did": did, "delta_etwinning": de, "delta_controllo": dc})
            safe_write_csv(pd.DataFrame(rows_comp),
                           os.path.join(outd, "did_competence_metrics.csv"),
                           columns=["metric","did","delta_etwinning","delta_controllo"])

            
            export_competence_did_with_pvalues(
                df,
                os.path.join(outd, "competence_did_with_pvalues.csv"),
                group_col=group_col,
                time_col=time_col,
                metrics=tuple(metrics_cols)
            )
        except Exception:
            with open(os.path.join(outd, "diagnostics", "competence_did_error.txt"), "w", encoding="utf-8") as _f:
                _f.write(traceback.format_exc())

        

        
        qid_col   = qid_col   or _pick_col(df.columns, ["question_id","id_domanda","qid","domanda_id"])
        qtext_col = qtext_col or _pick_col(df.columns, ["question","domanda","question_text","testo_domanda"])
        rtext_col = rtext_col or _pick_col(df.columns, ["text","risposta","response","open_answer"])

        normalize = bool(qcfg.get("normalize_responses", True))
        top_k     = int(qcfg.get("top_k", 20))
        id_col = qcfg.get("id_col", ("student_id" if "student_id" in df.columns else None))        
        area_col  = qcfg.get("area_col", "area")
        group_col = qcfg.get("group_col", "group")
        time_col  = qcfg.get("time_col", "time")

        
        WORD_QS_UPPER = {
            s.strip().upper() for s in (cfg.get("questions", {}).get("topk_word_ids", ["Q1","Q2","Q4","Q5"]))
        }
        def _word_mode(qid_val):
            return str(qid_val).strip().upper() in WORD_QS_UPPER

        
        WORD_QS_UPPER = {
            s.strip().upper() for s in (cfg.get("questions", {}).get("topk_word_ids", ["Q1","Q2","Q4","Q5"]))
        }
        def _word_mode(qid_val):
            return str(qid_val).strip().upper() in WORD_QS_UPPER


        

        
        special_qids = {"Q1_benessere", "Q2_aspirazioni", "Q4_punti_forza", "Q5_migliorare"}

        def _topk_for_subset(sub_df: pd.DataFrame, outfile: str):
            if sub_df.empty:
                logline(f"[INFO] Nessuna risposta utile per Top-K → {os.path.basename(outfile)}")
                return

            
            tmp = sub_df[[qid_col, qtext_col, rtext_col]].dropna(subset=[rtext_col]).copy()
            tmp["__norm"] = (
                tmp[rtext_col]
                .astype(str)
                .str.strip()
                .str.lower()
                .str.replace(r"\s+", " ", regex=True)
            )

            
            tmp = tmp[~tmp["__norm"].isin(ban_phrases)]

            rows = []
            for qid_val, g in tmp.groupby(qid_col, dropna=False):
                qtxt = g[qtext_col].iloc[0]

                if str(qid_val) in special_qids:
                    
                    from collections import Counter
                    tokens = []
                    for s in g["__norm"]:
                        for t in tok(s):  
                            
                            if t in ban_terms:
                                continue
                            if len(t) < 2:
                                continue
                            tokens.append(t)
                    if tokens:
                        cnt = Counter(tokens).most_common(top_k)
                        vc = pd.DataFrame(cnt, columns=["response", "n"])
                    else:
                        vc = pd.DataFrame(columns=["response", "n"])
                    vc["granularity"] = "word"
                else:
                    
                    vc = (
                        g["__norm"]
                        .value_counts(dropna=False)
                        .head(top_k)
                        .reset_index()
                        .rename(columns={"index": "response", "count": "n"})
                    )
                    vc["granularity"] = "verbatim"

                vc.insert(0, qid_col, qid_val)
                vc.insert(1, qtext_col, qtxt)
                rows.append(vc)

            if rows:
                out = pd.concat(rows, ignore_index=True)
                out.to_csv(outfile, index=False, encoding="utf-8")
                logline(f"[OK] Esportato {os.path.basename(outfile)} ({len(out)} righe)")
            else:
                logline(f"[INFO] Nessun dato esportato per Top-K → {os.path.basename(outfile)}")

        
        if qid_col and qtext_col and rtext_col and qid_col in df and qtext_col in df and rtext_col in df:
            _topk_for_subset(df, os.path.join(outd, "topk_answers_per_question.csv"))
        else:
            logline("[INFO] Colonne domanda/risposta non trovate: salto Top-K.")

        
        if qid_col and qtext_col and rtext_col and qid_col in df and qtext_col in df and rtext_col in df:
            for gname in ["etwinning", "controllo"]:
                sub = df[df[group_col].astype(str).str.lower().eq(gname)]
                _topk_for_subset(sub, os.path.join(outd, f"topk_answers_per_question_{gname}.csv"))

                        
        
        
        metrics = [c for c in ["sentiment_json","ttr_json","mtld_json","arousal_json","dominance_json"] if c in df.columns]
        p_rows = []
        if all(c in df.columns for c in [area_col, group_col, time_col]) and metrics:
            for area, g_area in df.groupby(area_col, dropna=False):
                for m in metrics:
                    
                    for grp, g_grp in g_area.groupby(group_col, dropna=False):
                        pre_vals  = g_grp.loc[g_grp[time_col].eq("pre"),  m].dropna().values
                        post_vals = g_grp.loc[g_grp[time_col].eq("post"), m].dropna().values
                        if len(pre_vals) > 2 and len(post_vals) > 2:
                            
                            if id_col and id_col in g_grp.columns:
                                pre_id  = g_grp.loc[g_grp[time_col].eq("pre"),  [id_col, m]].dropna()
                                post_id = g_grp.loc[g_grp[time_col].eq("post"), [id_col, m]].dropna()
                                merged  = pre_id.merge(post_id, on=id_col, suffixes=("_pre","_post"))
                                if len(merged) > 2:
                                    _, p_within = ttest_rel(merged[m+"_post"], merged[m+"_pre"], nan_policy="omit")
                                else:
                                    _, p_within = ttest_ind(post_vals, pre_vals, equal_var=False, nan_policy="omit")
                            else:
                                _, p_within = ttest_ind(post_vals, pre_vals, equal_var=False, nan_policy="omit")
                        else:
                            p_within = np.nan
                        p_rows.append({"area": area, "metric": m, "group": grp, "test": "pre_vs_post", "p_value": p_within})

                    
                    g_et  = g_area[g_area[group_col].astype(str).str.lower().eq("etwinning")]
                    g_ct  = g_area[g_area[group_col].astype(str).str.lower().eq("controllo")]
                    d_et  = g_et.loc[g_et[time_col].eq("post"), m].dropna().values - g_et.loc[g_et[time_col].eq("pre"),  m].dropna().mean() if len(g_et)>0 else np.array([])
                    d_ct  = g_ct.loc[g_ct[time_col].eq("post"), m].dropna().values - g_ct.loc[g_ct[time_col].eq("pre"),  m].dropna().mean() if len(g_ct)>0 else np.array([])
                    if d_et.size>2 and d_ct.size>2:
                        _, p_did = ttest_ind(d_et, d_ct, equal_var=False, nan_policy="omit")
                    else:
                        p_did = np.nan
                    p_rows.append({"area": area, "metric": m, "group": "DID(etwinning−controllo)", "test": "did_simple", "p_value": p_did})

            ptab = pd.DataFrame(p_rows)
            
            alpha = float(cfg.get("stats", {}).get("alpha", 0.05))  
            ptab["signif"]   = np.where(ptab["p_value"].astype(float) < alpha, "✓", "")
            ptab["sig_code"] = pd.cut(ptab["p_value"].astype(float),
                                      bins=[-np.inf, 0.001, 0.01, 0.05, np.inf],
                                      labels=["***","**","*",""])
            ptab.to_csv(os.path.join(outd, "pvalues_area_metrics.csv"), index=False, encoding="utf-8")
            logline(f"[OK] Esportato pvalues_area_metrics.csv ({len(ptab)} righe)")
        else:
            logline("[INFO] Colonne area/group/time o metriche mancanti: salto p-value.")

        
        if "quadrant" in df.columns and all(c in df.columns for c in [area_col, group_col, time_col]):
            quad_cnt = (
                df.groupby([area_col, group_col, time_col, "quadrant"], as_index=False)
                  .size()
                  .rename(columns={"size": "n"})
            )
            quad_cnt["share"] = quad_cnt.groupby([area_col, group_col, time_col])["n"].transform(lambda x: (x / x.sum()).round(4))
            quad_cnt.to_csv(os.path.join(outd, "quadrant_share_area_group_time.csv"), index=False, encoding="utf-8")

            
            try:
                q = quad_cnt.copy()
                pre  = q[q[time_col] == "pre"]
                post = q[q[time_col] == "post"]
                key = [area_col, group_col, "quadrant"]

                pre_s  = pre[key + ["share"]].rename(columns={"share": "share_pre"})
                post_s = post[key + ["share"]].rename(columns={"share": "share_post"})
                merged = pre_s.merge(post_s, on=key, how="outer")
                merged["delta_share"] = merged["share_post"].fillna(0.0) - merged["share_pre"].fillna(0.0)

                merged.to_csv(os.path.join(outd, "quadrant_share_area_group_deltas.csv"), index=False, encoding="utf-8")
            except Exception:
                pass

            
            try:
                if "quadrant" in df.columns:
                    dcomp = (df.groupby([group_col, time_col, "quadrant"], as_index=False)
                               .size().rename(columns={"size":"n"}))
                    dcomp["share"] = dcomp.groupby([group_col, time_col])["n"].transform(lambda x: (x/x.sum()).round(4))
                    dcomp.to_csv(os.path.join(outd, "quadrant_share_COMPETENZA_group_time.csv"), index=False, encoding="utf-8")

                    pre_c  = dcomp[dcomp[time_col] == "pre"]
                    post_c = dcomp[dcomp[time_col] == "post"]
                    key_c = [group_col, "quadrant"]
                    pre_c = pre_c[key_c + ["share"]].rename(columns={"share":"share_pre"})
                    post_c= post_c[key_c + ["share"]].rename(columns={"share":"share_post"})
                    merged_c = pre_c.merge(post_c, on=key_c, how="outer")
                    merged_c["delta_share"] = merged_c["share_post"].fillna(0.0) - merged_c["share_pre"].fillna(0.0)
                    merged_c.to_csv(os.path.join(outd, "quadrant_share_COMPETENZA_group_deltas.csv"), index=False, encoding="utf-8")
            except Exception:
                pass


            
            try:
                
                qcols = [c for c in quad_cnt.columns if c not in ("n",)]
                q = quad_cnt.copy()
                
                q = q[[c for c in qcols if c in ["area","group","time","quadrant","share"]]]
                rowsQ = []
                for (a,g,qd), sub in q.groupby(["area","group","quadrant"], dropna=False):
                    pre  = sub.loc[sub["time"]=="pre",  "share"]
                    post = sub.loc[sub["time"]=="post", "share"]
                    prev = float(pre.values[0]) if len(pre)==1 else np.nan
                    postv= float(post.values[0]) if len(post)==1 else np.nan
                    dv = postv - prev if (not np.isnan(prev) and not np.isnan(postv)) else np.nan
                    rowsQ.append({"area": a, "group": g, "quadrant": qd, "delta_share": dv})
                quad_deltas = pd.DataFrame(rowsQ)
                safe_write_csv(quad_deltas, os.path.join(outd, "quadrant_share_deltas_area_group.csv"))

                
                qq = df.groupby(["group","time","quadrant"]).size().reset_index(name="n")
                qq["share"] = qq.groupby(["group","time"])["n"].transform(lambda x: (x / x.sum()).round(4))
                safe_write_csv(qq, os.path.join(outd, "competence_quadrant_share_group_time.csv"))

                
                rowsQC = []
                for (g,qd), sub in qq.groupby(["group","quadrant"], dropna=False):
                    pre  = sub.loc[sub["time"]=="pre",  "share"]
                    post = sub.loc[sub["time"]=="post", "share"]
                    prev = float(pre.values[0]) if len(pre)==1 else np.nan
                    postv= float(post.values[0]) if len(post)==1 else np.nan
                    dv = postv - prev if (not np.isnan(prev) and not np.isnan(postv)) else np.nan
                    rowsQC.append({"group": g, "quadrant": qd, "delta_share": dv})
                comp_quad_deltas = pd.DataFrame(rowsQC)
                safe_write_csv(comp_quad_deltas, os.path.join(outd, "competence_quadrant_share_deltas.csv"))
            except Exception as _e:
                try:
                    with open(os.path.join(outd, "diagnostics", "quadrant_delta_error.txt"), "w", encoding="utf-8") as _f:
                        _f.write(str(_e))
                except Exception:
                    pass
            logline(f"[OK] Esportato quadrant_share_area_group_time.csv ({len(quad_cnt)} righe)")
            
            try:
                save_result_plots(df, outd, figs_dir)
                logline(f"[OK] Grafici riepilogo salvati in {figs_dir}")
            except Exception:
                
                try:
                    os.makedirs(os.path.join(outd, "diagnostics"), exist_ok=True)
                    with open(os.path.join(outd, "diagnostics", "plots_error.txt"), "w", encoding="utf-8") as _f:
                        _f.write(traceback.format_exc())
                    logline("[WARN] Errore nella generazione dei PNG. Vedi diagnostics/plots_error.txt")
                except Exception as _inner:
                    logline(f"[WARN] Errore nella generazione dei PNG e nella scrittura del diagnostics: {str(_inner)}")

        else:
            logline("[INFO] Colonna 'quadrant' non trovata: salto tabella percentuali quadranti.")
        
        with open(os.path.join(outd, "group_mapping_reason.txt"),"w",encoding="utf-8") as f:
            f.write("Gruppo determinato HARD-CODED da colonne AW/AX (riga 3+).\nRegola: AW='Si'/'Sì'→eTwinning; AX='No'→Controllo.\n")
            f.write(str(df["group"].value_counts(dropna=False)) + "\n")

        
        sent_w = cfg.get("sentiment",{}).get("lexicon",{})
        ttr_w  = cfg.get("ttr",{}).get("weights",{})
        mtld_w = cfg.get("mtld",{}).get("weights",{})
        
        def _save_weights_table(name, dct):
            tab = pd.DataFrame({"term": list(dct.keys()), "weight": [float(dct[k]) for k in dct.keys()]})
            tab = tab.sort_values("weight", ascending=False)
            safe_write_csv(tab, os.path.join(diag_dir, f"{name}_weights.csv"))
            plt.figure(figsize=(6,4)); 
            if not tab.empty: plt.hist(tab["weight"].astype(float), bins=50)
            plt.title(f"Distribuzione {name} weights"); plt.tight_layout()
            plt.savefig(os.path.join(diag_dir, f"{name}_weights_hist.png"), dpi=140); plt.close()
            top_pos = tab.head(50); top_neg = tab.sort_values("weight").head(50)
            safe_write_csv(top_pos, os.path.join(diag_dir, f"{name}_top_positive.csv"))
            safe_write_csv(top_neg, os.path.join(diag_dir, f"{name}_top_negative.csv"))
            return len(tab)
        n_sent = _save_weights_table("sentiment", sent_w)
        n_ttr  = _save_weights_table("ttr", ttr_w)
        n_mtld = _save_weights_table("mtld", mtld_w)

        
        tokens_all = []
        for txt in df["text"].astype(str).tolist():
            tokens_all.extend(tok_bow(txt))
        total_tokens = len(tokens_all)
        cov_sent = sum(1 for t in tokens_all if t in sent_w) / max(1, total_tokens)
        cov_ttr  = sum(1 for t in tokens_all if t in ttr_w) / max(1, total_tokens)
        cov_mtld = sum(1 for t in tokens_all if t in mtld_w) / max(1, total_tokens)
        cov = pd.DataFrame([{"metric":"sentiment","coverage_tokens":cov_sent},
                            {"metric":"ttr","coverage_tokens":cov_ttr},
                            {"metric":"mtld","coverage_tokens":cov_mtld}])
        safe_write_csv(cov, os.path.join(diag_dir, "weights_coverage.csv"))

        
        with open(os.path.join(diag_dir,"diagnostics_report.html"),"w",encoding="utf-8") as f:
            f.write("<!DOCTYPE html><meta charset='utf-8'><title>Diagnostics – Weights</title>")
            f.write("<h1>Diagnostics – Pesi e copertura</h1>")
            f.write(f"<p>Generato: {html.escape(datetime.now().strftime('%Y-%m-%d %H:%M'))}</p>")
            f.write("<ul>")
            f.write(f"<li>Sentiment lexicon: {n_sent} termini</li>")
            f.write(f"<li>TTR weights: {n_ttr} termini</li>")
            f.write(f"<li>MTLD weights: {n_mtld} termini</li>")
            f.write("</ul>")
            f.write("<h3>Copertura lessico (token-level)</h3>")
            f.write(f"<p>Coverage sentiment: {cov_sent:.3f} · TTR: {cov_ttr:.3f} · MTLD: {cov_mtld:.3f}</p>")
            for name in ["sentiment","ttr","mtld"]:
                f.write(f"<h3>Distribuzione pesi – {name.upper()}</h3>")
                f.write(f"<img src='{name}_weights_hist.png' style='width:520px;display:block;margin-bottom:16px'>")
            f.write("<p>Vedi anche: *_weights.csv, *_top_positive.csv, *_top_negative.csv.</p>")

        
        status = {
            "wordcloud_written": wc_n,
            "key_terms_files": kmade["key_terms"],
            "collocation_bi_files": kmade["colloc_bi"],
            "collocation_tri_files": kmade["colloc_tri"],
            "topics_terms_areas": topics_terms_count,
            "exemplars_buckets": ex_n,
            "group_present": bool(df["group"].notna().any()),
            "diagnostics_cov_sentiment": cov_sent,
            "diagnostics_cov_ttr": cov_ttr,
            "diagnostics_cov_mtld": cov_mtld
        }
        write_text(os.path.join(outd,"module_status.json"), json.dumps(status, ensure_ascii=False, indent=2))

        
        if WordCloud is None:
            self._log("⚠️ WordCloud non installato: create barre dei top termini in OUTPUT/graphs/. Per abilitare le nuvole vere: pip install wordcloud")

        
        
        try:
            import shutil
            if os.path.isdir(graphs_dir):
                os.makedirs(figs_legacy_dir, exist_ok=True)
                for _fn in os.listdir(graphs_dir):
                    if _fn.lower().endswith(".png"):
                        shutil.copy2(os.path.join(graphs_dir, _fn), os.path.join(figs_legacy_dir, _fn))
        except Exception:
            pass

        messagebox.showinfo(
            "Fatto",
            f"Analisi completata. Cartella: {outd}\nGrafici/heatmap/wordclouds in {os.path.join(outd,'graphs')}.",
        )
        self._log("OK.")




if __name__ == "__main__":
    App().mainloop()
