from __future__ import annotations
import os, re, html, pathlib, math, json, sys, traceback, statistics, warnings, io
from datetime import datetime
from typing import List, Dict, Tuple, Optional

import tkinter as tk
from tkinter import ttk, filedialog, messagebox

import numpy as np
import pandas as pd
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

warnings.filterwarnings("ignore", category=RuntimeWarning)

try:
    import ftfy
except Exception:
    ftfy = None
try:
    from wordcloud import WordCloud
except Exception:
    WordCloud = None

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from scipy.stats import ttest_ind, ttest_rel
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.cluster import KMeans


try:
    import statsmodels.formula.api as smf
except Exception:
    smf = None


from collections import Counter


import numpy as np
import pandas as pd
from typing import Tuple

def _excel_col_to_idx(col_letter: str) -> int:
    col_letter = str(col_letter).strip().upper()
    n = 0
    for ch in col_letter:
        if 'A' <= ch <= 'Z':
            n = n * 26 + (ord(ch) - ord('A') + 1)
    return n - 1  

def _pick_one_hot_category(df: pd.DataFrame, start_col: str, end_col: str,
                           truthy=("1","x","✓","true","yes","si","sì")) -> Tuple[pd.Series, pd.Series]:
    tset = set(s.lower() for s in truthy)
    c0, c1 = _excel_col_to_idx(start_col), _excel_col_to_idx(end_col)
    cols = df.columns.tolist()
    subcols = [c for i, c in enumerate(cols) if c0 <= i <= c1]

    cat_vals, warns = [], []
    for _, row in df.iterrows():
        hits = []
        for c in subcols:
            val = str(row.get(c, "")).strip().lower()
            if val in tset:
                hits.append(c)
        if len(hits) == 1:
            cat_vals.append(hits[0])     
            warns.append(np.nan)
        elif len(hits) == 0:
            cat_vals.append(np.nan)
            warns.append(f"NONE_SELECTED:{start_col}-{end_col}")
        else:
            cat_vals.append(np.nan)
            warns.append(f"MULTI_SELECTED:{start_col}-{end_col}:{'|'.join(hits)}")
    return pd.Series(cat_vals, index=df.index), pd.Series(warns, index=df.index)

def _resolve_etw_flag(df: pd.DataFrame, start_col: str, end_col: str,
                      true_vals=("si","sì","yes","true","1","x","✓"),
                      false_vals=("no","false","0")) -> Tuple[pd.Series, pd.Series]:
    tset, fset = set(s.lower() for s in true_vals), set(s.lower() for s in false_vals)
    c0, c1 = _excel_col_to_idx(start_col), _excel_col_to_idx(end_col)
    cols = df.columns.tolist()
    subcols = [c for i, c in enumerate(cols) if c0 <= i <= c1]

    flags, warns = [], []
    for _, row in df.iterrows():
        tr_hit = any(str(row.get(c, "")).strip().lower() in tset for c in subcols)
        fl_hit = any(str(row.get(c, "")).strip().lower() in fset for c in subcols)
        if tr_hit and not fl_hit:
            flags.append(True);  warns.append(np.nan)
        elif fl_hit and not tr_hit:
            flags.append(False); warns.append(np.nan)
        elif not tr_hit and not fl_hit:
            flags.append(np.nan); warns.append("ETW_NONE_SELECTED")
        else:
            flags.append(np.nan); warns.append("ETW_BOTH_TRUE_FALSE")
    return pd.Series(flags, index=df.index), pd.Series(warns, index=df.index)


import re as _re_inj, math as _math_inj
_WORD_RE_INJ = _re_inj.compile(r"[A-Za-zÀ-ÖØ-öø-ÿ']+")

def _simple_tokenize(text: str):
    return _WORD_RE_INJ.findall(str(text).lower())

def _score_arousal_dominance(text, aro_lex, dom_lex, boosters, negators, window=2, max_token_contrib=3.0):
    
    toks = _simple_tokenize(text)
    n = len(toks)
    tot_a = 0.0; cnt_a = 0
    tot_d = 0.0; cnt_d = 0
    for i, w in enumerate(toks):
        a = aro_lex.get(w) if aro_lex else None
        d = dom_lex.get(w) if dom_lex else None
        if a is None and d is None:
            continue
        L = max(0, i-window); R = min(n, i+1+window)
        ctx = toks[L:i] + toks[i+1:R]
        boost = 1.0
        if boosters:
            for b in ctx:
                if b in boosters:
                    try: boost *= float(boosters[b])
                    except Exception: pass
        
        if a is not None:
            contrib_a = max(-max_token_contrib, min(max_token_contrib, a * boost))
            tot_a += contrib_a; cnt_a += 1
        if d is not None:
            contrib_d = max(-max_token_contrib, min(max_token_contrib, d * boost))
            tot_d += contrib_d; cnt_d += 1
    arousal = (tot_a / cnt_a) if cnt_a else float("nan")
    dominance = (tot_d / cnt_d) if cnt_d else float("nan")
    return (float(arousal), float(dominance))

def _quadrant(valence, arousal, qcfg):
    
    if arousal is None or (isinstance(arousal, float) and _math_inj.isnan(arousal)):
        return "Neutral/MissingArousal"
    vpos = float(qcfg.get("val_pos", 0.05)); vneg = float(qcfg.get("val_neg", -0.05)); ah = float(qcfg.get("arousal_high", 0.25))
    if valence >= vpos and arousal >= ah: return "Positive-HighArousal"
    if valence >= vpos and arousal <  ah: return "Positive-LowArousal"
    if valence <= vneg and arousal >= ah: return "Negative-HighArousal"
    if valence <= vneg and arousal <  ah: return "Negative-LowArousal"
    return "Neutral/Mixed"



def _norm_time_label(x: str) -> str:
    s = str(x).strip().lower()
    if s in {"pre","ingresso","baseline","t0","0","before"}: return "pre"
    if s in {"post","uscita","followup","t1","1","after"}:   return "post"
    return s

def _is_treated(s: str) -> int:
    ss = str(s).strip().lower()
    return int("twin" in ss or "sper" in ss or "treat" in ss or "etwinning" in ss)

def _welch_t_pvalue(x_pre, x_post):
    
    import numpy as np
    x_pre = np.asarray(x_pre, float)
    x_post = np.asarray(x_post, float)
    x_pre = x_pre[~np.isnan(x_pre)]
    x_post = x_post[~np.isnan(x_post)]
    if len(x_pre) < 2 or len(x_post) < 2:
        return float("nan")
    m1, m2 = x_pre.mean(), x_post.mean()
    v1, v2 = x_pre.var(ddof=1), x_post.var(ddof=1)
    n1, n2 = len(x_pre), len(x_post)
    se = math.sqrt(v1/n1 + v2/n2)
    if se == 0: return float("nan")
    t = (m2 - m1) / se
    
    df = (v1/n1 + v2/n2)**2 / ((v1**2)/((n1**2)*(n1-1)) + (v2**2)/((n2**2)*(n2-1)))
    from scipy import stats
    return 2*stats.t.sf(abs(t), df)

def _did_ols_pvalue(df_area, metric, group_col="group", time_col="time"):
    
    if smf is None:
        return (float("nan"), float("nan"))
    d = df_area[[metric, group_col, time_col]].dropna().copy()
    if d.empty: return (float("nan"), float("nan"))
    d["treated"] = d[group_col].map(_is_treated)
    d["post"]    = d[time_col].map(_norm_time_label).eq("post").astype(int)
    try:
        model = smf.ols(f"{metric} ~ treated + post + treated:post", data=d).fit(cov_type="HC3")
        coef = model.params.get("treated:post", float("nan"))
        pval = model.pvalues.get("treated:post", float("nan"))
        return (float(coef), float(pval))
    except Exception:
        return (float("nan"), float("nan"))

def _did_ols_pvalue_cluster(df_area, metric, group_col="group", time_col="time", cluster_col=None):
    
    if smf is None or cluster_col is None or cluster_col not in df_area.columns:
        return (float("nan"), float("nan"))
    d = df_area[[metric, group_col, time_col, cluster_col]].dropna().copy()
    if d.empty: 
        return (float("nan"), float("nan"))
    d["treated"] = d[group_col].map(_is_treated)
    d["post"]    = d[time_col].map(_norm_time_label).eq("post").astype(int)
    try:
        model = smf.ols(f"{metric} ~ treated + post + treated:post", data=d).fit(
            cov_type="cluster", cov_kwds={"groups": d[cluster_col]}
        )
        coef = model.params.get("treated:post", float("nan"))
        pval = model.pvalues.get("treated:post", float("nan"))
        return (float(coef), float(pval))
    except Exception:
        return (float("nan"), float("nan"))






SCHOOL_COLS: list[str] = [chr(c) for c in range(ord("J"), ord("X")+1)]  
BAN_TERMS_DEFAULT  = {"amo","tv","ex"}  

_MOJI_MAP = {"‚Äô":"’","‚Äù":"”","‚Äú":"“","â€™":"’","â€œ":"“","â€\x9d":"”","â€“":"–","â€”":"—","√®":"è","√†":"à","√≤":"ò","√©":"é","√™":"™"}
def fix_mojibake_text(s: str) -> str:
    if not isinstance(s, str):
        return s
    try:
        import ftfy as _f
        return _f.fix_text(s)
    except Exception:
        pass
    for k, v in _MOJI_MAP.items():
        if k in s: s = s.replace(k, v)
    return s


_DEF_A2I = {chr(i+65): i for i in range(26)}
def excel_col_to_index(col: str) -> int:
    col = col.strip().upper()
    idx = 0
    for ch in col:
        if 'A' <= ch <= 'Z':
            idx = idx * 26 + (_DEF_A2I[ch] + 1)
    return idx - 1


def load_questionnaire_lettered(path: str) -> tuple[pd.DataFrame, dict]:
    df = pd.read_csv(
        path,
        header=None,
        dtype=str,
        sep=None,             
        engine="python",
        keep_default_na=False 
    )
    
    def excel_col(i: int) -> str:
        out = ""
        i0 = i
        while True:
            i0, r = divmod(i0, 26)
            out = chr(r + 65) + out
            if i0 == 0:
                break
            i0 -= 1
        return out
    df.columns = [excel_col(i) for i in range(df.shape[1])]
    
    desc = df.iloc[1]
    desc = desc.apply(fix_mojibake_text)
    school_map = {c: desc[c] for c in SCHOOL_COLS if c in df.columns}
    
    df = df.apply(lambda col: col.map(fix_mojibake_text))
    return df, school_map

def ensure_dir(p: str):
    pathlib.Path(p).mkdir(parents=True, exist_ok=True)

def write_text(path: str, content: str):
    with open(path, "w", encoding="utf-8") as f:
        f.write(content)

def safe_write_csv(df: pd.DataFrame, path: str, columns: Optional[List[str]] = None):
    if df is None:
        df = pd.DataFrame(columns=columns or [])
    elif df.empty and columns is not None:
        df = pd.DataFrame(columns=columns)
    df.to_csv(path, index=False, encoding="utf-8")


_IT_STOP_BASE = set("""
 a ad al allo alla ai agli alle all' allo' anche come con col coi cosa così cosi da dal dallo dalla dai dagli dalle dal' del dello della dei degli delle di in ne nei negli nelle nell' no non o od oppure per però pero più piu poiché poiche quale quali che chi cui se si sia siano suo sua suoi sue sul sullo sulla sui sugli sulle tra fra un una uno gli il la lo i le degli delle dei
""".split())
_word_re = re.compile(r"[A-Za-zÀ-ÖØ-öø-ÿ']+")
def build_tokenizer(stop_extra: set, keep_terms: set, ban_terms: set):
    stop = (_IT_STOP_BASE | set(stop_extra)) - set(keep_terms)
    ban_terms = set(ban_terms or [])
    def _tok(text: str) -> List[str]:
        text = fix_mojibake_text(text or "")
        tokens = [t.lower() for t in _word_re.findall(text)]
        return [t for t in tokens if (len(t) > 1 and (t in keep_terms or (t not in stop and t not in ban_terms)))]  
    return _tok



_SPACY_IT = None
def _init_spacy_it():
    global _SPACY_IT
    if _SPACY_IT is not None:
        return _SPACY_IT
    try:
        import spacy  
        
        for name in ("it_core_news_sm", "it_core_news_md", "it_core_news_lg"):
            try:
                _SPACY_IT = spacy.load(name)
                break
            except Exception:
                continue
        if _SPACY_IT is None:
            _SPACY_IT = "TOKENIZER_ONLY"
    except Exception:
        _SPACY_IT = None
    return _SPACY_IT

def build_bow_tokenizer(stop_extra: set, keep_terms: set, ban_terms: set, lemmatize: bool = True):
    
    stop = (_IT_STOP_BASE | set(stop_extra)) - set(keep_terms)
    ban_terms = set(ban_terms or [])
    nlp = _init_spacy_it() if lemmatize else None

    def _tok_bow(text: str):
        text = fix_mojibake_text(text or "")
        if nlp and nlp not in (None, "TOKENIZER_ONLY"):
            doc = nlp(text)
            toks = [(t.lemma_ or t.text).lower() for t in doc if t.is_alpha]
        else:
            toks = [t.lower() for t in _word_re.findall(text)]
        out = []
        for t in toks:
            if len(t) <= 1: 
                continue
            if t in keep_terms:
                out.append(t); continue
            if t in ban_terms: 
                continue
            if t in stop: 
                continue
            out.append(t)
        return out
    return _tok_bow



def derive_stopwords_from_corpus(texts: List[str], keep_terms: set, min_doc_frac: float = 0.35, min_tf_short: int = 10) -> set:
    from collections import Counter
    tf = Counter(); df = Counter()
    N = max(1, len(texts))
    for s in texts:
        toks = [w.lower() for w in _word_re.findall(s or "")]
        toks_set = set(toks)
        for w in toks: tf[w]+=1
        for w in toks_set: df[w]+=1
    out = set()
    for w in list(tf.keys()):
        if w in keep_terms: continue
        if w.isdigit(): continue
        dfr = df[w]/N
        if dfr >= min_doc_frac or (len(w)<=3 and tf[w]>=min_tf_short):
            out.add(w)
    return out




_POS = set(" bene buono ottimo positivo felice contento soddisfatto sereno tranquillo migliorato riuscito efficace utili ".split())
_NEG = set(" male cattivo pessimo negativo triste arrabbiato frustrato ansioso stressato difficile noioso fallito inutile ".split())

def sentiment_baseline(texts: List[str], tokenizer) -> pd.Series:
    vals = []
    for t in texts:
        toks = tokenizer(t)
        pos = sum(1 for w in toks if w in _POS)
        neg = sum(1 for w in toks if w in _NEG)
        score = (pos - neg) / (len(toks) + 1e-9)
        vals.append(score)
    return pd.Series(vals)

def ttr_simple(text: str, tokenizer) -> float:
    toks = tokenizer(text)
    return len(set(toks)) / max(1, len(toks))

def mtld_std(text: str, tokenizer, ttr_threshold: float = 0.72) -> float:
    toks = tokenizer(text)
    if len(toks) < 50:
        return float('nan')
    factors = 0
    start = 0
    types = set()
    for i, tok in enumerate(toks):
        types.add(tok)
        span = i - start + 1
        if len(types) / span < ttr_threshold:
            factors += 1
            start = i + 1
            types = set()
    if i - start + 1 > 0:
        denom = (i - start + 1)
        val = len(types) / denom if denom else 0
        factors += (1 - (val) / ttr_threshold)
    if factors == 0:
        return float('nan')
    return len(toks) / factors

def sentiment_from_json(texts, tokenizer, cfg):
    
    import pandas as pd

    lx = {k.lower(): float(v) for k, v in cfg["sentiment"].get("lexicon", {}).items()}
    negators = set(t.lower() for t in cfg["sentiment"].get("negators", []))
    boosters = {k.lower(): float(v) for k, v in cfg["sentiment"].get("boosters", {}).items()}
    window = int(cfg["sentiment"].get("window", 3))
    max_token_contrib = float(cfg["sentiment"].get("max_token_contrib", 3.0))

    
    aro_lex = {k.lower(): float(v) for k, v in cfg["sentiment"].get("arousal_lexicon", {}).items()}
    dom_lex = {k.lower(): float(v) for k, v in cfg["sentiment"].get("dominance_lexicon", {}).items()}
    

    vals = []
    for s in texts:
        toks = tokenizer(str(s))  
        total = 0.0

        for i, w in enumerate(toks):
            base = float(lx.get(w, 0.0))
            if base == 0.0:
                continue

            
            left = max(0, i - window)
            right = min(len(toks), i + 1 + window)
            neigh = toks[left:i] + toks[i+1:right]

            
            if (w in negators) or any(n in negators for n in neigh):
                base = -base

            
            boost = 1.0
            for b in neigh:
                if b in boosters:
                    try:
                        boost *= float(boosters[b])
                    except Exception:
                        pass

            contrib = base * boost

            
            if contrib > 0:
                contrib = min(contrib, max_token_contrib)
            else:
                contrib = max(contrib, -max_token_contrib)

            total += contrib

        
        denom = (len(toks) + 1e-9)
        vals.append(total / denom)

    return pd.Series(vals)


def _weights_lookup(weights: Dict[str, float], token: str, default: float = 1.0) -> float:
    try:
        return float(weights.get(token, default))
    except Exception:
        return default

def _weighted_ttr(tokens: List[str], weights: Dict[str, float]) -> float:
    if not tokens:
        return float('nan')
    denom = sum(_weights_lookup(weights, t) for t in tokens) + 1e-12
    types = set(tokens)
    numer = sum(_weights_lookup(weights, t) for t in types)
    return numer / denom

def ttr_from_json(text: str, tokenizer, cfg: dict) -> float:
    method = cfg["ttr"].get("method","msttr").lower()
    win = int(cfg["ttr"].get("window", 10))
    min_tokens = int(cfg["ttr"].get("min_tokens", 5))
    weights = cfg["ttr"].get("weights", {})  
    toks = tokenizer(text)
    if len(toks) < min_tokens:
        return float('nan')
    if method == "msttr" and len(toks) >= win and win > 0:
        vals = []
        for i in range(0, len(toks) - win + 1):
            span = toks[i:i+win]
            vals.append(_weighted_ttr(span, weights))
        return float(np.mean(vals)) if vals else _weighted_ttr(toks, weights)
    else:
        return _weighted_ttr(toks, weights)

def mtld_from_json(text: str, tokenizer, cfg: dict) -> float:
    thr = float(cfg["mtld"].get("threshold", 0.72))
    min_tokens = int(cfg["mtld"].get("min_tokens", 30))
    fb = cfg["mtld"].get("short_text_fallback", "cttr").lower()
    k = float(cfg["mtld"].get("cttr_k", 2.0))
    weights = cfg["mtld"].get("weights", {})
    toks = tokenizer(text)
    n_tokens = len(toks)
    if n_tokens < min_tokens:
        if fb == "cttr":
            types = set(toks)
            types_w = sum(_weights_lookup(weights, t) for t in types)
            return types_w / math.sqrt(k * max(1, n_tokens))
        else:
            return float('nan')
    factors = 0.0
    types = set()
    types_w = 0.0
    seg_token_w = 0.0
    for i, tok in enumerate(toks):
        w = _weights_lookup(weights, tok)
        seg_token_w += w
        if tok not in types:
            types.add(tok)
            types_w += _weights_lookup(weights, tok)
        ratio = types_w / max(1e-9, seg_token_w)
        if ratio < thr:
            factors += 1.0
            types.clear()
            types_w = 0.0
            seg_token_w = 0.0
    if seg_token_w > 0.0:
        ratio = types_w / max(1e-9, seg_token_w)
        factors += (1.0 - (ratio / thr))
    if factors <= 0:
        return float('nan')
    return n_tokens / factors





SPEC_TO_AREA_DEFAULT = {
    "Cittadinanza Digitale": "Cittadinanza Digitale",
    "Valori e atteggiamenti sociali": "Valori e atteggiamenti sociali",
    "Abilità e impegno civico": "Abilità e impegno civico",
}


OPEN_QS = [
    
    {"qid":"SC_A1_BZ", "title":"Cittadinanza Digitale – riflessione", 
     "excel_cols":["BZ"], "competences":["Cittadinanza Digitale"]},

    
    {"qid":"SC_A2_FL", "title":"Valori e atteggiamenti sociali – riflessione (FL)", 
     "excel_cols":["FL"], "competences":["Valori e atteggiamenti sociali"]},
    {"qid":"SC_A2_GX", "title":"Valori e atteggiamenti sociali – riflessione (GX)", 
     "excel_cols":["GX"], "competences":["Valori e atteggiamenti sociali"]},
    {"qid":"SC_A2_NU", "title":"Valori e atteggiamenti sociali – riflessione (NU)", 
     "excel_cols":["NU"], "competences":["Valori e atteggiamenti sociali"]},

    
    {"qid":"SC_A3_HI", "title":"Abilità e impegno civico – riflessione (HI)", 
     "excel_cols":["HI"], "competences":["Abilità e impegno civico"]},
    {"qid":"SC_A3_LI", "title":"Abilità e impegno civico – riflessione (LI)", 
     "excel_cols":["LI"], "competences":["Abilità e impegno civico"]},
    {"qid":"SC_A3_ME", "title":"Abilità e impegno civico – riflessione (ME)", 
     "excel_cols":["ME"], "competences":["Abilità e impegno civico"]},
]




def build_tables_with_uid_and_group(path_csv: str, time_label: str) -> pd.DataFrame:
    df, school_map = load_questionnaire_lettered(path_csv)
    def first_school(row):
        for c in SCHOOL_COLS:
            if c not in row: 
                continue
            val = row[c]
            if isinstance(val, str) and val.strip() != "" and val.strip().lower() != "nan":
                return school_map.get(c, "SCONOSCIUTA")
        return "SCONOSCIUTA"
    df_data = df.iloc[2:].reset_index(drop=True).copy()
    school = df_data.apply(first_school, axis=1)
    AY_i = excel_col_to_index("AY")
    AW_i = excel_col_to_index("AW")
    AX_i = excel_col_to_index("AX")

    AY = (df_data.iloc[:, AY_i] if AY_i < df_data.shape[1] else pd.Series([np.nan]*len(df_data))).astype(str).str.strip()
    aw = (df_data.iloc[:, AW_i] if AW_i < df_data.shape[1] else pd.Series([np.nan]*len(df_data))).astype(str).str.strip().str.lower()
    ax = (df_data.iloc[:, AX_i] if AX_i < df_data.shape[1] else pd.Series([np.nan]*len(df_data))).astype(str).str.strip().str.lower()
    is_etw = aw.isin({"si","sì","sí"})
    is_ctrl = ax.eq("no")
    group = pd.Series(index=df_data.index, dtype="object")
    group[is_etw] = "etwinning"
    group[is_ctrl] = "controllo"
    uid = school + "_" + AY

    rows = []
    for q in OPEN_QS:
        cols = q["excel_cols"]
        idxs = [excel_col_to_index(c) for c in cols]
        usable = [df.columns[i] for i in idxs if 0 <= i < df.shape[1]]
        sub = df.iloc[2:][usable].reset_index(drop=True) if usable else pd.DataFrame()
        if sub.empty:
            continue
        texts = sub.apply(lambda r: "; ".join([x for x in r.astype(str).tolist() if str(x).strip() not in ("","nan")]), axis=1)
        for i, t in enumerate(texts.tolist()):
            t = (t or "").strip()
            if not t: continue
            for comp in q["competences"]:
                rows.append({
                    "UID": uid.iloc[i],
                    "group": group.iloc[i],
                    "qid": q["qid"],
                    "question": q["title"],
                    "competenza": comp,
                    "text": fix_mojibake_text(t),
                    "time": time_label
                })
    out = pd.DataFrame(rows)
    return out




def _save_top_bar(freq_dict: dict, path_png: str, title: str):
    
    items = sorted(freq_dict.items(), key=lambda x: x[1], reverse=True)[:30]
    if not items:
        plt.figure(figsize=(8,5)); plt.text(0.5,0.5,"Dati insufficienti", ha="center", va="center")
        plt.axis("off"); plt.savefig(path_png, dpi=140); plt.close(); return
    labels = [w for w,_ in items]; values = [v for _,v in items]
    plt.figure(figsize=(10,6))
    plt.barh(range(len(labels)), values)
    plt.yticks(range(len(labels)), labels)
    plt.gca().invert_yaxis()
    plt.title(title); plt.tight_layout()
    plt.savefig(path_png, dpi=140); plt.close()

def build_wordclouds(df: pd.DataFrame, tok, figs_dir: str):

    
    n = 0
    from collections import Counter
    for area, sub in df.groupby("area"):
        for when in ["pre","post"]:
            docs = sub[sub["time"]==when]["text"].astype(str).tolist()
            freq = Counter()
            for dtext in docs:
                for t in tok(dtext): freq[t]+=1
            base = f"{area}_{when}".replace(" ", "_")
            wc_path = os.path.join(figs_dir, f"wordcloud_{base}.png")
            if WordCloud is not None and len(freq)>0:
                try:
                    wc = WordCloud(width=1400, height=900, background_color="white")
                    wc.generate_from_frequencies(freq)
                    wc.to_file(wc_path); n+=1
                except Exception:
                    
                    _save_top_bar(freq, wc_path, f"Top termini – {area} · {when}")
                    n+=1
            else:
                
                _save_top_bar(freq, wc_path, f"Top termini – {area} · {when}")
                n+=1
    return n

def key_terms_and_collocations(df, tok, ban_terms, outd):
    made = {"key_terms":0, "colloc_bi":0, "colloc_tri":0}
    for area, sub in df.groupby("area"):
        for when in ["pre","post"]:
            docs_etw = sub[(sub["time"]==when) & (sub["group"]=="etwinning")]["text"].astype(str).tolist()
            docs_ctrl= sub[(sub["time"]==when) & (sub["group"]=="controllo")]["text"].astype(str).tolist()
            
            v = CountVectorizer(tokenizer=tok, min_df=1, token_pattern=None)
            groups = {"etwinning": docs_etw, "controllo": docs_ctrl}
            vocab_counts = {}; group_counts = {}
            for g, docs in groups.items():
                if not docs: continue
                X = v.fit_transform(docs)
                counts = np.asarray(X.sum(axis=0)).ravel()
                vocab = np.array(v.get_feature_names_out())
                dvc = pd.Series(counts, index=vocab)
                group_counts[g] = dvc
                vocab_counts.setdefault("_all", pd.Series(dtype=np.int64))
                vocab_counts["_all"] = vocab_counts["_all"].add(dvc, fill_value=0)
            cols = ["term","group","log_odds","count","area","time"]
            out_parts = []
            if group_counts:
                alpha0 = vocab_counts["_all"] + 1.0
                alpha0 = alpha0 / alpha0.sum()
                for g, dvc in group_counts.items():
                    other = sum((group_counts[k] for k in group_counts if k != g), pd.Series(dtype=np.int64))
                    other = other.reindex(alpha0.index).fillna(0)
                    dvc = dvc.reindex(alpha0.index).fillna(0)
                    alpha = alpha0 * 1000.0
                    num = dvc + alpha
                    den = other + alpha
                    log_odds = np.log((num / num.sum()) / (den / den.sum()))
                    dfk = pd.DataFrame({"term": alpha0.index, "group": g, "log_odds": log_odds.values, "count": dvc.values})
                    dfk = dfk[dfk["count"] >= 3].sort_values(["count","log_odds"], ascending=[False, False])
                    if not dfk.empty:
                        dfk["area"]=area; dfk["time"]=when
                        out_parts.append(dfk)
            if out_parts:
                out = pd.concat(out_parts, ignore_index=True)
            else:
                out = pd.DataFrame(columns=cols)
            path = os.path.join(outd, f"key_terms_{area}_{when}.csv")
            safe_write_csv(out, path, columns=cols)
            made["key_terms"] += 1

            
            docs = docs_etw + docs_ctrl
            tokens = []
            for dtx in docs:
                tokens.extend(tok(dtx))
            from collections import Counter
            uni = Counter(tokens)
            def colloc(n, min_freq):
                rows = []
                if n==2:
                    pairs = list(zip(tokens, tokens[1:]))
                    ngrams = Counter(pairs)
                else:
                    trip = list(zip(tokens, tokens[1:], tokens[2:]))
                    ngrams = Counter(trip)
                total = sum(uni.values())
                for ng, f in ngrams.items():
                    if f < min_freq: continue
                    if n==2:
                        a,b = ng
                        p = f / (total - 1 if total > 1 else 1)
                        p_a = uni[a]/total if total else 0
                        p_b = uni[b]/total if total else 0
                        denom = (p_a*p_b) if (p_a and p_b) else 1e-12
                        pmi = math.log2(p/denom + 1e-12)
                    else:
                        a,b,c = ng
                        p = f / (total - 2 if total > 2 else 1)
                        p_a = uni[a]/total if total else 0
                        p_b = uni[b]/total if total else 0
                        p_c = uni[c]/total if total else 0
                        denom = (p_a*p_b*p_c) if (p_a and p_b and p_c) else 1e-18
                        pmi = math.log2(p/denom + 1e-12)
                    rows.append({"ngram":" ".join(ng), "freq":f, "PMI":pmi})
                cols2=["ngram","freq","PMI"]
                return pd.DataFrame(rows).sort_values(["freq","PMI"], ascending=[False, False])[cols2]
            bi = colloc(2, 5)
            tri = colloc(3, 3)
            path_bi = os.path.join(outd, f"collocations_{area}_{when}_bi.csv")
            path_tri = os.path.join(outd, f"collocations_{area}_{when}_tri.csv")
            safe_write_csv(bi, path_bi, columns=["ngram","freq","PMI"])
            safe_write_csv(tri, path_tri, columns=["ngram","freq","PMI"])
            made["colloc_bi"] += 1; made["colloc_tri"] += 1
    return made

def topic_model_area(df_area: pd.DataFrame, tok, n_topics: int = None, max_features: int = 4000):
    docs = df_area["text"].astype(str).tolist()
    if len(docs) < 10:
        return None
    cv = CountVectorizer(tokenizer=tok, min_df=2, max_df=0.9, max_features=max_features, token_pattern=None)
    X = cv.fit_transform(docs)
    if X.shape[0] < 10 or X.shape[1] < 10:
        return None
    if n_topics is None:
        n_topics = min(max(4, X.shape[0] // 40), 10)
    lda = LatentDirichletAllocation(n_components=n_topics, learning_method="batch", random_state=42)
    W = lda.fit_transform(X)  
    H = lda.components_       
    vocab = np.array(cv.get_feature_names_out())
    return {"lda": lda, "W": W, "H": H, "vocab": vocab, "vectorizer": cv}

def export_topics(area: str, model: dict, df_area: pd.DataFrame, outd: str):
    lda, W, H, vocab = model["lda"], model["W"], model["H"], model["vocab"]
    rows = []
    for k in range(lda.n_components):
        comp = H[k] / (H[k].sum() + 1e-9)
        top_idx = np.argsort(comp)[::-1][:20]
        for rank, j in enumerate(top_idx, start=1):
            rows.append({"area":area, "topic":k, "rank":rank, "term": vocab[j], "weight": float(comp[j])})
    topics_terms = pd.DataFrame(rows)
    safe_write_csv(topics_terms, os.path.join(outd, f"topics_terms_area_{area}.csv"), columns=["area","topic","rank","term","weight"])
    df = df_area.reset_index(drop=True).copy()
    if "group" not in df.columns: df["group"] = None
    if "time" not in df.columns: df["time"] = None
    main_topic = np.argmax(W, axis=1)
    df["_topic"] = main_topic
    prev_rows = []
    for (g,t), sub in df.groupby(["group","time"], dropna=False):
        n = sub.shape[0]
        for k in range(lda.n_components):
            nk = int((sub["_topic"]==k).sum())
            prop = nk / n if n else 0.0
            prev_rows.append({"area":area,"group":g,"time":t,"topic":k,"n":nk,"tot":n,"prop":prop})
    topics_prev = pd.DataFrame(prev_rows)
    prev_path = os.path.join(outd, "topics_prevalence_area_group.csv")
    if os.path.exists(prev_path):
        old = pd.read_csv(prev_path)
        topics_prev = pd.concat([old, topics_prev], ignore_index=True).drop_duplicates()
    safe_write_csv(topics_prev, prev_path, columns=["area","group","time","topic","n","tot","prop"])
    return topics_prev


def export_top_responses_by_question(df_rows, cfg, out_csv_path, tokenizer=_simple_tokenize):
    
    qcfg = cfg.get("questions", {}) if isinstance(cfg, dict) else {}
    qid_col  = qcfg.get("question_id_col", "question_id")
    qtxt_col = qcfg.get("question_text_col", "question")
    resp_col = qcfg.get("response_text_col", "text")
    top_k    = int(qcfg.get("top_k", 20))
    do_norm  = bool(qcfg.get("normalize_responses", True))

    
    for col in (qid_col, qtxt_col, resp_col):
        if col not in df_rows.columns:
            
            import pandas as pd
            pd.DataFrame(columns=["question_id","question","response_norm","count","wordcount","rank"])              .to_csv(out_csv_path, index=False)
            return

    def _normalize_resp(s: str) -> str:
        s = str(s).strip()
        if not do_norm:
            return s
        
        s = " ".join(s.lower().split())
        
        return s

    rows = []
    
    questions_order = list(dict.fromkeys(df_rows[qid_col].tolist()))
    for qid in questions_order:
        sub = df_rows[df_rows[qid_col] == qid]
        qtext = sub[qtxt_col].iloc[0] if not sub.empty else ""
        counts = Counter(_normalize_resp(s) for s in sub[resp_col].astype(str).tolist() if str(s).strip() != "")
        if not counts:
            continue
        
        top = counts.most_common(top_k)
        rank = 1
        for resp, cnt in top:
            wc = len(tokenizer(resp))
            rows.append({
                "question_id": qid,
                "question": qtext,
                "response_norm": resp,
                "count": int(cnt),
                "wordcount": int(wc),
                "rank": rank
            })
            rank += 1

    import pandas as pd
    out = pd.DataFrame(rows, columns=["question_id","question","response_norm","count","wordcount","rank"])
    
    
    order_map = {qid:i for i, qid in enumerate(questions_order)}
    out["_qorder"] = out["question_id"].map(order_map)
    out = out.sort_values(by=["_qorder","rank"]).drop(columns=["_qorder"])
    out.to_csv(out_csv_path, index=False)


def export_area_group_pvalues(df_rows, out_csv_path, area_col="area", group_col="group", time_col="time",
                              metrics=("sentiment","mtld")):
    
    import pandas as pd
    rows = []
    d = df_rows.copy()
    d[time_col] = d[time_col].map(_norm_time_label)
    for (a,g), sub in d.groupby([area_col, group_col], dropna=False):
        for m in metrics:
            if m not in sub.columns: continue
            x_pre  = sub.loc[sub[time_col]=="pre",  m].astype(float)
            x_post = sub.loc[sub[time_col]=="post", m].astype(float)
            p = _welch_t_pvalue(x_pre, x_post)
            rows.append({"area":a, "group":g, "metric":m, "p_prepost":p})
    pd.DataFrame(rows).to_csv(out_csv_path, index=False)

def export_area_pvalues(df_rows, out_csv_path, area_col="area", time_col="time",
                        metrics=("sentiment","mtld")):
    
    import pandas as pd
    rows = []
    d = df_rows.copy()
    d[time_col] = d[time_col].map(_norm_time_label)
    for a, sub in d.groupby(area_col, dropna=False):
        for m in metrics:
            if m not in sub.columns: continue
            x_pre  = sub.loc[sub[time_col]=="pre",  m].astype(float)
            x_post = sub.loc[sub[time_col]=="post", m].astype(float)
            p = _welch_t_pvalue(x_pre, x_post)
            rows.append({"area":a, "metric":m, "p_prepost_allgroups":p})
    pd.DataFrame(rows).to_csv(out_csv_path, index=False)

def export_did_with_pvalues(df_rows, out_csv_path, area_col="area", group_col="group", time_col="time",
                            metrics=("sentiment","mtld")):
    
    import pandas as pd
    d = df_rows.copy()
    d[time_col] = d[time_col].map(_norm_time_label)
    rows = []
    for a, sub in d.groupby(area_col, dropna=False):
        for m in metrics:
            if m not in sub.columns: continue
            
            def _delta_for(is_treated):
                mask = sub[group_col].map(_is_treated) == int(is_treated)
                kk = sub[mask]
                pre  = kk.loc[kk[time_col]=="pre",  m].astype(float)
                post = kk.loc[kk[time_col]=="post", m].astype(float)
                if len(pre)==0 or len(post)==0:
                    return float("nan")
                return float(post.mean() - pre.mean())
            de = _delta_for(True)
            dc = _delta_for(False)
            did = (de - dc) if (not math.isnan(de) and not math.isnan(dc)) else float("nan")
            coef, p = _did_ols_pvalue(sub[[m, group_col, time_col]], m, group_col, time_col)
            coef_s, p_s = _did_ols_pvalue_cluster(sub, m, group_col, time_col, cluster_col='school')
            coef_c, p_c = _did_ols_pvalue_cluster(sub, m, group_col, time_col, cluster_col='class')
            rows.append({"area":a, "metric":m, "did":did, "p_did":p, "p_did_cluster_school":p_s, "p_did_cluster_class":p_c,
                         "delta_etwinning":de, "delta_controllo":dc})
    pd.DataFrame(rows).to_csv(out_csv_path, index=False)

def export_quadrant_table(df_rows, out_csv_path, area_col="area", group_col="group", time_col="time", quad_col="quadrant"):
    
    import pandas as pd
    if quad_col not in df_rows.columns:
        pd.DataFrame(columns=[area_col,group_col,time_col]).to_csv(out_csv_path, index=False)
        return
    d = df_rows.copy()
    d[time_col] = d[time_col].map(_norm_time_label)
    counts = d.groupby([area_col,group_col,time_col,quad_col]).size().reset_index(name="n")
    totals = counts.groupby([area_col,group_col,time_col])["n"].transform("sum")
    counts["pct"] = 100.0 * counts["n"] / totals
    
    wide = counts.pivot_table(index=[area_col,group_col,time_col], columns=quad_col, values="pct", fill_value=0.0)
    wide = wide.reset_index()
    wide.columns.name = None
    wide.to_csv(out_csv_path, index=False)




def _norm_area_key_for_overrides(s: str) -> str:
    return re.sub(r"\s+", " ", str(s).strip().lower())



TOPIC_LABEL_OVERRIDES_NORM: Dict[str, Dict[int, str]] = {}

def _register_topic_overrides(area_aliases, mapping):
    for a in area_aliases:
        TOPIC_LABEL_OVERRIDES_NORM[_norm_area_key_for_overrides(a)] = dict(mapping)


_register_topic_overrides(
    [
        "2.1 Values and social attitudes (Valori e atteggiamenti sociali)",
        "2.1 Valori e atteggiamenti sociali",
        "Valori e atteggiamenti sociali",
        "Values and social attitudes",
        "Values and social attitudes (Valori e atteggiamenti sociali)",
    ],
    {
        0: "Global issues & sustainability (war, pollution, change)",
        3: "Rights & equality (equal opportunities, education)",
        9: "Human dignity & respect",
        6: "Respecting opinions & civic duty",
        2: "Diversity & viewpoints",
        4: "Rights & dignity (principles-based)",
        1: "Gender & racism (bias/discrimination)",
        7: "Respect for others (general)",
        5: "Sustainability, energy & technology",
        8: "Society & technology (improvement / collective future)",
    }
)


_register_topic_overrides(
    [
        "2.2 Skills and civic engagement (Abilità e impegno civico)",
        "2.2 Abilità e impegno civico",
        "Abilità e impegno civico",
        "Skills and civic engagement",
        "Skills and civic engagement (Abilità e impegno civico)",
    ],
    {
        2: "Working alone vs group preference",
        0: "Helping others & listening to different opinions",
        8: "Duty and participation in group work",
        3: "Helping / perspective-taking",
        6: "Participating in projects and civic experiences",
        4: "Positive group climate & belonging",
        7: "Group dynamics & interdependence",
        5: "Supporting peers / relationships (feeling bad, being close)",
        9: "Remembering civic/helping experiences",
        1: "School/community environment (general)",
    }
)


_register_topic_overrides(
    [
        "2.3 Digital citizenship (Cittadinanza digitale)",
        "2.3 Cittadinanza digitale",
        "Cittadinanza digitale",
        "Digital citizenship",
        "Digital citizenship (Cittadinanza digitale)",
    ],
    {
        3: "Civic discussion & participation (projects/initiatives)",
        7: "Online experience and responding (confidence/ability)",
        9: "Commenting, videos & online participation frequency",
        6: "Debates online (often/rarely; class discussion frame)",
        4: "Social media as an 'opportunity' (engagement)",
        2: "Petitions, risk perception & civic beliefs",
        5: "School initiatives/events & participation",
        1: "Forums/Reddit and 'benefits' of discussion",
        8: "Signing petitions (with friends/others)",
        0: "Protests/petitions/bullying and media (generic civic action)",
    }
)

def _build_topic_legend_from_terms(outd: str, top_n_terms: int = 8) -> pd.DataFrame:
    
    term_files = sorted(pathlib.Path(outd).glob("topics_terms_area_*.csv"))
    legend_rows = []
    for fp in term_files:
        try:
            tdf = pd.read_csv(fp, dtype={"area": str, "topic": int, "rank": int, "term": str})
        except Exception:
            continue
        required = {"area", "topic", "rank", "term"}
        if not required.issubset(set(tdf.columns)):
            continue

        tdf = tdf.sort_values(["area", "topic", "rank"], kind="mergesort")
        
        top = tdf[tdf["rank"].between(1, top_n_terms, inclusive="both")].copy()

        def _uniq_terms(s: pd.Series) -> str:
            seen = set()
            out = []
            for x in s.tolist():
                if x is None:
                    continue
                tx = str(x).strip()
                if not tx or tx.lower() == "nan":
                    continue
                if tx in seen:
                    continue
                seen.add(tx)
                out.append(tx)
            return " / ".join(out)

        g = (
            top.groupby(["area", "topic"], dropna=False)["term"]
            .apply(_uniq_terms)
            .reset_index(name="top_terms")
        )
        g["topic_label"] = g["top_terms"]
        
        try:
            def _override_label(area_val, topic_val, default):
                akey = _norm_area_key_for_overrides(area_val)
                try:
                    tkey = int(topic_val)
                except Exception:
                    return default
                amap = TOPIC_LABEL_OVERRIDES_NORM.get(akey)
                if not amap:
                    return default
                return amap.get(tkey, default)
            g["topic_label"] = g.apply(lambda r: _override_label(r["area"], r["topic"], r["topic_label"]), axis=1)
        except Exception:
            pass
        g["topic_name"] = g.apply(lambda r: f"T{int(r['topic'])}: {r['topic_label']}", axis=1)
        legend_rows.append(g[["area", "topic", "topic_label", "topic_name", "top_terms"]])

    if legend_rows:
        legend = (
            pd.concat(legend_rows, ignore_index=True)
            .drop_duplicates(subset=["area", "topic"], keep="first")
            .sort_values(["area", "topic"], kind="mergesort")
            .reset_index(drop=True)
        )
    else:
        legend = pd.DataFrame(columns=["area", "topic", "topic_label", "topic_name", "top_terms"])

    
    safe_write_csv(
        legend[["area", "topic", "topic_name", "top_terms"]],
        os.path.join(outd, "topics_legend_area.csv"),
        columns=["area", "topic", "topic_name", "top_terms"],
    )
    return legend


def _attach_topic_labels(df: pd.DataFrame, legend: pd.DataFrame) -> pd.DataFrame:
    
    if df is None or df.empty:
        return df
    if legend is None or legend.empty:
        return df
    if "area" not in df.columns or "topic" not in df.columns:
        return df
    
    try:
        df2 = df.copy()
        df2["topic"] = pd.to_numeric(df2["topic"], errors="coerce").astype("Int64")
        leg = legend.copy()
        leg["topic"] = pd.to_numeric(leg["topic"], errors="coerce").astype("Int64")
        merged = df2.merge(
            leg[["area", "topic", "topic_label", "topic_name"]],
            on=["area", "topic"],
            how="left",
        )
        
        if "topic" in merged.columns:
            merged["topic"] = merged["topic"].astype("Int64")
        return merged
    except Exception:
        return df


def _reorder_topic_label_cols(df: pd.DataFrame) -> pd.DataFrame:
    
    if df is None or df.empty:
        return df
    cols = list(df.columns)
    if "topic" not in cols:
        return df
    for c in ["topic_name", "topic_label"]:
        if c not in cols:
            continue
        cols.remove(c)
    
    tpos = cols.index("topic") + 1
    new_cols = cols[:tpos] + [c for c in ["topic_name", "topic_label"] if c in df.columns] + cols[tpos:]
    
    new_cols = [c for c in new_cols if c in df.columns]
    return df[new_cols]


def _rewrite_all_topic_exports_with_labels(outd: str, legend: pd.DataFrame, skip: Optional[set] = None):
    
    skip = skip or set()
    for fp in sorted(pathlib.Path(outd).glob("*.csv")):
        if fp.name in skip:
            continue
        try:
            df = pd.read_csv(fp)
        except Exception:
            continue
        if ("topic" in df.columns) and ("area" in df.columns):
            df2 = _attach_topic_labels(df, legend)
            df2 = _reorder_topic_label_cols(df2)
            safe_write_csv(df2, str(fp))


def _export_themes_summary(shifts: pd.DataFrame, outd: str, top_k: int = 5):
    
    if shifts is None or shifts.empty:
        safe_write_csv(
            pd.DataFrame(columns=["area","group","direction","rank","topic","topic_name","pre_prop","post_prop","delta_prop"]),
            os.path.join(outd, "themes_summary_area_group.csv"),
            columns=["area","group","direction","rank","topic","topic_name","pre_prop","post_prop","delta_prop"],
        )
        return

    df = shifts.copy()
    
    df["delta_prop"] = pd.to_numeric(df.get("delta_prop"), errors="coerce")
    df = df.dropna(subset=["delta_prop"])

    out_rows = []
    for (area, group), sub in df.groupby(["area", "group"], dropna=False):
        sub = sub.sort_values("delta_prop", kind="mergesort")
        
        dec = sub.head(top_k).copy()
        for i, row in enumerate(dec.itertuples(index=False), start=1):
            out_rows.append({
                "area": area,
                "group": group,
                "direction": "decrease",
                "rank": i,
                "topic": int(row.topic) if pd.notna(row.topic) else row.topic,
                "topic_name": getattr(row, "topic_name", None),
                "pre_prop": getattr(row, "pre_prop", None),
                "post_prop": getattr(row, "post_prop", None),
                "delta_prop": getattr(row, "delta_prop", None),
            })
        
        inc = sub.sort_values("delta_prop", ascending=False, kind="mergesort").head(top_k).copy()
        for i, row in enumerate(inc.itertuples(index=False), start=1):
            out_rows.append({
                "area": area,
                "group": group,
                "direction": "increase",
                "rank": i,
                "topic": int(row.topic) if pd.notna(row.topic) else row.topic,
                "topic_name": getattr(row, "topic_name", None),
                "pre_prop": getattr(row, "pre_prop", None),
                "post_prop": getattr(row, "post_prop", None),
                "delta_prop": getattr(row, "delta_prop", None),
            })

    out_df = pd.DataFrame(out_rows)
    safe_write_csv(
        out_df,
        os.path.join(outd, "themes_summary_area_group.csv"),
        columns=["area","group","direction","rank","topic","topic_name","pre_prop","post_prop","delta_prop"],
    )


def export_topic_shifts(outd: str, top_n_terms: int = 8, top_k_themes: int = 5):
    p = os.path.join(outd, "topics_prevalence_area_group.csv")
    if not os.path.exists(p):
        return
    prev = pd.read_csv(p, dtype={"area":str,"group":str,"time":str,"topic":int,"n":int,"tot":int,"prop":float})
    rows = []
    for (area, group, topic), sub in prev.groupby(["area","group","topic"]):
        pre = float(sub.loc[sub["time"]=="pre", "prop"].sum()) if (sub["time"]=="pre").any() else float("nan")
        post= float(sub.loc[sub["time"]=="post","prop"].sum()) if (sub["time"]=="post").any() else float("nan")
        d = post - pre if (not math.isnan(pre) and not math.isnan(post)) else float("nan")
        rows.append({"area":area,"group":group,"topic":topic,"pre_prop":pre,"post_prop":post,"delta_prop":d})
    shifts = pd.DataFrame(rows)

    
    legend = _build_topic_legend_from_terms(outd, top_n_terms=top_n_terms)

    
    shifts = _attach_topic_labels(shifts, legend)
    shifts = _reorder_topic_label_cols(shifts)

    safe_write_csv(
        shifts,
        os.path.join(outd, "topics_shift_area_group.csv"),
        columns=["area","group","topic","topic_name","topic_label","pre_prop","post_prop","delta_prop"],
    )

    
    _rewrite_all_topic_exports_with_labels(
        outd,
        legend,
        skip={"topics_shift_area_group.csv", "topics_legend_area.csv", "themes_summary_area_group.csv"},
    )

    
    _export_themes_summary(shifts, outd, top_k=top_k_themes)

    
    try:
        export_topic_graphs(outd, top_k=top_k_themes)
    except Exception as _e_graph:
        print(f"[WARN] Graph export failed: {_e_graph}")

def _safe_filename(s: str) -> str:
    s = str(s)
    s = re.sub(r"[^A-Za-z0-9._-]+", "_", s.strip())
    s = re.sub(r"_+", "_", s)
    return s.strip("_") or "NA"

def _shorten_label(s: str, max_len: int = 90) -> str:
    s = str(s)
    if len(s) <= max_len:
        return s
    return s[: max_len - 1] + "…"

def export_topic_graphs(outd: str, top_k: int = 5, top_n_terms: int = 8):
    
    graphs_dir = os.path.join(outd, "graphs")
    os.makedirs(graphs_dir, exist_ok=True)

    themes_path = os.path.join(outd, "themes_summary_area_group.csv")
    shift_path  = os.path.join(outd, "topics_shift_area_group.csv")

    if not os.path.exists(themes_path) or not os.path.exists(shift_path):
        return

    themes = pd.read_csv(themes_path)
    shifts = pd.read_csv(shift_path)

    
    need_themes = {"area","group","direction","rank","topic","topic_name","pre_prop","post_prop","delta_prop"}
    if not need_themes.issubset(set(themes.columns)):
        return
    need_shift = {"area","group","topic","delta_prop"}
    if not need_shift.issubset(set(shifts.columns)):
        return

    
    if "topic_name" not in shifts.columns or shifts["topic_name"].isna().all():
        legend = _build_topic_legend_from_terms(outd, top_n_terms=top_n_terms)
        shifts = _attach_topic_labels(shifts, legend)

    
    def _to_pct(x):
        try:
            return float(x) * 100.0
        except Exception:
            return float("nan")

    for c in ["pre_prop","post_prop","delta_prop"]:
        if c in themes.columns:
            themes[c+"_pct"] = themes[c].apply(_to_pct)
    if "delta_prop" in shifts.columns:
        shifts["delta_pct"] = shifts["delta_prop"].apply(_to_pct)

    
    areas = [a for a in sorted(themes["area"].dropna().unique())]
    groups = [g for g in sorted(themes["group"].dropna().unique())]

    for area in areas:
        for group in groups:
            sub = themes[(themes["area"]==area) & (themes["group"]==group)].copy()
            if sub.empty:
                continue

            
            sub["direction_order"] = sub["direction"].map({"decrease":0, "increase":1}).fillna(2)
            sub = sub.sort_values(["direction_order","rank"], kind="mergesort")
            sub["topic_name_plot"] = sub["topic_name"].fillna(sub.get("topic_label", "")).fillna(sub["topic"].astype(str))
            sub["topic_name_plot"] = sub["topic_name_plot"].apply(lambda x: _shorten_label(x, 85))

            y = np.arange(len(sub))

            
            fig = plt.figure(figsize=(12, max(5.5, 0.42*len(sub)+1.2)))
            ax = fig.add_subplot(111)
            for i, r in enumerate(sub.itertuples(index=False)):
                x0 = getattr(r, "pre_prop_pct", float("nan"))
                x1 = getattr(r, "post_prop_pct", float("nan"))
                ax.plot([x0, x1], [i, i], linewidth=1.5)
                ax.scatter([x0], [i], s=35, zorder=3, marker="o")
                ax.scatter([x1], [i], s=35, zorder=3, marker="o")
            ax.axvline(0, linewidth=0.8, alpha=0.3)
            ax.set_yticks(y)
            ax.set_yticklabels(list(sub["topic_name_plot"]))
            ax.invert_yaxis()
            ax.set_xlabel("Proportion (%)")
            ax.set_title(f"Topic shifts (pre → post) — Area: {area} — Group: {group}")
            ax.grid(True, axis="x", alpha=0.25)
            fig.tight_layout()
            fn = f"dumbbell_{_safe_filename(area)}_{_safe_filename(group)}.png"
            fig.savefig(os.path.join(graphs_dir, fn), dpi=220)
            plt.close(fig)

            
            fig = plt.figure(figsize=(12, max(5.5, 0.42*len(sub)+1.2)))
            ax = fig.add_subplot(111)
            ax.barh(y, sub["delta_prop_pct"].values)
            ax.axvline(0, linewidth=1.0)
            ax.set_yticks(y)
            ax.set_yticklabels(list(sub["topic_name_plot"]))
            ax.invert_yaxis()
            ax.set_xlabel("Δ proportion (post − pre), percentage points")
            ax.set_title(f"Top theme deltas — Area: {area} — Group: {group}")
            ax.grid(True, axis="x", alpha=0.25)
            fig.tight_layout()
            fn = f"delta_bars_{_safe_filename(area)}_{_safe_filename(group)}.png"
            fig.savefig(os.path.join(graphs_dir, fn), dpi=220)
            plt.close(fig)

    
    for area in sorted(shifts["area"].dropna().unique()):
        s = shifts[shifts["area"]==area].copy()
        if s.empty:
            continue

        
        sel_topics = set(themes.loc[themes["area"]==area, "topic"].dropna().astype(int).tolist())
        if not sel_topics:
            
            tmp = s.copy()
            tmp["absd"] = tmp["delta_pct"].abs()
            sel_topics = set(tmp.sort_values("absd", ascending=False).head(10)["topic"].dropna().astype(int).tolist())

        s = s[s["topic"].isin(list(sel_topics))].copy()
        if s.empty:
            continue

        
        if "topic_name" in s.columns:
            s["topic_name_plot"] = s["topic_name"]
        elif "topic_label" in s.columns:
            s["topic_name_plot"] = s["topic_label"]
        else:
            s["topic_name_plot"] = s["topic"].astype(str)
        s["topic_name_plot"] = s["topic_name_plot"].fillna(s["topic"].astype(str)).apply(lambda x: _shorten_label(x, 85))

        pivot = s.pivot_table(index="topic_name_plot", columns="group", values="delta_pct", aggfunc="mean")
        pivot = pivot.fillna(0.0)

        
        if "ETW" in pivot.columns:
            pivot = pivot.sort_values("ETW", ascending=False)
        else:
            pivot["_absmean"] = pivot.abs().mean(axis=1)
            pivot = pivot.sort_values("_absmean", ascending=False).drop(columns=["_absmean"])

        data = pivot.values
        fig = plt.figure(figsize=(10, max(5.5, 0.40*len(pivot)+1.2)))
        ax = fig.add_subplot(111)
        im = ax.imshow(data, aspect="auto")
        ax.set_yticks(np.arange(len(pivot.index)))
        ax.set_yticklabels(list(pivot.index))
        ax.set_xticks(np.arange(len(pivot.columns)))
        ax.set_xticklabels([str(c) for c in pivot.columns])
        ax.set_title(f"Δ topic proportions heatmap — Area: {area}")
        ax.set_xlabel("Group")
        ax.set_ylabel("Topic")

        
        for i in range(data.shape[0]):
            for j in range(data.shape[1]):
                ax.text(j, i, f"{data[i,j]:.1f}", ha="center", va="center", fontsize=8)

        cbar = fig.colorbar(im, ax=ax)
        cbar.set_label("Δ percentage points")

        fig.tight_layout()
        fn = f"heatmap_delta_{_safe_filename(area)}.png"
        fig.savefig(os.path.join(graphs_dir, fn), dpi=220)
        plt.close(fig)



def exemplar_quotes(df, tok, outd, k_per_bucket: int = 6):
    made = 0
    for (area, when, grp), sub in df.groupby(["area","time","group"], dropna=False):
        docs = sub["text"].astype(str).tolist()
        if len(docs) < max(8, k_per_bucket):
            path = os.path.join(outd, f"exemplars_{area}_{when}_{grp or 'NA'}.csv")
            safe_write_csv(pd.DataFrame(columns=["rank","uid","qid","text","score","cluster"]), path, columns=["rank","uid","qid","text","score","cluster"])
            continue
        tfv = TfidfVectorizer(tokenizer=tok, min_df=2, max_df=0.95, token_pattern=None)
        X = tfv.fit_transform(docs)
        n_clusters = min(6, max(2, X.shape[0]//20))
        km = KMeans(n_clusters=n_clusters, n_init=10, random_state=42)
        labels = km.fit_predict(X)
        import numpy as _np
        exemplars = []
        for c in range(n_clusters):
            doc_idx = _np.where(labels==c)[0]
            if len(doc_idx)==0: continue
            centroid = X[doc_idx].mean(axis=0)
            from numpy.linalg import norm
            d = []
            for di in doc_idx:
                v = X[di]
                dist = norm((v - centroid).A)
                d.append((di, float(dist)))
            d.sort(key=lambda x: x[1])
            take = d[:max(1, k_per_bucket//n_clusters)]
            for di, dist in take:
                exemplars.append((di, dist, c))
        exemplars.sort(key=lambda x: x[1])
        rows = []
        for rank, (di, dist, c) in enumerate(exemplars[:k_per_bucket], start=1):
            rows.append({
                "rank": rank,
                "uid": sub.iloc[di]["UID"],
                "qid": sub.iloc[di]["qid"],
                "text": sub.iloc[di]["text"],
                "score": dist,
                "cluster": c
            })
        dfex = pd.DataFrame(rows, columns=["rank","uid","qid","text","score","cluster"])
        path = os.path.join(outd, f"exemplars_{area}_{when}_{grp or 'NA'}.csv")
        safe_write_csv(dfex, path, columns=["rank","uid","qid","text","score","cluster"])
        html_path = os.path.join(outd, f"exemplars_{area}_{when}_{grp or 'NA'}.html")
        buf = io.StringIO()
        buf.write("<!DOCTYPE html><meta charset='utf-8'><title>Exemplar quotes</title>")
        buf.write(f"<h1>Exemplar – Area: {html.escape(str(area))} · {html.escape(str(when))} · Gruppo: {html.escape(str(grp))}</h1>")
        for _,r in dfex.iterrows():
            buf.write("<div style='border-left:4px solid #ddd;padding:8px 12px;margin:10px 0'>")
            buf.write(f"<div style='color:#666'><b>UID:</b> {html.escape(str(r['uid']))} · <b>QID:</b> {html.escape(str(r['qid']))} · <b>Cluster:</b> {int(r['cluster'])} · <b>Score:</b> {r['score']:.3f}</div>")
            buf.write(f"<div>{html.escape(str(r['text']))}</div></div>")
        write_text(html_path, buf.getvalue()); made += 1
    return made



def export_lexicon_contributions(df_rows: pd.DataFrame, cfg: dict, out_dir: str, tok_for_lex=_simple_tokenize):
    
    sent_cfg = (cfg or {}).get("sentiment", {}) or {}
    lex = {str(k).lower(): float(v) for k, v in (sent_cfg.get("lexicon", {}) or {}).items()}
    boosters = {str(k).lower(): float(v) for k, v in (sent_cfg.get("boosters", {}) or {}).items()}
    negators = set(str(w).lower() for w in (sent_cfg.get("negators", []) or []))
    try:
        aro_lex = {str(k).lower(): float(v) for k, v in (sent_cfg.get("arousal_lexicon", {}) or {}).items()}
        dom_lex = {str(k).lower(): float(v) for k, v in (sent_cfg.get("dominance_lexicon", {}) or {}).items()}
    except Exception:
        aro_lex, dom_lex = {}, {}
    from collections import defaultdict
    contrib_sent = defaultdict(float); contrib_arou = defaultdict(float); contrib_dom  = defaultdict(float)
    for s in df_rows["text"].astype(str):
        toks = tok_for_lex(s); n = len(toks)
        for i, w in enumerate(toks):
            L = max(0, i-2); R = min(n, i+3); ctx = toks[L:i] + toks[i+1:R]
            if w in lex:
                sign = -1.0 if any(b in negators for b in ctx) else 1.0
                boost = 1.0
                for b in ctx:
                    if b in boosters:
                        try: boost *= float(boosters[b])
                        except Exception: pass
                contrib_sent[w] += sign * lex[w] * boost
            if w in aro_lex or w in dom_lex:
                boost2 = 1.0
                for b in ctx:
                    if b in boosters:
                        try: boost2 *= float(boosters[b])
                        except Exception: pass
                if w in aro_lex: contrib_arou[w] += aro_lex[w] * boost2
                if w in dom_lex: contrib_dom[w]  += dom_lex[w]  * boost2
    import pandas as pd
    def _top(dfm: pd.DataFrame, k=200):
        if dfm.empty: return dfm
        dfm["_abs"] = dfm["contribution"].abs()
        return dfm.sort_values("_abs", ascending=False).drop(columns=["_abs"]).head(k)
    s_df = pd.DataFrame([{"token":k, "contribution":v} for k,v in contrib_sent.items()])
    a_df = pd.DataFrame([{"token":k, "contribution":v} for k,v in contrib_arou.items()])
    d_df = pd.DataFrame([{"token":k, "contribution":v} for k,v in contrib_dom.items()])
    _od = out_dir
    try: os.makedirs(_od, exist_ok=True)
    except Exception: pass
    _top(s_df).to_csv(os.path.join(_od, "lexicon_contributions_sentiment.csv"), index=False, encoding="utf-8")
    _top(a_df).to_csv(os.path.join(_od, "lexicon_contributions_arousal.csv"), index=False, encoding="utf-8")
    _top(d_df).to_csv(os.path.join(_od, "lexicon_contributions_dominance.csv"), index=False, encoding="utf-8")



def export_area_descriptors(df_rows: pd.DataFrame, tok, out_csv_path: str, top_k: int = 8):
    from collections import Counter
    rows = []
    for area, sub in df_rows.groupby("area"):
        toks = []
        for s in sub["text"].astype(str):
            toks.extend(tok(s))
        cnt = Counter(toks)
        top_terms = [w for w,_ in cnt.most_common(top_k)]
        rows.append({"area": area, "descriptor_terms": "; ".join(top_terms)})
    import pandas as pd, os
    os.makedirs(os.path.dirname(out_csv_path), exist_ok=True) if os.path.dirname(out_csv_path) else None
    pd.DataFrame(rows, columns=["area","descriptor_terms"]).to_csv(out_csv_path, index=False, encoding="utf-8")




class App(tk.Tk):
    def __init__(self):
        super().__init__()
        self.title("eTwinning – Analisi qualitativa: Sociale e Civica (v1.10.3)")
        self.geometry("1180x860")
        self._build()

    def _build(self):
        f = ttk.Frame(self); f.pack(fill=tk.BOTH, expand=True, padx=12, pady=12)
        r = 0
        ttk.Label(f, text="Questionario ENTRATA (CSV)").grid(row=r, column=0, sticky="w")
        self.e_pre = ttk.Entry(f, width=92); self.e_pre.grid(row=r, column=1, sticky="we", padx=8)
        self.e_pre.insert(0, r"/Users/jacopocondo/Desktop/Indire/DATI FINALI/1 - questionari/entrata/QUESTIONARIO PER LA VALUTAZIONE DELLE COMPETENZE CHIAVE in entrata pt.2.csv")
        ttk.Button(f, text="Scegli", command=lambda: self._pick(self.e_pre)).grid(row=r, column=2); r+=1

        ttk.Label(f, text="Questionario USCITA (CSV)").grid(row=r, column=0, sticky="w")
        self.e_post = ttk.Entry(f, width=92); self.e_post.grid(row=r, column=1, sticky="we", padx=8)
        self.e_post.insert(0, r"/Users/jacopocondo/Desktop/Indire/DATI FINALI/1 - questionari/uscita/QUESTIONARIO PER LA VALUTAZIONE DELLE COMPETENZE CHIAVE in uscita pt.2 Maggio.csv")
        ttk.Button(f, text="Scegli", command=lambda: self._pick(self.e_post)).grid(row=r, column=2); r+=1

        ttk.Label(f, text="BAN list file (.txt) – parole da escludere (obbligatorio)").grid(row=r, column=0, sticky="w")
        self.e_ban = ttk.Entry(f, width=92); self.e_ban.grid(row=r, column=1, sticky="we", padx=8)
        self.e_ban.insert(0, r"/Users/jacopocondo/Desktop/Indire/DATI FINALI/script/support material/QUALITATIVO/COMP1 personale/stopwords_it_derived_from_zip.txt")
        ttk.Button(f, text="Scegli", command=lambda: self._pick(self.e_ban, "*.txt")).grid(row=r, column=2); r+=1

        ttk.Label(f, text="Config JSON (metric_weights) – OBBLIGATORIO").grid(row=r, column=0, sticky="w")
        self.e_json = ttk.Entry(f, width=92); self.e_json.grid(row=r, column=1, sticky="we", padx=8)
        self.e_json.insert(0, r"/Users/jacopocondo/Desktop/Indire/DATI FINALI/script/support material/QUALITATIVO/COMP1 personale/sentiment/2/metric_weights_config.v6.with_arousal_dominance_autotune.CONSOLIDATED.json")
        ttk.Button(f, text="Scegli", command=lambda: self._pick(self.e_json, "*.json")).grid(row=r, column=2); r+=1

        ttk.Label(f, text="Cartella OUTPUT (se vuota → reports/<timestamp>)").grid(row=r, column=0, sticky="w")
        self.e_out = ttk.Entry(f, width=92); self.e_out.grid(row=r, column=1, sticky="we", padx=8)
        ttk.Button(f, text="Scegli cartella", command=self._pickdir).grid(row=r, column=2); r+=1

        self.log = tk.Text(f, height=24, wrap="word", bg="#101010", fg="#d6d6d6")
        self.log.grid(row=r, column=0, columnspan=3, sticky="nsew", pady=(10,6)); r+=1
        f.rowconfigure(r-1, weight=1); f.columnconfigure(1, weight=1)

        ttk.Button(f, text="Esegui analisi", command=self.run).grid(row=r, column=0, sticky="w", pady=6)

    def _pick(self, entry, pattern="*"):
        p = filedialog.askopenfilename(title="Seleziona file", filetypes=[("File", pattern), ("Tutti i file","*.*")])
        if p:
            entry.delete(0, tk.END); entry.insert(0, p)

    def _pickdir(self):
        d = filedialog.askdirectory(title="Cartella output")
        if d:
            self.e_out.delete(0, tk.END); self.e_out.insert(0, d)

    def _log(self, msg):
        self.log.insert(tk.END, msg+"\n"); self.log.see(tk.END); self.update_idletasks()

    def run(self):
        pre = self.e_pre.get().strip()
        post= self.e_post.get().strip()
        banf= self.e_ban.get().strip()
        jsnf= self.e_json.get().strip()
        outd= self.e_out.get().strip()

        if not pre or not post:
            messagebox.showwarning("Attenzione", "Seleziona i CSV di Entrata e Uscita."); return
        if not banf:
            messagebox.showwarning("BAN list mancante", "Seleziona un file .txt con le parole da escludere."); return
        if not jsnf:
            messagebox.showwarning("Config JSON mancante", "Seleziona il file JSON delle metriche."); return

        
        if outd:
            ensure_dir(outd)
        else:
            outd = os.path.join(os.getcwd(), "reports", datetime.now().strftime("%Y%m%d_%H%M%S"))
            ensure_dir(outd)
        figs_dir = os.path.join(outd, "figs"); ensure_dir(figs_dir)
        val_dir  = os.path.join(outd, "validation"); ensure_dir(val_dir)
        diag_dir = os.path.join(outd, "diagnostics"); ensure_dir(diag_dir)
        self._log(f"Output: {outd}")

        runlog = os.path.join(outd, "run.log")
        def logline(s):
            self._log(s)
            with open(runlog, "a", encoding="utf-8") as f:
                f.write(s+"\n")
        
        try:
            with open(jsnf, "r", encoding="utf-8") as f:
                cfg = json.load(f)
            logline(f"[OK] Config caricato: {jsnf}")
        except Exception as e:
            messagebox.showerror("Config JSON non valido", f"Errore nel leggere {jsnf}:\n{e}")
            return

        
        pair = (cfg.get("pairing") or {})
        code_col_letter = str((pair.get("code_col") or "AY")).strip().upper()

        
        df_pre_full,  school_map_pre  = load_questionnaire_lettered(pre)
        df_post_full, school_map_post = load_questionnaire_lettered(post)

        
        def _first_nonempty_in_range(row: pd.Series, start_col: str, end_col: str):
            c0 = _excel_col_to_idx(str(start_col).strip().upper())
            c1 = _excel_col_to_idx(str(end_col).strip().upper())
            cols = list(row.index)
            subcols = [c for i, c in enumerate(cols) if c0 <= i <= c1]
            hits = []
            for c in subcols:
                v = str(row.get(c, "")).strip()
                if v and v.lower() != "nan":
                    hits.append(c)
            if len(hits) == 1:
                return (hits[0], np.nan)
            elif len(hits) == 0:
                return (np.nan, f"NONE_SELECTED:{start_col}-{end_col}")
            else:
                return (np.nan, f"MULTI_SELECTED:{start_col}-{end_col}:{'|'.join(hits)}")

        
        def _normalize_identity_lettered(df_lettered: pd.DataFrame, school_map: dict, time_label: str) -> pd.DataFrame:
            
            if len(df_lettered) < 2:
                return pd.DataFrame(columns=["school","class","section","group","student_id","time","code","_warn_identity"])
            
            desc = df_lettered.iloc[1]

            def _label_map(start_col: str, end_col: str) -> dict:
                c0 = _excel_col_to_idx(start_col); c1 = _excel_col_to_idx(end_col)
                cols = list(df_lettered.columns)
                return {c: desc.get(c, c) for i, c in enumerate(cols) if c0 <= i <= c1}

            
            class_map   = _label_map("Y",  "AA")
            section_map = _label_map("AB", "AV")

            
            df = df_lettered.iloc[2:].reset_index(drop=True).copy()

            
            pairs_s = [_first_nonempty_in_range(row, "J",  "X")  for _, row in df.iterrows()]
            sc,  warn_s  = zip(*pairs_s)  if pairs_s else ([], [])

            pairs_c = [_first_nonempty_in_range(row, "Y",  "AA") for _, row in df.iterrows()]
            cl,  warn_c  = zip(*pairs_c)  if pairs_c else ([], [])

            pairs_se = [_first_nonempty_in_range(row, "AB", "AV") for _, row in df.iterrows()]
            se,  warn_se = zip(*pairs_se) if pairs_se else ([], [])
            df["class_col"]   = pd.Series(cl, index=df.index)
            df["section_col"] = pd.Series(se, index=df.index)
            df["class"]   = df["class_col"].map(  lambda c: class_map.get(c,   np.nan))
            df["section"] = df["section_col"].map(lambda c: section_map.get(c, np.nan))

            
            true_vals  = set(map(str.lower, (pair.get("etw", {}) or {}).get("true_values",  ["si","sì","yes","true","1","x","✓"])))
            false_vals = set(map(str.lower, (pair.get("etw", {}) or {}).get("false_values", ["no","false","0"])))
            def _etw_flag(row: pd.Series):
                c0 = _excel_col_to_idx("AW"); c1 = _excel_col_to_idx("AX")
                cols = list(row.index)
                subcols = [c for i, c in enumerate(cols) if c0 <= i <= c1]
                vals = [str(row.get(c, "")).strip().lower() for c in subcols]
                tr = any(v in true_vals  for v in vals)
                fl = any(v in false_vals for v in vals)
                if tr and not fl:     return True,  np.nan
                if fl and not tr:     return False, np.nan
                if not tr and not fl: return np.nan, "ETW_NONE_SELECTED"
                return np.nan, "ETW_BOTH_TRUE_FALSE"
            ef, warn_e = zip(*[_etw_flag(row) for _, row in df.iterrows()])
            df["is_etw"] = pd.Series(ef, index=df.index)

            
            gl = pair.get("group_labels", {"treated":"etwinning","control":"controllo"})
            df["group"] = np.where(df["is_etw"]==True,  gl.get("treated","etwinning"),
                          np.where(df["is_etw"]==False, gl.get("control","controllo"), np.nan))

            
            code_idx = _excel_col_to_idx(code_col_letter)
            df["code"] = df.iloc[:, code_idx].astype(str).str.strip() if 0 <= code_idx < df.shape[1] else np.nan

            
            fmt = pair.get("student_id_format", "{school}|{class}|{section}|{code}")
            df["student_id"] = df.apply(
                lambda r: fmt.format_map({
                    "school":  str(r.get("school",  "")).strip(),
                    "class":   str(r.get("class",   "")).strip(),
                    "section": str(r.get("section", "")).strip(),
                    "code":    str(r.get("code",    "")).strip(),
                }),
                axis=1
            )
            
            df["time"] = time_label
            df["_warn_identity"] = (pd.Series(warn_s).fillna("").astype(str) + "|" +
                                    pd.Series(warn_c).fillna("").astype(str) + "|" +
                                    pd.Series(warn_se).fillna("").astype(str) + "|" +
                                    pd.Series(warn_e).fillna("").astype(str))
            return df

        
        df_pre_norm  = _normalize_identity_lettered(df_pre_full,  school_map_pre,  "pre")
        df_post_norm = _normalize_identity_lettered(df_post_full, school_map_post, "post")
        
        for _d in (df_pre_norm, df_post_norm):
            for _c in ("school","class","section","code","_warn_identity","group","time","student_id"):
                if _c not in _d.columns:
                    _d[_c] = np.nan
        df = pd.concat([df_pre_norm, df_post_norm], ignore_index=True)

        logline(f"[ID] colonne create: student_id / group / time / school / class / section / code")

        
        bad = df[df["_warn_identity"].str.contains("MULTI_SELECTED|NONE_SELECTED|ETW_", na=False)]
        if len(bad) > 0:
            bad_path = os.path.join(diag_dir, "identity_warnings.csv")
            bad.to_csv(bad_path, index=False, encoding="utf-8")
            logline(f"[WARN] Incongruenze rilevate su selezioni J..X / Y..AA / AB..AV / AW..AX → {bad_path}")
        else:
            logline("[OK] Nessuna incongruenza nelle selezioni (J..X / Y..AA / AB..AV / AW..AX)")

        
        roster_cols = ["student_id","school","class","section","group","time","code"]
        try:
            roster = df[roster_cols].drop_duplicates()
            roster.to_csv(os.path.join(diag_dir, "identity_roster.csv"), index=False, encoding="utf-8")
            dup_mask = roster.duplicated(subset=["student_id"], keep=False)
            dups = roster.loc[dup_mask].sort_values("student_id")
            dups.to_csv(os.path.join(diag_dir, "identity_duplicates.csv"), index=False, encoding="utf-8")
            logline(f"[OK] Esportati roster e duplicati → identity_roster.csv / identity_duplicates.csv")
        except Exception as _e:
            logline(f"[INFO] Impossibile creare roster/duplicati: {_e}")
            
        
        def _bad_mask(df_norm: pd.DataFrame) -> pd.Series:
            core = ["school","class","section","code"]
            
            dsub = df_norm.reindex(columns=core, fill_value=np.nan)

            
            if "code" in df_norm.columns:
                code_series = df_norm["code"]
            else:
                code_series = pd.Series([""] * len(df_norm), index=df_norm.index)

            miss_core = dsub.isna().any(axis=1) | code_series.astype(str).str.strip().eq("")

            
            if "_warn_identity" in df_norm.columns:
                warn_series = df_norm["_warn_identity"].astype(str)
            else:
                warn_series = pd.Series([""] * len(df_norm), index=df_norm.index)

            warn_bad = warn_series.str.contains("MULTI_SELECTED|NONE_SELECTED|ETW_", na=False)

            return (miss_core | warn_bad)

        def _identity_stats(df_norm: pd.DataFrame) -> dict:
            bm = _bad_mask(df_norm)
            total = int(len(df_norm))
            lost  = int(bm.sum())
            ok    = int(total - lost)
            uniq_ok = int(df_norm.loc[~bm, "student_id"].nunique())
            dup_ok  = int(df_norm.loc[~bm, "student_id"].duplicated(keep=False).sum())
            return {"rows_total": total, "rows_lost": lost, "rows_identifiable": ok,
                    "unique_students_identifiable": uniq_ok, "dup_ids_within_time": dup_ok}

        pre_stats  = _identity_stats(df_pre_norm)
        post_stats = _identity_stats(df_post_norm)

        
        pre_ok_ids  = set(df_pre_norm.loc[~_bad_mask(df_pre_norm),  "student_id"].dropna().astype(str))
        post_ok_ids = set(df_post_norm.loc[~_bad_mask(df_post_norm), "student_id"].dropna().astype(str))
        matched_ids = pre_ok_ids & post_ok_ids

        summary_rows = [
            {"time":"pre",  **pre_stats},
            {"time":"post", **post_stats},
            {"time":"matched_pre_and_post", "unique_students_identifiable": len(matched_ids)}
        ]
        pd.DataFrame(summary_rows).to_csv(os.path.join(diag_dir, "identity_summary_counts.csv"),
                                          index=False, encoding="utf-8")

        
        def _reason(df_norm: pd.DataFrame) -> pd.Series:
            def r(row):
                rsn = []
                if pd.isna(row.get("school")):  rsn.append("missing_school")
                if pd.isna(row.get("class")):   rsn.append("missing_class")
                if pd.isna(row.get("section")): rsn.append("missing_section")
                if not row.get("code") or str(row.get("code")).strip()=="":
                    rsn.append("missing_code")
                w = str(row.get("_warn_identity","")).strip()
                if w: rsn.append(w)
                return "|".join(rsn) if rsn else "ok"
            return df_norm.apply(r, axis=1)

        bad_pre  = df_pre_norm.loc[_bad_mask(df_pre_norm)].copy()
        bad_post = df_post_norm.loc[_bad_mask(df_post_norm)].copy()
        if not bad_pre.empty:
            bad_pre["time"] = "pre";  bad_pre["reason"]  = _reason(bad_pre)
            bad_pre[["student_id","school","class","section","code","time","_warn_identity","reason"]]                .to_csv(os.path.join(diag_dir, "identity_unidentifiable_pre.csv"), index=False, encoding="utf-8")
        if not bad_post.empty:
            bad_post["time"] = "post"; bad_post["reason"] = _reason(bad_post)
            bad_post[["student_id","school","class","section","code","time","_warn_identity","reason"]]                .to_csv(os.path.join(diag_dir, "identity_unidentifiable_post.csv"), index=False, encoding="utf-8")

        logline("[OK] Esportato identity_summary_counts.csv (+ unidentifiable_pre/post.csv se presenti)")
        
        try:
            tbl_pre  = build_tables_with_uid_and_group(pre,  "pre")
            tbl_post = build_tables_with_uid_and_group(post, "post")
        except Exception as e:
            traceback.print_exc()
            messagebox.showerror("Errore CSV", str(e)); return
        df = pd.concat([tbl_pre, tbl_post], ignore_index=True)
        if df.empty:
            messagebox.showerror("Vuoto", "Nessuna risposta aperta trovata."); return
        df["area"] = df["competenza"].map(SPEC_TO_AREA_DEFAULT).fillna(df["competenza"])

        
        pf = df.groupby(["area","time","group"], dropna=False).size().reset_index(name="n")
        safe_write_csv(pf, os.path.join(outd, "preflight_counts.csv"), columns=["area","time","group","n"])

        
        import re  

        ban_terms = set(BAN_TERMS_DEFAULT)
        try:
            with open(banf, "r", encoding="utf-8") as f:
                for line in f:
                    w = line.strip().lower()
                    if w:
                        
                        w = re.sub(r"\s+", " ", w)
                        ban_terms.add(w)

            
            ban_phrases = {w for w in ban_terms if " " in w}

            
            with open(os.path.join(outd, "effective_banlist.txt"), "w", encoding="utf-8") as f:
                for w in sorted(ban_terms):
                    f.write(w + "\n")
        except Exception as e:
            messagebox.showerror("BAN list", f"Errore nel leggere il file BAN: {e}")
            return
        
        
        keep_terms = set(map(str.lower, cfg.get("tokenizer", {}).get("keep_terms", [])))
        derived = derive_stopwords_from_corpus(
            df["text"].astype(str).tolist(),
            keep_terms,
            min_doc_frac=0.35,
            min_tf_short=10
        )
        tok = build_tokenizer(stop_extra=derived, keep_terms=keep_terms, ban_terms=ban_terms)
        tok_bow = build_bow_tokenizer(stop_extra=derived, keep_terms=keep_terms, ban_terms=ban_terms, lemmatize=bool(cfg.get("nlp", {}).get("lemmatize_bow", True)))

        with open(os.path.join(outd, "effective_stopwords.txt"), "w", encoding="utf-8") as f:
            for w in sorted(derived):
                f.write(w + "\n")

        
        df["sentiment_base"] = sentiment_baseline(df["text"].astype(str).tolist(), tokenizer=tok)
        df["ttr_base"]       = df["text"].astype(str).map(lambda s: ttr_simple(s, tok))
        df["mtld_base"]      = df["text"].astype(str).map(lambda s: mtld_std(s, tok))
        df["sentiment_json"] = sentiment_from_json(df["text"].astype(str).tolist(), tokenizer=tok, cfg=cfg)
        df["ttr_json"]       = df["text"].astype(str).map(lambda s: ttr_from_json(s, tok, cfg))
        df["mtld_json"]      = df["text"].astype(str).map(lambda s: mtld_from_json(s, tok, cfg))
        
        _sent = cfg.get("sentiment", {}) if isinstance(cfg, dict) else {}

        
        aro_lex = {str(k).lower(): float(v) for k, v in (_sent.get("arousal_lexicon", {}) or {}).items()}
        dom_lex = {str(k).lower(): float(v) for k, v in (_sent.get("dominance_lexicon", {}) or {}).items()}

        
        boosters = {str(k).lower(): float(v) for k, v in (_sent.get("boosters", {}) or {}).items()}
        negators = set(str(w).lower() for w in (_sent.get("negators", []) or []))
        window = int(_sent.get("window", 3))
        max_token_contrib = float(_sent.get("max_token_contrib", 3.0))

        
        if not aro_lex and not dom_lex:
            df["arousal_json"] = np.nan
            df["dominance_json"] = np.nan
        else:
            tmp_ad = df["text"].astype(str).map(
                lambda s: _score_arousal_dominance(
                    s, aro_lex, dom_lex, boosters, negators, window, max_token_contrib
                )
            )
            df[["arousal_json", "dominance_json"]] = pd.DataFrame(tmp_ad.tolist(), index=df.index)

        
        qcfg = _sent.get("quadrants", {"val_pos": 0.05, "val_neg": -0.05, "arousal_high": 0.25})
        
        if isinstance(qcfg, dict) and str(qcfg.get("mode","")).lower() == "percentile":
            import numpy as _np
            v = df["sentiment_json"].astype(float).to_numpy()
            a = df["arousal_json"].astype(float).to_numpy()
            p_pos  = float(qcfg.get("val_pos_p", 66.0))
            p_neg  = float(qcfg.get("val_neg_p", 33.0))
            p_ar_h = float(qcfg.get("arousal_high_p", 66.0))
            if _np.isfinite(v).sum() >= 5:
                try:
                    qcfg["val_pos"] = float(_np.nanpercentile(v, p_pos))
                    qcfg["val_neg"] = float(_np.nanpercentile(v, p_neg))
                except Exception:
                    pass
            if _np.isfinite(a).sum() >= 5:
                try:
                    qcfg["arousal_high"] = float(_np.nanpercentile(a, p_ar_h))
                except Exception:
                    pass
            try:
                _thp = os.path.join(outd, "quadrant_thresholds_used.json")
                with open(_thp, "w", encoding="utf-8") as _f:
                    json.dump({"used_thresholds": {"val_pos": qcfg.get("val_pos"),
                                                   "val_neg": qcfg.get("val_neg"),
                                                   "arousal_high": qcfg.get("arousal_high")},
                               "mode": "percentile",
                               "params": {"val_pos_p": p_pos, "val_neg_p": p_neg, "arousal_high_p": p_ar_h}}, _f, ensure_ascii=False, indent=2)
            except Exception:
                pass
        df["quadrant"] = [_quadrant(v, a, qcfg) for v, a in zip(df["sentiment_json"], df["arousal_json"])]

        
        df["delta_sentiment"] = df["sentiment_json"] - df["sentiment_base"]
        df["delta_ttr"]       = df["ttr_json"]       - df["ttr_base"]
        df["delta_mtld"]      = df["mtld_json"]      - df["mtld_base"]

        
        safe_write_csv(df, os.path.join(outd, "open_answers_long.csv"))
        
        try:
            export_lexicon_contributions(df, cfg, os.path.join(outd, "diagnostics"), tok_for_lex=_simple_tokenize)
        except Exception:
            pass
        
        def _nan_r(a, b):
            a = pd.Series(a, dtype="float64"); b = pd.Series(b, dtype="float64")
            m = a.notna() & b.notna()
            if m.sum() < 3: return float('nan')
            a1, b1 = a[m], b[m]
            if a1.nunique() <= 1 or b1.nunique() <= 1: return float('nan')
            return float(a1.corr(b1))
        summ = {"sentiment_r": _nan_r(df["sentiment_base"], df["sentiment_json"]),
                "ttr_r": _nan_r(df["ttr_base"], df["ttr_json"]),
                "mtld_r": _nan_r(df["mtld_base"], df["mtld_json"]),
                "n_rows": int(df.shape[0])}
        pd.DataFrame([summ]).to_csv(os.path.join(outd, "validation", "summary_overall.csv"), index=False, encoding="utf-8")
        (df[["UID","group","area","time","qid","sentiment_base","sentiment_json","delta_sentiment",
              "ttr_base","ttr_json","delta_ttr","mtld_base","mtld_json","delta_mtld","text"]]
         .to_csv(os.path.join(outd, "validation", "metrics_per_response.csv"), index=False, encoding="utf-8"))

        
        area_summary = df.groupby(["area","time"]).agg(
            sentiment=("sentiment_json","mean"),
            ttr=("ttr_json","mean"),
            mtld=("mtld_json","mean"),
            arousal=("arousal_json","mean"),
            dominance=("dominance_json","mean"),
            n=("text","count")
        ).reset_index()
        safe_write_csv(area_summary, os.path.join(outd, "area_summary.csv"))
        try:
            export_area_descriptors(df, tok_bow, os.path.join(outd, "area_descriptors.csv"))
        except Exception:
            pass
        if df["group"].notna().any():
            ag = df.dropna(subset=["group"]).groupby(["area","group","time"]).agg(
                sentiment=("sentiment_json","mean"),
                ttr=("ttr_json","mean"),
                mtld=("mtld_json","mean"),
                arousal=("arousal_json","mean"),
                dominance=("dominance_json","mean"),
                n=("text","count")
            ).reset_index()
            safe_write_csv(ag, os.path.join(outd, "area_group_summary.csv"))

        
        wc_n = build_wordclouds(df, tok_bow, figs_dir)
        self._log(f"Wordclouds create: {wc_n} file (OUTPUT/figs/)")

        
        def build_qid_wordclouds(df: pd.DataFrame,
                                 qids: list,
                                 tok,
                                 figs_dir: str,
                                 qid_col: str = "qid",
                                 text_col: str = "text",
                                 group_col: str = "group",
                                 ban_terms: set = None,
                                 ban_phrases: set = None,
                                 topN_words: int = 500) -> int:
            
            import os, re
            import matplotlib.pyplot as plt
            from collections import Counter

            os.makedirs(figs_dir, exist_ok=True)
            made = 0

            def _save_wc(freqs: dict, out_path: str):
                nonlocal made
                if not freqs:
                    return
                if WordCloud is None:
                    
                    items = sorted(freqs.items(), key=lambda x: x[1], reverse=True)[:min(75, len(freqs))]
                    if not items:
                        return
                    labels = [k for k, _ in items]
                    vals   = [v for _, v in items]
                    plt.figure(figsize=(12, 6))
                    plt.bar(range(len(vals)), vals)
                    plt.xticks(range(len(vals)), labels, rotation=90, fontsize=8)
                    plt.tight_layout()
                    plt.savefig(out_path, dpi=160)
                    plt.close()
                    made += 1
                    return

                wc = WordCloud(width=1600, height=900, background_color="white",
                               prefer_horizontal=0.9, collocations=False,
                               normalize_plurals=False)
                wc.generate_from_frequencies(freqs)
                plt.figure(figsize=(12, 7))
                plt.imshow(wc, interpolation="bilinear")
                plt.axis("off")
                plt.tight_layout(pad=0)
                plt.savefig(out_path, dpi=180)
                plt.close()
                made += 1

            for qid in qids:
                base = df[df[qid_col].astype(str).eq(str(qid))]
                for label, gname in [("all", None), ("etwinning", "etwinning"), ("controllo", "controllo")]:
                    sub = base if gname is None else base[base[group_col].astype(str).str.lower().eq(gname)]
                    if sub.empty:
                        continue

                    tokens = []
                    for s in sub[text_col].astype(str):
                        s_norm = re.sub(r"\s+", " ", s.strip().lower())
                        
                        if ban_phrases and s_norm in ban_phrases:
                            continue
                        for t in tok(s_norm):
                            if ban_terms and t in ban_terms:
                                continue
                            if len(t) < 2:
                                continue
                            tokens.append(t)

                    if not tokens:
                        continue

                    cnt = Counter(tokens)
                    freqs = dict(cnt.most_common(topN_words))
                    out_png = os.path.join(figs_dir, f"wordcloud_{qid}_{label}.png")
                    _save_wc(freqs, out_png)

            return made

        
        qcfg = cfg.get("questions", {})

        def _pick_col(cols, candidates):
            for c in candidates:
                if c in cols:
                    return c
            return None

        qid_col   = qcfg.get("question_id_col") or _pick_col(df.columns, ["question_id","id_domanda","qid","domanda_id"])
        qtext_col = qcfg.get("question_text_col") or _pick_col(df.columns, ["question","domanda","question_text","testo_domanda"])
        rtext_col = (qcfg.get("response_text_col") 
                     or _pick_col(df.columns, ["text","risposta","response","open_answer"]) 
                     or "text")
        area_col  = qcfg.get("area_col", "area")
        group_col = qcfg.get("group_col", "group")
        time_col  = qcfg.get("time_col", "time")
        id_col    = qcfg.get("id_col", ("student_id" if "student_id" in df.columns else None))


        
        qid_list_for_wc = [q["qid"] for q in OPEN_QS]  
        wc_qid_n = build_qid_wordclouds(
            df=df,
            qids=qid_list_for_wc,
            tok=tok_bow,
            figs_dir=figs_dir,
            qid_col=qid_col,          
            text_col=rtext_col,
            group_col=group_col,
            ban_terms=ban_terms,      
            ban_phrases=ban_phrases   
        )
        self._log(f"[OK] Wordcloud QID-specifiche create: {wc_qid_n} file")

        
        kmade = key_terms_and_collocations(df, tok_bow, ban_terms, outd)

        
        topics_all_prev = []
        topics_terms_count = 0
        for area, sub in df.groupby("area"):
            model = topic_model_area(sub, tok_bow)
            if model is None:
                safe_write_csv(pd.DataFrame(columns=["area","topic","rank","term","weight"]), os.path.join(outd, f"topics_terms_area_{area}.csv"), columns=["area","topic","rank","term","weight"])
                continue
            topics_prev = export_topics(area, model, sub, outd)
            topics_all_prev.append(topics_prev)
            topics_terms_count += 1
        if topics_all_prev:
            prev_all = pd.concat(topics_all_prev, ignore_index=True)
            prev_path = os.path.join(outd, "topics_prevalence_area_group.csv")
            if os.path.exists(prev_path):
                old = pd.read_csv(prev_path)
                prev_all = pd.concat([old, prev_all], ignore_index=True).drop_duplicates()
            safe_write_csv(prev_all, prev_path)
        else:
            safe_write_csv(pd.DataFrame(columns=["area","group","time","topic","n","tot","prop"]), os.path.join(outd,"topics_prevalence_area_group.csv"),
                           columns=["area","group","time","topic","n","tot","prop"])
        export_topic_shifts(outd)

        
        ex_n = exemplar_quotes(df, tok_bow, outd, k_per_bucket=6)

        
        prog = df.groupby(["area","group","time"]).agg(sentiment=("sentiment_json","mean"), mtld=("mtld_json","mean")).reset_index()
        safe_write_csv(prog, os.path.join(outd,"progression_area_group.csv"))
        did_rows = []
        for area, sub in prog.groupby("area"):
            for metric in ["sentiment","mtld"]:
                def val(g,t):
                    v = sub[(sub["group"]==g) & (sub["time"]==t)][metric]
                    return float(v.values[0]) if len(v)==1 else float("nan")
                etw_delta = (val("etwinning","post") - val("etwinning","pre")) if all(~pd.isna([val("etwinning","post"), val("etwinning","pre")])) else float("nan")
                ctl_delta = (val("controllo","post") - val("controllo","pre")) if all(~pd.isna([val("controllo","post"), val("controllo","pre")])) else float("nan")
                did = (etw_delta - ctl_delta) if (not math.isnan(etw_delta) and not math.isnan(ctl_delta)) else float("nan")
                did_rows.append({"area":area,"metric":metric,"did":did,"delta_etwinning":etw_delta,"delta_controllo":ctl_delta})
        safe_write_csv(pd.DataFrame(did_rows), os.path.join(outd,"did_area_metrics.csv"), columns=["area","metric","did","delta_etwinning","delta_controllo"])
        try:
            export_did_with_pvalues(df, os.path.join(outd, "did_with_pvalues.csv"))
        except Exception:
            pass
        

        
        qid_col   = qid_col   or _pick_col(df.columns, ["question_id","id_domanda","qid","domanda_id"])
        qtext_col = qtext_col or _pick_col(df.columns, ["question","domanda","question_text","testo_domanda"])
        rtext_col = rtext_col or _pick_col(df.columns, ["text","risposta","response","open_answer"])

        normalize = bool(qcfg.get("normalize_responses", True))
        top_k     = int(qcfg.get("top_k", 20))
        id_col = qcfg.get("id_col", ("student_id" if "student_id" in df.columns else None))        
        area_col  = qcfg.get("area_col", "area")
        group_col = qcfg.get("group_col", "group")
        time_col  = qcfg.get("time_col", "time")

        
        WORD_QS_UPPER = {
            s.strip().upper() for s in (cfg.get("questions", {}).get("topk_word_ids", ["Q1","Q2","Q4","Q5"]))
        }
        def _word_mode(qid_val):
            return str(qid_val).strip().upper() in WORD_QS_UPPER

        
        WORD_QS_UPPER = {qid.upper() for qid in qid_list_for_wc}
        def _word_mode(qid_val):
            return str(qid_val).strip().upper() in WORD_QS_UPPER


        
        from typing import Optional

        
        special_qids = set(qid_list_for_wc)  

        def _topk_for_subset(sub_df: pd.DataFrame, outfile: str, force_granularity: Optional[str] = None):
            
            if sub_df.empty:
                logline(f"[INFO] Nessuna risposta utile per Top-K → {os.path.basename(outfile)}")
                return

            
            tmp = sub_df[[qid_col, qtext_col, rtext_col]].dropna(subset=[rtext_col]).copy()
            tmp["__norm"] = (
                tmp[rtext_col]
                .astype(str)
                .str.strip()
                .str.lower()
                .str.replace(r"\s+", " ", regex=True)
            )

            
            tmp = tmp[~tmp["__norm"].isin(ban_phrases)]

            rows = []
            for qid_val, g in tmp.groupby(qid_col, dropna=False):
                qtxt = g[qtext_col].iloc[0]

                
                if force_granularity == "word":
                    mode = "word"
                elif force_granularity == "verbatim":
                    mode = "verbatim"
                else:
                    mode = "word" if str(qid_val) in special_qids else "verbatim"

                if mode == "word":
                    
                    from collections import Counter
                    tokens = []
                    for s in g["__norm"]:
                        for t in tok(s):
                            if t in ban_terms or len(t) < 2:
                                continue
                            tokens.append(t)
                    if tokens:
                        cnt = Counter(tokens).most_common(top_k)
                        vc = pd.DataFrame(cnt, columns=["response", "n"])
                    else:
                        vc = pd.DataFrame(columns=["response", "n"])
                    vc["granularity"] = "word"
                else:
                    
                    
                    
                    vc = (
                        g["__norm"]
                        .value_counts(dropna=False)
                        .head(top_k)
                        .reset_index(name="n")
                        .rename(columns={"__norm": "response"})
                    )
                    vc["granularity"] = "verbatim"

                vc.insert(0, qid_col, qid_val)
                vc.insert(1, qtext_col, qtxt)
                rows.append(vc)

            if rows:
                out = pd.concat(rows, ignore_index=True)
                out.to_csv(outfile, index=False, encoding="utf-8")
                logline(f"[OK] Esportato {os.path.basename(outfile)} ({len(out)} righe)")
            else:
                logline(f"[INFO] Nessun dato esportato per Top-K → {os.path.basename(outfile)}")

        
        if qid_col and qtext_col and rtext_col and qid_col in df and qtext_col in df and rtext_col in df:
            _topk_for_subset(df, os.path.join(outd, "topk_answers_per_question.csv"))  
            _topk_for_subset(df, os.path.join(outd, "topk_answers_per_question_verbatim.csv"), force_granularity="verbatim")
        else:
            logline("[INFO] Colonne domanda/risposta non trovate: salto Top-K.")

        
        if qid_col and qtext_col and rtext_col and qid_col in df and qtext_col in df and rtext_col in df:
            for gname in ["etwinning", "controllo"]:
                sub = df[df[group_col].astype(str).str.lower().eq(gname)]
                _topk_for_subset(sub, os.path.join(outd, f"topk_answers_per_question_{gname}.csv"))
                _topk_for_subset(sub, os.path.join(outd, f"topk_answers_per_question_{gname}_verbatim.csv"), force_granularity="verbatim")

                        
        
        
        metrics = [c for c in ["sentiment_json","ttr_json","mtld_json","arousal_json","dominance_json"] if c in df.columns]
        p_rows = []
        if all(c in df.columns for c in [area_col, group_col, time_col]) and metrics:
            for area, g_area in df.groupby(area_col, dropna=False):
                for m in metrics:
                    
                    for grp, g_grp in g_area.groupby(group_col, dropna=False):
                        pre_vals  = g_grp.loc[g_grp[time_col].eq("pre"),  m].dropna().values
                        post_vals = g_grp.loc[g_grp[time_col].eq("post"), m].dropna().values
                        if len(pre_vals) > 2 and len(post_vals) > 2:
                            
                            if id_col and id_col in g_grp.columns:
                                pre_id  = g_grp.loc[g_grp[time_col].eq("pre"),  [id_col, m]].dropna()
                                post_id = g_grp.loc[g_grp[time_col].eq("post"), [id_col, m]].dropna()
                                merged  = pre_id.merge(post_id, on=id_col, suffixes=("_pre","_post"))
                                if len(merged) > 2:
                                    _, p_within = ttest_rel(merged[m+"_post"], merged[m+"_pre"], nan_policy="omit")
                                else:
                                    _, p_within = ttest_ind(post_vals, pre_vals, equal_var=False, nan_policy="omit")
                            else:
                                _, p_within = ttest_ind(post_vals, pre_vals, equal_var=False, nan_policy="omit")
                        else:
                            p_within = np.nan
                        p_rows.append({"area": area, "metric": m, "group": grp, "test": "pre_vs_post", "p_value": p_within})

                    
                    g_et  = g_area[g_area[group_col].astype(str).str.lower().eq("etwinning")]
                    g_ct  = g_area[g_area[group_col].astype(str).str.lower().eq("controllo")]
                    d_et  = g_et.loc[g_et[time_col].eq("post"), m].dropna().values - g_et.loc[g_et[time_col].eq("pre"),  m].dropna().mean() if len(g_et)>0 else np.array([])
                    d_ct  = g_ct.loc[g_ct[time_col].eq("post"), m].dropna().values - g_ct.loc[g_ct[time_col].eq("pre"),  m].dropna().mean() if len(g_ct)>0 else np.array([])
                    if d_et.size>2 and d_ct.size>2:
                        _, p_did = ttest_ind(d_et, d_ct, equal_var=False, nan_policy="omit")
                    else:
                        p_did = np.nan
                    p_rows.append({"area": area, "metric": m, "group": "DID(etwinning−controllo)", "test": "did_simple", "p_value": p_did})

            ptab = pd.DataFrame(p_rows)
            
            alpha = float(cfg.get("stats", {}).get("alpha", 0.05))  
            ptab["signif"]   = np.where(ptab["p_value"].astype(float) < alpha, "✓", "")
            ptab["sig_code"] = pd.cut(ptab["p_value"].astype(float),
                                      bins=[-np.inf, 0.001, 0.01, 0.05, np.inf],
                                      labels=["***","**","*",""])
            ptab.to_csv(os.path.join(outd, "pvalues_area_metrics.csv"), index=False, encoding="utf-8")
            logline(f"[OK] Esportato pvalues_area_metrics.csv ({len(ptab)} righe)")
        else:
            logline("[INFO] Colonne area/group/time o metriche mancanti: salto p-value.")

        
        if "quadrant" in df.columns and all(c in df.columns for c in [area_col, group_col, time_col]):
            quad_cnt = (
                df.groupby([area_col, group_col, time_col, "quadrant"], as_index=False)
                  .size()
                  .rename(columns={"size": "n"})
            )
            quad_cnt["share"] = quad_cnt.groupby([area_col, group_col, time_col])["n"].transform(lambda x: (x / x.sum()).round(4))
            quad_cnt.to_csv(os.path.join(outd, "quadrant_share_area_group_time.csv"), index=False, encoding="utf-8")
            logline(f"[OK] Esportato quadrant_share_area_group_time.csv ({len(quad_cnt)} righe)")
        else:
            logline("[INFO] Colonna 'quadrant' non trovata: salto tabella percentuali quadranti.")
        
        with open(os.path.join(outd, "group_mapping_reason.txt"),"w",encoding="utf-8") as f:
            f.write("Gruppo determinato HARD-CODED da colonne AW/AX (riga 3+).\nRegola: AW='Si'/'Sì'→eTwinning; AX='No'→Controllo.\n")
            f.write(str(df["group"].value_counts(dropna=False)) + "\n")

        
        sent_w = cfg.get("sentiment",{}).get("lexicon",{})
        ttr_w  = cfg.get("ttr",{}).get("weights",{})
        mtld_w = cfg.get("mtld",{}).get("weights",{})
        
        def _save_weights_table(name, dct):
            tab = pd.DataFrame({"term": list(dct.keys()), "weight": [float(dct[k]) for k in dct.keys()]})
            tab = tab.sort_values("weight", ascending=False)
            safe_write_csv(tab, os.path.join(diag_dir, f"{name}_weights.csv"))
            plt.figure(figsize=(6,4)); 
            if not tab.empty: plt.hist(tab["weight"].astype(float), bins=50)
            plt.title(f"Distribuzione {name} weights"); plt.tight_layout()
            plt.savefig(os.path.join(diag_dir, f"{name}_weights_hist.png"), dpi=140); plt.close()
            top_pos = tab.head(50); top_neg = tab.sort_values("weight").head(50)
            safe_write_csv(top_pos, os.path.join(diag_dir, f"{name}_top_positive.csv"))
            safe_write_csv(top_neg, os.path.join(diag_dir, f"{name}_top_negative.csv"))
            return len(tab)
        n_sent = _save_weights_table("sentiment", sent_w)
        n_ttr  = _save_weights_table("ttr", ttr_w)
        n_mtld = _save_weights_table("mtld", mtld_w)

        
        tokens_all = []
        for txt in df["text"].astype(str).tolist():
            tokens_all.extend(tok_bow(txt))
        total_tokens = len(tokens_all)
        cov_sent = sum(1 for t in tokens_all if t in sent_w) / max(1, total_tokens)
        cov_ttr  = sum(1 for t in tokens_all if t in ttr_w) / max(1, total_tokens)
        cov_mtld = sum(1 for t in tokens_all if t in mtld_w) / max(1, total_tokens)
        cov = pd.DataFrame([{"metric":"sentiment","coverage_tokens":cov_sent},
                            {"metric":"ttr","coverage_tokens":cov_ttr},
                            {"metric":"mtld","coverage_tokens":cov_mtld}])
        safe_write_csv(cov, os.path.join(diag_dir, "weights_coverage.csv"))

        
        with open(os.path.join(diag_dir,"diagnostics_report.html"),"w",encoding="utf-8") as f:
            f.write("<!DOCTYPE html><meta charset='utf-8'><title>Diagnostics – Weights</title>")
            f.write("<h1>Diagnostics – Pesi e copertura</h1>")
            f.write(f"<p>Generato: {html.escape(datetime.now().strftime('%Y-%m-%d %H:%M'))}</p>")
            f.write("<ul>")
            f.write(f"<li>Sentiment lexicon: {n_sent} termini</li>")
            f.write(f"<li>TTR weights: {n_ttr} termini</li>")
            f.write(f"<li>MTLD weights: {n_mtld} termini</li>")
            f.write("</ul>")
            f.write("<h3>Copertura lessico (token-level)</h3>")
            f.write(f"<p>Coverage sentiment: {cov_sent:.3f} · TTR: {cov_ttr:.3f} · MTLD: {cov_mtld:.3f}</p>")
            for name in ["sentiment","ttr","mtld"]:
                f.write(f"<h3>Distribuzione pesi – {name.upper()}</h3>")
                f.write(f"<img src='{name}_weights_hist.png' style='width:520px;display:block;margin-bottom:16px'>")
            f.write("<p>Vedi anche: *_weights.csv, *_top_positive.csv, *_top_negative.csv.</p>")

        
        status = {
            "wordcloud_written": wc_n,
            "key_terms_files": kmade["key_terms"],
            "collocation_bi_files": kmade["colloc_bi"],
            "collocation_tri_files": kmade["colloc_tri"],
            "topics_terms_areas": topics_terms_count,
            "exemplars_buckets": ex_n,
            "group_present": bool(df["group"].notna().any()),
            "diagnostics_cov_sentiment": cov_sent,
            "diagnostics_cov_ttr": cov_ttr,
            "diagnostics_cov_mtld": cov_mtld
        }
        write_text(os.path.join(outd,"module_status.json"), json.dumps(status, ensure_ascii=False, indent=2))

        
        if WordCloud is None:
            self._log("⚠️ WordCloud non installato: create barre dei top termini in OUTPUT/figs/. Per abilitare le nuvole vere: pip install wordcloud")

        messagebox.showinfo("Fatto", f"Analisi completata. Cartella: {outd}\nWordclouds in {os.path.join(outd,'figs')}.")
        self._log("OK.")




if __name__ == "__main__":
    App().mainloop()
