import os
os.environ["OMP_NUM_THREADS"]        = "1"   
os.environ["OPENBLAS_NUM_THREADS"]   = "1"   
os.environ["MKL_NUM_THREADS"]        = "1"   
os.environ["VECLIB_MAXIMUM_THREADS"] = "1"   
import json, os, re, warnings, tkinter as tk
from dataclasses import dataclass, field
from tkinter import filedialog, messagebox


DEFAULT_PRE_CSV  = r"""/Users/jacopocondo/Desktop/Indire/DATI FINALI/1 - questionari/entrata/QUESTIONARIO PER LA VALUTAZIONE DELLE COMPETENZE CHIAVE in entrata pt.3.csv"""
DEFAULT_POST_CSV = r"""/Users/jacopocondo/Desktop/Indire/DATI FINALI/1 - questionari/uscita/QUESTIONARIO PER LA VALUTAZIONE DELLE COMPETENZE CHIAVE in uscita pt.3 Maggio.csv"""
DEFAULT_MAPPING_JSON = r"""/Users/jacopocondo/Desktop/Indire/DATI FINALI/script/support material/QUANTITATIVO/COMP3 imprenditoriale/update/mapping_competenza_imprenditoriale.json"""
DEFAULT_MAPPING_JSON_FALLBACK = r""""""
DEFAULT_OUT_DIR  = r"""/Users/jacopocondo/Desktop/Indire/DATI FINALI/OUTPUT_ANALISI/COMP3_imprenditoriale"""

def _resolve_path(primary: str, fallback: str | None = None) -> str:
    
    try:
        if primary and os.path.exists(primary):
            return primary
        if fallback and os.path.exists(fallback):
            return fallback
    except Exception:
        pass
    return primary if primary else (fallback or "")




import os
os.environ["OMP_NUM_THREADS"] = "1"      
os.environ["OPENBLAS_NUM_THREADS"] = "1" 
os.environ["MKL_NUM_THREADS"] = "1"      


def eta_boot_ci(df, formula, term="is_etw", n_boot=100, alpha=0.05):
    
    etas = []
    for _ in range(n_boot):
        d   = df.sample(frac=1, replace=True)
        try:
            mdl = smf.ols(formula, d).fit()
            aov = anova_lm(mdl, typ=2)          
            ss  = aov.at[term,      "sum_sq"]
            sse = aov.at["Residual", "sum_sq"]
            etas.append(ss / (ss + sse))
        except Exception:
            continue                            
    if len(etas) < 20:
        return np.nan, np.nan
    lo, hi = np.percentile(etas, [100*alpha/2, 100*(1-alpha/2)])
    return lo, hi

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.patches import Patch 
from scipy import stats
from scipy.stats import norm
import statsmodels.formula.api as smf
from statsmodels.stats.anova import anova_lm
from statsmodels.stats.multitest import multipletests
from sklearn.decomposition import PCA
from sklearn.manifold import MDS
import textwrap 

warnings.filterwarnings("ignore", category=FutureWarning, module="pandas")
plt.rcParams["figure.dpi"] = 110

warnings.filterwarnings("ignore",
                        message="covariance of constraints",
                        module="statsmodels")
warnings.filterwarnings("ignore",
                        message="invalid value encountered in divide",
                        module="statsmodels")


def hedges_g_ci(x, y, alpha=0.05):
    
    x, y = np.asarray(x, float), np.asarray(y, float)
    n1, n2 = len(x), len(y)
    if min(n1, n2) < 2:
        return np.nan, np.nan, np.nan
    m1, m2 = x.mean(), y.mean()
    s1, s2 = x.std(ddof=1), y.std(ddof=1)
    s_pooled = np.sqrt(((n1-1)*s1**2 + (n2-1)*s2**2) / (n1+n2-2))
    d = (m1 - m2) / s_pooled
    J = 1 - 3 / (4*(n1+n2) - 9)
    g = d * J
    se = np.sqrt((n1+n2)/(n1*n2) + g**2/(2*(n1+n2-2))) * J
    z  = norm.ppf(1 - alpha/2)
    return g, g - z*se, g + z*se


def partial_eta_squared(model, term="is_etw"):
    aov = anova_lm(model, typ=2)
    ss_eff = aov.loc[term, "sum_sq"]
    ss_err = aov.loc["Residual", "sum_sq"]
    return ss_eff / (ss_eff + ss_err)


def eta_boot_ci(df, formula, term="is_etw", n_boot=100, alpha=0.05):
    etas = []
    for _ in range(n_boot):
        d   = df.sample(frac=1, replace=True)
        try:
            mdl = smf.ols(formula, d).fit()
            aov = anova_lm(mdl, typ=2)              
            ss  = aov.at[term,      "sum_sq"]
            sse = aov.at["Residual", "sum_sq"]
            etas.append(ss / (ss + sse))
        except Exception:
            continue
    if len(etas) < 20:
        return np.nan, np.nan
    lo, hi = np.percentile(etas, [100*alpha/2, 100*(1-alpha/2)])
    return lo, hi



def forest_plot(df, eff, lo, hi, *, title, path):
    df = df.sort_values(eff)
    y  = np.arange(len(df))
    plt.figure(figsize=(6, .45*len(df)+1.5))
    plt.errorbar(df[eff], y,
                 xerr=[df[eff]-df[lo], df[hi]-df[eff]],
                 fmt='o', color='k', capsize=3)
    plt.axvline(0, color='grey', ls='--')
    plt.yticks(y, df["entity"])
    plt.xlabel("Hedges g (95 % CI)")
    plt.title(title)
    plt.tight_layout(); plt.savefig(path); plt.close()




AREA_MAP: dict[str, list[str]] = {
    "Idee e opportunità": [
        "Individuare opportunità",
        "Creatività e pensiero esplorativo",
        "Visione e creatività verso il futuro",
        "Valorizzare idee e pensiero etico",
        "Iniziativa individuale",
    ],
    "Gestione risorse": [
        "Consapevolezza di sé e autoefficacia",
        "Motivazione, perseveranza e adattabilità",
        "Mobilitare risorse e alfabetizzazione finanziaria",
        "Motivare gli altri e lavorare con gli altri",
        "Gestire l’apprendimento e mentalità da crescita",
    ],
    "Trasformare le idee in azione": [
        "Pianificazione e gestione",
        "Apprendimento dall’esperienza",
        "Copyright e licenze",
    ],
}

AREA_COLOR: dict[str, str] = {
    "Idee e opportunità": "navy",
    "Gestione risorse": "darkorange",
    "Trasformare le idee in azione": "darkgreen",
}


AGGR_LABEL: str = "Competenza imprenditoriale"
SCHOOL_COLS: list[str]         = [chr(c) for c in range(ord("J"), ord("X")+1)]
SEZIONE_COLS: list[str]       = ['AB', 'AC', 'AD', 'AE', 'AF', 'AG', 'AH', 'AI', 'AJ', 'AK', 'AL', 'AM', 'AN', 'AO', 'AP', 'AQ', 'AR', 'AS', 'AT', 'AU', 'AV']
INVERTED_START_COLS = ["BU", "NS", "MN", "MY"]

ACCENT_SUB: dict[str, str] = {"Ã¨": "è", "Ã©": "é", "Ã": "à",
                              "Ã¹": "ù", "â€™": "’"}

def fix_enc(s: str) -> str:
    
    if not isinstance(s, str):
        return s
    return re.sub("|".join(map(re.escape, ACCENT_SUB)),
                  lambda m: ACCENT_SUB[m.group(0)], s)

def excel_col(i: int) -> str:
    
    out = ""
    while True:
        i, r = divmod(i, 26)
        out = chr(r + 65) + out
        if i == 0:
            break
        i -= 1
    return out

gen_cols = lambda n: [excel_col(i) for i in range(n)]


q = lambda name: f'Q("{name}")'


@dataclass
class DataLoader:
    path: str
    inverted: list[str]
    school_map: dict = field(init=False)

    def load_raw(self):
        df = pd.read_csv(self.path, header=None, dtype=str)
        df.columns = gen_cols(df.shape[1])
        desc = df.iloc[1].apply(fix_enc)
        self.school_map = {c: desc[c] for c in SCHOOL_COLS}
        df = df.drop(index=[0, 1], errors="ignore").reset_index(drop=True).apply(lambda c: c.map(fix_enc))
        return df

    
    def _score(self, block, rev):
        def s(r):
            first = r.first_valid_index()
            if first is None:
                return np.nan
            v = block.columns.get_loc(first) + 1
            return 6 - v if rev else v
        return block.apply(s, axis=1)

    def likert_numeric(self, df, mapping):
        scored = {}
        for q, info in mapping.items():
            start = info["start_col"]; rev = start in self.inverted
            i0 = df.columns.get_loc(start)
            scored[q] = self._score(df.iloc[:, i0:i0+5], rev)
        out = pd.DataFrame(scored)
        
        present_school_cols = [c for c in SCHOOL_COLS if c in df.columns]
        if not present_school_cols:
            out["school"] = np.nan
        else:
            out["school"] = df.apply(
                lambda r: next(
                    (self.school_map[c] for c in present_school_cols
                     if pd.notna(r.get(c))),
                    ""
                ),
                axis=1
            )
            out["school"] = out["school"].fillna("").astype(str).str.strip()

        out["is_etw"] = df["AW"].eq("Si").fillna(False)
        out["AY"]     = df["AY"]
        
        def _sezione(row):
            for col in SEZIONE_COLS:
                v = str(row.get(col, "")).strip()
                if v and v.lower() not in ("nan", "none"):
                    return v
            return np.nan

        out["sezione"] = df.apply(_sezione, axis=1)

        
        def _classe(row):
            
            for col, lbl in (("Y", "prima"),
                             ("Z", "seconda"),
                             ("AA", "terza")):
                v = str(row.get(col, "")).strip().lower()
                if v in ("prima", "seconda", "terza"):
                    return lbl
            return np.nan

        out["classe"] = df.apply(_classe, axis=1)

        out["eta"] = out["classe"].map({"prima": 15,
                                        "seconda": 16,
                                        "terza": 17})
        return out


@dataclass
class Analyzer:
    pre: pd.DataFrame
    post: pd.DataFrame
    mapping: dict

    SCALE   = 7/4          
    TICKS   = list(range(1, 9))
    TICKLAB = ["A1","A2","B1","B2","C1","C2","D1","D2"]

    @staticmethod
    def _sanitize_filename(name: str) -> str:
        
        s = str(name).strip()
        s = re.sub(r"\s+", "_", s)
        s = re.sub(r"[^A-Za-z0-9._-]+", "_", s)
        s = re.sub(r"_+", "_", s).strip("_")
        return s or "school"


    @staticmethod
    def _impute(df):
        for col in df.select_dtypes("number"):
            df[col] = df[col].fillna(df[col].mean()) if df[col].notna().any() else 0.0
        return df

    
    def process(self):
        def _norm_ay(x):
            if pd.isna(x):
                return ""
            s = str(x).strip()
            if s.endswith(".0"):
                s = s[:-2]
            s = re.sub(r"\s+", "", s)
            return s

        for d in (self.pre, self.post):
            d["AY_norm"] = d["AY"].map(_norm_ay)
            d["grp"] = d["is_etw"].map(lambda v: "ETW" if bool(v) else "CTL")
            for c in ("school", "classe", "sezione"):
                if c in d.columns:
                    d[c] = d[c].fillna("").astype(str)
                else:
                    d[c] = ""
            d["UID"] = d["school"] + "_" + d["grp"] + "_" + d["classe"] + "_" + d["sezione"] + "_" + d["AY_norm"]
        self.pre, self.post = (d.drop_duplicates("UID") for d in (self.pre, self.post))

        cmp_map = {}
        for q, info in self.mapping.items():
            for c in info["competencies"]:
                cmp_map.setdefault(c, []).append(q)

        def agg(df):
            res = {c: df[qs].astype(float).mean(axis=1) for c, qs in cmp_map.items()}
            return pd.concat([pd.DataFrame(res),
                              df[["UID","school","is_etw","eta","classe","sezione"]]], axis=1)

        m = pd.merge(agg(self.pre), agg(self.post), on="UID", suffixes=("_pre","_post"))

        for col in ("is_etw","school","classe","sezione","eta"):
            m[col] = m.pop(f"{col}_pre") if f"{col}_pre" in m else m.pop(f"{col}_post")

        
        m.drop(columns=[c for c in m.columns
                        if c.endswith(("_pre","_post"))
                        and c.split("_")[0] in ("is_etw","school","classe","sezione","eta")],
               inplace=True, errors="ignore")

        
        prepost = [c for c in m.columns if c.endswith(("_pre","_post"))]
        m[prepost] = m[prepost].apply(pd.to_numeric, errors="coerce")
        m[prepost] = 1 + (m[prepost]-1)*self.SCALE

        entities = list(cmp_map.keys())
        for c in cmp_map:
            m[f"{c}_delta"] = m[f"{c}_post"] - m[f"{c}_pre"]

        for area, comps in AREA_MAP.items():
            entities.append(area)
            m[f"{area}_pre"]   = m[[f"{c}_pre"  for c in comps]].mean(axis=1)
            m[f"{area}_post"]  = m[[f"{c}_post" for c in comps]].mean(axis=1)
            m[f"{area}_delta"] = m[f"{area}_post"] - m[f"{area}_pre"]

        entities.insert(0, "CompetenzaChiave")
        m["CompetenzaChiave_pre"]   = m[[f"{a}_pre"  for a in AREA_MAP]].mean(axis=1)
        m["CompetenzaChiave_post"]  = m[[f"{a}_post" for a in AREA_MAP]].mean(axis=1)
        m["CompetenzaChiave_delta"] = m["CompetenzaChiave_post"] - m["CompetenzaChiave_pre"]

        return self._impute(m), entities



    @staticmethod
    def export_group_means(df: pd.DataFrame,
                           entities: list[str],
                           out_dir: str,
                           filename: str = "means_pre_post_delta_by_group.csv") -> pd.DataFrame:
        
        overall_col = "CompetenzaChiave"
        area_names = list(AREA_MAP.keys())
        specific_order = []
        for comps in AREA_MAP.values():
            specific_order.extend(comps)

        ordered = []
        seen = set()

        def _add(x):
            if x in entities and x not in seen:
                ordered.append(x)
                seen.add(x)

        _add(overall_col)
        for a in area_names:
            _add(a)
        for s in specific_order:
            _add(s)
        for e in entities:
            if e not in seen:
                ordered.append(e)
                seen.add(e)

        def _measure_label(m: str) -> str:
            if m == overall_col:
                return "Key competence (overall)"
            if m in area_names:
                return f"Area: {m}"
            return f"Specific competence: {m}"

        group_specs = [(True, "eTwinning"), (False, "Control")]
        group_order = {"eTwinning": 0, "Control": 1}
        measure_order = {m: i for i, m in enumerate(ordered)}

        rows = []
        for mname in ordered:
            pre_col = f"{mname}_pre"
            post_col = f"{mname}_post"
            delta_col = f"{mname}_delta"

            if pre_col not in df.columns or post_col not in df.columns:
                continue

            for flag, grp_label in group_specs:
                cols = [pre_col, post_col] + ([delta_col] if delta_col in df.columns else [])
                sub = df.loc[df["is_etw"] == flag, cols].copy()
                sub = sub.dropna(subset=[pre_col, post_col])
                n = int(sub.shape[0])

                if n == 0:
                    pre_m = post_m = delta_m = float("nan")
                else:
                    pre_m = float(sub[pre_col].mean())
                    post_m = float(sub[post_col].mean())
                    if delta_col in sub.columns:
                        delta_m = float(sub[delta_col].mean())
                    else:
                        delta_m = float((sub[post_col] - sub[pre_col]).mean())

                rows.append({
                    "Measure": _measure_label(mname),
                    "Group": grp_label,
                    "N": n,
                    "Pre_M": pre_m,
                    "Post_M": post_m,
                    "Delta_M": delta_m,
                    "__m_order": measure_order.get(mname, 10**9),
                    "__g_order": group_order.get(grp_label, 10**9),
                })

        out_df = pd.DataFrame(rows)
        if not out_df.empty:
            for c in ("Pre_M", "Post_M", "Delta_M"):
                if c in out_df.columns:
                    out_df[c] = out_df[c].round(3)
            out_df = (out_df
                      .sort_values(["__m_order", "__g_order"], kind="mergesort")
                      .drop(columns=["__m_order", "__g_order"]))

        if out_dir:
            os.makedirs(out_dir, exist_ok=True)
            out_df.to_csv(os.path.join(out_dir, filename), index=False)

        return out_df
    
    @staticmethod
    def _safe_kde(vals):
        try:
            return stats.gaussian_kde(vals) if len(np.unique(vals)) > 1 else None
        except Exception:
            return None

    @classmethod
    def kde_pair(cls, df, col, title, path):
        delta = "delta" in col
        xlim  = (-1.75, 1.75) if delta else (1, 8)
        ticks = np.linspace(*xlim, 5) if delta else cls.TICKS

        fig, axs = plt.subplots(2, 1, sharex=True, figsize=(5, 4))
        for ax, flag, lbl, color in zip(
                axs, [True, False], ["eTwinning", "Controllo"], ["tab:blue", "red"]):
            vals = df[col][df["is_etw"] == flag].dropna().astype(float)
            kde  = cls._safe_kde(vals)
            xs   = np.linspace(*xlim, 300)
            if kde is not None:
                ax.plot(xs, kde(xs), color=color)
            ax.set_title(lbl, fontsize=8)
            ax.set_xlim(*xlim); ax.set_xticks(ticks)
            if not delta:
                ax.set_xticklabels(cls.TICKLAB)
        fig.suptitle(title)
        fig.tight_layout(); plt.savefig(path); plt.close()

    
    @classmethod
    def bar_delta(cls, df, col, title, path):
        mean_etw  = df.loc[df["is_etw"],   col].mean()
        mean_ctrl = df.loc[~df["is_etw"],  col].mean()
        means = pd.Series([mean_etw, mean_ctrl],
                          index=["eTwinning", "Controllo"])

        max_abs = abs(means).max()
        if pd.isna(max_abs):                       
            return
        ylim = (-max_abs * 1.25, max_abs * 1.25)   

        plt.figure(figsize=(4, 4))
        means.plot(kind="bar", color=["tab:blue", "red"])
        plt.ylim(ylim)
        plt.ylabel("Δ medio (scala 1-8)")
        plt.title(title)
        plt.grid(ls=":", axis="y", alpha=.6)
        plt.tight_layout()
        plt.savefig(path, dpi=300)
        plt.close()
        
    
    @classmethod
    def box_plot(cls, df, col, title, path):
        
        data = [df[col][df["is_etw"] == flag].dropna().astype(float)
                for flag in (True, False)]

        plt.figure(figsize=(4, 4))
        plt.boxplot(data, positions=[1, 2], patch_artist=True,
                    boxprops=dict(color="k", facecolor="w"),
                    medianprops=dict(color="k", linewidth=2),
                    whiskerprops=dict(color="k"), capprops=dict(color="k"))

        for i, vals in enumerate(data, 1):
            plt.scatter(np.random.normal(i, 0.04, len(vals)),
                        vals, s=12, alpha=.5,
                        color=("tab:blue" if i == 1 else "red"))

        if "delta" not in col:
            plt.ylim(1, 8); plt.yticks(cls.TICKS, cls.TICKLAB)
        else:
            vmin, vmax = np.nanmin(np.hstack(data)), np.nanmax(np.hstack(data))
            pad = max((vmax - vmin) * 0.05, 0.1)
            plt.ylim(vmin - pad, vmax + pad); plt.ylabel("Δ livello (1-8)")

        plt.title(title); plt.tight_layout(); plt.savefig(path); plt.close()

    
    @classmethod
    def violin_plot(cls, df, col, title, path):
        
        data = [df[col][df["is_etw"] == flag].dropna().astype(float)
                for flag in (True, False)]

        fig, ax = plt.subplots(figsize=(4, 4))
        parts = ax.violinplot(data, positions=[1, 2],
                              showmeans=False, showextrema=False, widths=0.8)
        for pc, color in zip(parts["bodies"], ["tab:blue", "red"]):
            pc.set_facecolor(color); pc.set_alpha(0.40)

        ax.boxplot(data, positions=[1, 2], widths=0.18, showcaps=False,
                   boxprops=dict(color='k'), medianprops=dict(color='k'))

        for i, vals in enumerate(data, 1):
            ax.scatter(np.random.normal(i, 0.04, len(vals)),
                       vals, s=10, alpha=.6,
                       color=("tab:blue" if i == 1 else "red"))

        ax.set_xticks([1, 2]); ax.set_xticklabels(["eTwin", "Ctrl"])
        if "delta" in col:
            ax.set_ylabel("Δ livello (1-8)")
        else:
            ax.set_ylabel("Livello (1-8)")
            ax.set_yticks(cls.TICKS); ax.set_yticklabels(cls.TICKLAB)
        ax.set_title(title); fig.tight_layout(); plt.savefig(path); plt.close()

    
    @classmethod
    def dist_bar(cls,
                 df: pd.DataFrame,
                 col: str,
                 title: str,
                 path: str) -> None:
        
        fig, axs = plt.subplots(2, 1, figsize=(4, 4),
                                 sharex=True,
                                 gridspec_kw={"hspace": 0.30})

        for ax, flag, lbl, color in zip(
                axs,
                (True, False),                     
                ("eTwinning", "Controllo"),
                ("tab:blue",  "red")):

            vals = (df[col][df["is_etw"] == flag]
                    .dropna()
                    .round()
                    .astype(int))

            pct = (vals.value_counts(normalize=True)
                        .sort_index()
                        .reindex(range(1, 9), fill_value=0) * 100)

            bars = ax.bar(pct.index, pct.values,
                          color=color, alpha=0.60)

            ax.set_ylim(0, max(pct.values.max(), 5) * 1.20)
            ax.set_ylabel("%")
            ax.set_title(lbl, fontsize=9)

            
            for b, v in zip(bars, pct.values):
                if v:  
                    ax.text(b.get_x() + b.get_width()/2,
                            b.get_height() + 1,
                            f"{v:.1f}%",
                            ha="center", va="bottom", fontsize=8)

        axs[-1].set_xticks(cls.TICKS)
        axs[-1].set_xticklabels(cls.TICKLAB)
        fig.suptitle(title.replace("Likert", "livelli"))
        fig.tight_layout()
        plt.savefig(path, dpi=300)
        plt.close(fig)

    
    @staticmethod
    def _radar(ax, labels, vals_etw, vals_ctrl, title):
        
        import textwrap                               

        n       = len(labels)
        angles  = np.linspace(0, 2*np.pi, n, endpoint=False).tolist()
        angles += angles[:1]                          

        
        vmin, vmax = np.nanmin([*vals_etw, *vals_ctrl]), np.nanmax([*vals_etw, *vals_ctrl])
        shift      = -vmin if vmin < 0 else 0

        v_etw  = (np.array(vals_etw)  + shift).tolist() + [shift + vals_etw[0]]
        v_ctrl = (np.array(vals_ctrl) + shift).tolist() + [shift + vals_ctrl[0]]

        ax.set_ylim(0, (vmax + shift) * 1.05)         

        
        ax.set_theta_offset(np.pi/2)
        ax.set_theta_direction(-1)
        ax.grid(color="lightgray", ls=":", lw=.8)
        yticks = np.linspace(shift + vmin, shift + vmax, 5)
        ax.set_yticks(yticks)
        ax.set_yticklabels([f"{t - shift:.2f}" for t in yticks], fontsize=7)

        
        wrapped = [textwrap.fill(lab, width=22, break_long_words=False) for lab in labels]
        ax.set_xticks(angles[:-1])
        ax.set_xticklabels(wrapped, fontsize=8)

        
        ax.plot(angles, v_etw,  color="tab:blue", lw=1.3)
        ax.fill(angles, v_etw,  color="tab:blue", alpha=.80)
        ax.plot(angles, v_ctrl, color="red",      lw=1.3)
        ax.fill(angles, v_ctrl, color="red",      alpha=.80)

        ax.set_title(title, pad=25, fontsize=13)     

    
    @classmethod
    def radar_area(cls, df: pd.DataFrame, out_dir: str) -> None:
        import textwrap                              

        labs = list(AREA_MAP.keys())
        etw  = [df[f"{a}_delta"][df["is_etw"]].mean()  for a in labs]
        ctrl = [df[f"{a}_delta"][~df["is_etw"]].mean() for a in labs]

        fig, ax = plt.subplots(figsize=(6, 6),
                               subplot_kw={"projection": "polar"})
        cls._radar(ax, labs, etw, ctrl, "")          
        ax.set_title("Δ medio per area", pad=35)

        
        for tick, area in zip(ax.get_xticklabels(), labs):
            tick.set_color(AREA_COLOR[area])
            tick.set_text(textwrap.fill(tick.get_text(), 18))

        ax.legend(handles=[Patch(color="tab:blue", label="eTwinning"),
                           Patch(color="red",      label="Controllo")],
                  loc="upper right", bbox_to_anchor=(1.12, 1.12))
        fig.tight_layout()
        fig.savefig(os.path.join(out_dir, "radar_3aree.png"), dpi=300)
        plt.close(fig)

    
    @classmethod
    def radar_comp(cls, df: pd.DataFrame, out_dir: str) -> None:
        import textwrap                              

        
        labs = sum(AREA_MAP.values(), [])
        etw  = [df[f"{c}_delta"][df["is_etw"]].mean()  for c in labs]
        ctrl = [df[f"{c}_delta"][~df["is_etw"]].mean() for c in labs]

        fig, ax = plt.subplots(figsize=(8, 8),
                               subplot_kw={"projection": "polar"})
        cls._radar(ax, labs, etw, ctrl, "")          
        ax.set_title("Δ medio competenze (11)", pad=40)

        
        for tick, comp in zip(ax.get_xticklabels(), labs):
            area = next(a for a, lst in AREA_MAP.items() if comp in lst)
            tick.set_color(AREA_COLOR[area])
            tick.set_text(textwrap.fill(comp, 22))

        ax.legend(handles=[Patch(color="tab:blue", label="eTwinning"),
                           Patch(color="red",      label="Controllo")],
                  loc="upper right", bbox_to_anchor=(1.18, 1.18))
        fig.tight_layout()
        fig.savefig(os.path.join(out_dir, "radar_11comp.png"), dpi=300)
        plt.close(fig)

        
    @classmethod
    def run_pca(cls, df, ents, out_dir, dim=2):
        delta_cols = [f"{e}_delta" for e in ents]
        X = df[delta_cols].dropna()
        labels = df.loc[X.index, "is_etw"]

        pca = PCA().fit(X)
        evr = pca.explained_variance_ratio_
        cum = np.cumsum(evr)

        
        plt.figure(figsize=(4, 3))
        plt.bar(range(1, len(evr)+1), evr*100, color="grey")
        plt.plot(range(1, len(cum)+1), cum*100, marker="o")
        plt.axhline(80, color="r", ls="--")
        plt.ylabel("% varianza"); plt.xlabel("Componenti")
        plt.title("Scree plot PCA")
        plt.tight_layout(); plt.savefig(os.path.join(out_dir, "pca_scree.png")); plt.close()

        n_comp = max(dim, np.where(cum >= .8)[0][0] + 1)
        pca = PCA(n_components=n_comp).fit(X)
        comp = pca.transform(X)

        
        fig = plt.figure(figsize=(5, 4))
        if dim == 3:
            from mpl_toolkits.mplot3d import Axes3D  
            ax = fig.add_subplot(111, projection="3d")
            for flag, color, lbl in [(True, "tab:blue", "eTwinning"),
                                     (False, "red", "Controllo")]:
                pts = comp[labels == flag]
                ax.scatter(pts[:, 0], pts[:, 1], pts[:, 2],
                           label=lbl, alpha=.7, s=30, color=color)
            ax.set_zlabel("PC3")
        else:
            ax = fig.add_subplot(111)
            for flag, color, lbl in [(True, "tab:blue", "eTwinning"),
                                     (False, "red", "Controllo")]:
                pts = comp[labels == flag]
                ax.scatter(pts[:, 0], pts[:, 1],
                           label=lbl, alpha=.7, s=30, color=color)
        ax.set_xlabel("PC1"); ax.set_ylabel("PC2"); ax.legend()
        plt.title(f"PCA scatter ({dim}D)")
        plt.tight_layout(); plt.savefig(os.path.join(out_dir, f"pca_scatter_{dim}d.png")); plt.close()

        
        load_df = pd.DataFrame(pca.components_.T, index=delta_cols,
                               columns=[f"PC{i+1}" for i in range(n_comp)])
        load_df.to_csv(os.path.join(out_dir, "pca_loadings.csv"))

    @classmethod
    def run_mds(cls, df, ents, out_dir, dim=2):
        delta_cols = [f"{e}_delta" for e in ents]
        X = df[delta_cols].dropna()
        labels = df.loc[X.index, "is_etw"]

        mds = MDS(n_components=dim, dissimilarity="euclidean", random_state=1)
        coords = mds.fit_transform(X)

        fig = plt.figure(figsize=(5, 4))
        if dim == 3:
            from mpl_toolkits.mplot3d import Axes3D  
            ax = fig.add_subplot(111, projection="3d")
            for flag, color, lbl in [(True, "tab:blue", "eTwinning"),
                                     (False, "red", "Controllo")]:
                pts = coords[labels == flag]
                ax.scatter(pts[:, 0], pts[:, 1], pts[:, 2],
                           label=lbl, color=color, alpha=.7, s=30)
            ax.set_zlabel("MDS3")
        else:
            ax = fig.add_subplot(111)
            for flag, color, lbl in [(True, "tab:blue", "eTwinning"),
                                     (False, "red", "Controllo")]:
                pts = coords[labels == flag]
                ax.scatter(pts[:, 0], pts[:, 1],
                           label=lbl, color=color, alpha=.7, s=30)
        ax.set_xlabel("MDS1"); ax.set_ylabel("MDS2"); ax.legend()
        plt.title(f"MDS scatter ({dim}D)")
        plt.tight_layout(); plt.savefig(os.path.join(out_dir, f"mds_scatter_{dim}d.png")); plt.close()



def shapiro_francia(x):
    x = np.asarray(x, float)
    x = x[~np.isnan(x)]
    if len(x) < 5 or len(x) > 5000:
        stat, p = stats.normaltest(x, nan_policy="omit")
    else:
        stat, p = stats.shapiro(x)
    return stat, p


class App(tk.Tk):

    
    @staticmethod
    def _bh_fdr(pvals):
        import numpy as _np
        p = _np.asarray(pvals, float)
        if p.size == 0:
            return p
        n = (~_np.isnan(p)).sum()
        if n == 0:
            return p
        idx = _np.argsort(p, kind="mergesort")
        rank = _np.empty_like(idx); rank[idx] = _np.arange(1, len(p)+1)
        q = p * len(p) / rank
        q[idx[::-1]] = _np.minimum.accumulate(q[idx[::-1]])
        return _np.clip(q, 0, 1)

    
    @staticmethod
    def _audit_mapping(df_raw, mapping, out_dir):
        import numpy as _np, pandas as _pd, os as _os
        rows = []
        intervals = []
        for q, info in mapping.items():
            s = info["start_col"].strip().upper()
            ok = (s in df_raw.columns)
            if not ok:
                rows.append([q, s, "", ";".join(info.get("competencies", [])), False])
                continue
            i0 = df_raw.columns.get_loc(s)
            ok = (i0+4 < df_raw.shape[1])
            e  = df_raw.columns[i0+4] if ok else ""
            rows.append([q, s, e, ";".join(info.get("competencies", [])), ok])
            if ok:
                intervals.append((q, s, e, i0, i0+4))
        _pd.DataFrame(rows, columns=["question","start","end","competencies","valid_5cols"])          .to_csv(_os.path.join(out_dir, "mapping_audit.csv"), index=False)
        
        ov = []
        for i in range(len(intervals)):
            qi, si, ei, a, b = intervals[i]
            for j in range(i+1, len(intervals)):
                qj, sj, ej, c, d = intervals[j]
                if max(a,c) <= min(b,d):
                    ov.append([qi, qj, df_raw.columns[max(a,c)], df_raw.columns[min(b,d)]])
        if ov:
            _pd.DataFrame(ov, columns=["Q_i","Q_j","overlap_start","overlap_end"])              .to_csv(_os.path.join(out_dir, "mapping_overlaps.csv"), index=False)

    
    @staticmethod
    def _detect_multi_marks(df_raw, mapping, out_dir):
        import numpy as _np, pandas as _pd, os as _os
        warn = []
        for q, info in mapping.items():
            s = info["start_col"].strip().upper()
            if s not in df_raw.columns:
                continue
            i0 = df_raw.columns.get_loc(s)
            blk = df_raw.iloc[:, i0:i0+5]
            valid = blk.applymap(lambda x: isinstance(x, str) and x.strip() != "")
            cnt = valid.sum(axis=1)
            bad = cnt[cnt > 1]
            if not bad.empty:
                for idx, n in bad.items():
                    warn.append([q, int(idx), int(n)])
        if warn:
            _pd.DataFrame(warn, columns=["question","row_idx","n_marked"])              .to_csv(_os.path.join(out_dir, "scoring_warnings_multimark.csv"), index=False)

    
    def _two_stage(self, df, ents, out):
        import numpy as _np, pandas as _pd, os as _os
        from scipy.stats import t as _t
        rows = []
        for ent in ents:
            d = df[[f"{ent}_delta","school","is_etw"]].dropna()
            if d.empty:
                rows.append([ent, 0]+[ _np.nan ]*6); continue
            grp = d.groupby(["school","is_etw"])[f"{ent}_delta"].mean().unstack()
            if grp is None or grp.shape[1] < 2:
                rows.append([ent, 0]+[ _np.nan ]*6); continue
            inter = grp.dropna().dropna(axis=0, how="any")
            if inter.empty or inter.shape[0] < 2:
                rows.append([ent, 0]+[ _np.nan ]*6); continue
            diff = inter[True] - inter[False]
            n = len(diff); m = diff.mean(); s = diff.std(ddof=1)
            if n < 2 or not _np.isfinite(s) or s == 0:
                rows.append([ent, n, m, s, _np.nan, _np.nan, _np.nan, _np.nan]); continue
            t = m / (s / _np.sqrt(n))
            p = 2*(1 - _t.cdf(abs(t), df=n-1))
            z = _t.ppf(0.975, df=n-1)
            lo, hi = m - z*s/_np.sqrt(n), m + z*s/_np.sqrt(n)
            rows.append([ent, n, m, s, t, p, lo, hi])
        _pd.DataFrame(rows, columns=["entity","N_schools","mean_diff","SD","t","p","CI_low","CI_high"])           .to_csv(_os.path.join(out, "two_stage_school_summary.csv"), index=False)
    def __init__(self):
        super().__init__()
        self.title("eTwinning Analyzer")
        self.geometry("1080x580")
        self._build()

    
    def _build(self):
        self.paths = {k: tk.StringVar() for k in ("pre", "post", "map", "out")}
        
        
        self.paths["pre"].set(DEFAULT_PRE_CSV)
        self.paths["post"].set(DEFAULT_POST_CSV)
        self.paths["map"].set(_resolve_path(DEFAULT_MAPPING_JSON, DEFAULT_MAPPING_JSON_FALLBACK))
        self.paths["out"].set(DEFAULT_OUT_DIR)
        rows = [("CSV entrata", "pre"),
                ("CSV uscita", "post"),
                ("Mapping JSON", "map"),
                ("Cartella output", "out")]

        for i, (lab, k) in enumerate(rows):
            tk.Label(self, text=lab).grid(row=i, column=0, sticky="w", padx=6, pady=6)
            tk.Entry(self, textvariable=self.paths[k], width=80)              .grid(row=i, column=1, padx=4)
            tk.Button(self, text="…", command=lambda k=k: self._pick(k))              .grid(row=i, column=2)

        
        self.do_norm = tk.BooleanVar(value=True)
        self.do_t    = tk.BooleanVar(value=True)
        self.do_anc  = tk.BooleanVar(value=True)
        self.t_step  = tk.BooleanVar()

        opts = [("Test normalità", self.do_norm, 0),
                ("Welch / MW t-test", self.do_t, 1),
                ("ANCOVA", self.do_anc, 2),
                ("t-test step-wise", self.t_step, 3)]
        for txt, var, col in opts:
            tk.Checkbutton(self, text=txt, var=var)              .grid(row=4, column=col, sticky="w", padx=6)

        
        tk.Label(self, text="Riduzione dimensioni:")          .grid(row=5, column=0, sticky="e", padx=6)
        self.do_pca = tk.BooleanVar()
        self.do_mds = tk.BooleanVar()
        tk.Checkbutton(self, text="PCA", var=self.do_pca)          .grid(row=5, column=1, sticky="w")
        tk.Checkbutton(self, text="MDS", var=self.do_mds)          .grid(row=5, column=2, sticky="w")
        self.dim = tk.IntVar(value=2)
        for i, d in enumerate((2, 3)):
            tk.Radiobutton(self, text=f"{d}D", variable=self.dim, value=d)              .grid(row=5, column=3 + i, sticky="w")

        
        tk.Label(self, text="Covariate ANCOVA:")          .grid(row=6, column=0, sticky="e", padx=6)
        self.cov_eta = tk.BooleanVar()
        self.cov_scu = tk.BooleanVar()
        tk.Checkbutton(self, text="Età", var=self.cov_eta)          .grid(row=6, column=1, sticky="w")
        tk.Checkbutton(self, text="Scuola", var=self.cov_scu)          .grid(row=6, column=2, sticky="w")
        
        
        tk.Label(self, text="Outlier negativo da rimuovere").grid(row=7, column=0, sticky="e", padx=6)
        self.n_out_neg = tk.IntVar(value=0)
        tk.Entry(self, textvariable=self.n_out_neg, width=5).grid(row=7, column=1, sticky="w")

        tk.Label(self, text="Ambito neg:").grid(row=7, column=3, sticky="e", padx=6)
        self.ambito_neg = tk.StringVar(value="globale")
        for i, txt in enumerate(("globale", "eTwinning", "controllo")):
            tk.Radiobutton(self, text=txt, variable=self.ambito_neg, value=txt)              .grid(row=7, column=4 + i, sticky="w")

        
        tk.Label(self, text="Outlier positivo da rimuovere").grid(row=8, column=0, sticky="e", padx=6)
        self.n_out_pos = tk.IntVar(value=0)
        tk.Entry(self, textvariable=self.n_out_pos, width=5).grid(row=8, column=1, sticky="w")

        tk.Label(self, text="Ambito pos:").grid(row=8, column=3, sticky="e", padx=6)
        self.ambito_pos = tk.StringVar(value="globale")
        for i, txt in enumerate(("globale", "eTwinning", "controllo")):
            tk.Radiobutton(self, text=txt, variable=self.ambito_pos, value=txt)              .grid(row=8, column=4 + i, sticky="w")

        
        tk.Button(self, text="Processa", command=self._run)          .grid(row=9, column=0, columnspan=6, pady=20)

    def _pick(self, key):
        p = filedialog.askopenfilename() if key != "out" else filedialog.askdirectory()
        if p:
            self.paths[key].set(p)

    
    def _run(self):
        if not all(self.paths[k].get() for k in self.paths):
            messagebox.showerror("Errore", "Seleziona tutti i file")
            return

        
        mapping = json.load(open(self.paths["map"].get(), encoding="utf-8"))
        pre_ld  = DataLoader(self.paths["pre"].get(),  INVERTED_START_COLS)
        post_ld = DataLoader(self.paths["post"].get(), INVERTED_START_COLS)
        pre_raw  = pre_ld.load_raw()
        post_raw = post_ld.load_raw()
        out_dir = self.paths["out"].get()
        os.makedirs(out_dir, exist_ok=True)


        pre_num  = pre_ld.likert_numeric(pre_raw,  mapping)
        post_num = post_ld.likert_numeric(post_raw, mapping)

        merged, ents = Analyzer(
            pre_num,
            post_num,
            mapping
        ).process()

        
        trimmed = self._trim_outliers(merged.copy(), ents)

        
        try:
            self.export_questionnaires_without_outliers(
                pre_num=pre_num,
                post_num=post_num,
                keep_uids=trimmed["UID"],
                out_dir=out_dir
            )
        except Exception as e:
            print(f"[WARN] export_questionnaires_without_outliers failed: {e}")



        
        out_dir = self.paths["out"].get()
        os.makedirs(out_dir, exist_ok=True)
        trimmed.to_csv(os.path.join(out_dir, "results_competenza_imprenditoriale.csv"),
                       index=False)

        
        
        
        try:
            Analyzer.export_group_means(trimmed, ents, out_dir)
        except Exception:
            pass
        
        for ent in ents:
            Analyzer.kde_pair(trimmed, f"{ent}_pre",   f"{ent} – PRE",
                              os.path.join(out_dir, f"{ent}_pre_kde.png"))
            Analyzer.kde_pair(trimmed, f"{ent}_post",  f"{ent} – POST",
                              os.path.join(out_dir, f"{ent}_post_kde.png"))
            Analyzer.kde_pair(trimmed, f"{ent}_delta", f"{ent} – Δ",
                              os.path.join(out_dir, f"{ent}_delta_kde.png"))

            Analyzer.bar_delta(trimmed, f"{ent}_delta", f"Δ medio – {ent}",
                               os.path.join(out_dir, f"{ent}_delta_bar.png"))

            Analyzer.box_plot(trimmed, f"{ent}_delta", f"{ent} – Δ box-plot",
                              os.path.join(out_dir, f"{ent}_delta_box.png"))

            Analyzer.violin_plot(trimmed, f"{ent}_delta", f"{ent} – Δ raincloud",
                                 os.path.join(out_dir, f"{ent}_delta_violin.png"))

            Analyzer.dist_bar(trimmed, f"{ent}_pre",
                              f"{ent} – distribuzione livelli PRE",
                              os.path.join(out_dir, f"{ent}_pre_dist.png"))
            Analyzer.dist_bar(trimmed, f"{ent}_post",
                              f"{ent} – distribuzione livelli POST",
                              os.path.join(out_dir, f"{ent}_post_dist.png"))

        
        
        Analyzer.radar_area(trimmed, out_dir)   
        Analyzer.radar_comp(trimmed, out_dir)   


        
        dim = self.dim.get()
        if self.do_pca.get():
            Analyzer.run_pca(trimmed, ents, out_dir, dim=dim)
        if self.do_mds.get():
            Analyzer.run_mds(trimmed, ents, out_dir, dim=dim)

        
        if self.do_norm.get():
            self._normality(trimmed, ents, out_dir)
        if self.do_t.get():
            self._welch_MW(trimmed, ents, out_dir)
        if self.t_step.get():
            self._ttest_step(trimmed, ents, out_dir)
        if self.do_anc.get():
            self._ancova(trimmed, ents, out_dir)

    
        
        try:
            self.export_school_deltas(trimmed, out_dir)
            self.export_school_audit(pre_num, post_num, out_dir)
        except Exception as e:
            
            print(f"[WARN] schools export failed: {e}")
    def export_school_deltas(self, df: pd.DataFrame,
                             out_dir: str,
                             filename_csv: str = "summary_deltas_by_school.csv",
                             filename_txt: str = "summary_deltas_by_school.txt",
                             per_school: bool = True) -> bool:
        
        if "school" not in df.columns:
            return False
        
        school_series = df["school"].fillna("").astype(str).str.strip()
        if (school_series == "").all():
            return False

        schools_dir = os.path.join(out_dir, "schools")
        os.makedirs(schools_dir, exist_ok=True)

        work = df.copy()
        work["school"] = school_series
        work = work[work["school"] != ""]
        if work.empty:
            return False

        work["group"] = work["is_etw"].apply(lambda v: "ETW" if bool(v) else "CTRL")

        agg = (work.groupby(["school","group"], dropna=False)
                    .agg(N=("UID","size"),
                         Pre_Mean=("CompetenzaChiave_pre","mean"),
                         Post_Mean=("CompetenzaChiave_post","mean"),
                         Delta_Mean=("CompetenzaChiave_delta","mean"),
                         Delta_SD=("CompetenzaChiave_delta","std"))
                    .reset_index())
        agg = agg.sort_values(["school","group"])

        csv_path = os.path.join(schools_dir, filename_csv)
        agg.to_csv(csv_path, index=False)

        
        txt_path = os.path.join(schools_dir, filename_txt)
        with open(txt_path, "w", encoding="utf-8") as f:
            f.write("Per-school deltas for CompetenzaChiave (within this competence)\n")
            f.write("Complete-case UID-matched only.\n\n")
            for school in sorted(agg["school"].unique()):
                f.write(f"=== {school} ===\n")
                sub = agg[agg["school"] == school].set_index("group")
                for g in ["ETW","CTRL"]:
                    if g in sub.index:
                        r = sub.loc[g]
                        f.write(
                            f"{g}: N={int(r.N)}  Pre={r.Pre_Mean:.3f}  Post={r.Post_Mean:.3f}  "
                            f"Delta={r.Delta_Mean:.3f}  SD={0.0 if pd.isna(r.Delta_SD) else r.Delta_SD:.3f}\n"
                        )
                    else:
                        f.write(f"{g}: N=0 (no matched data)\n")
                f.write("\n")

        if per_school:
            for school in agg["school"].unique():
                safe = Analyzer._sanitize_filename(school)
                sub = agg[agg["school"] == school].copy()
                sub.to_csv(os.path.join(schools_dir, f"{safe}__deltas.csv"), index=False)
                with open(os.path.join(schools_dir, f"{safe}__deltas.txt"), "w", encoding="utf-8") as f:
                    f.write(f"{school}\n")
                    for _, r in sub.sort_values("group").iterrows():
                        f.write(
                            f"{r.group}: N={int(r.N)}  Pre={r.Pre_Mean:.3f}  Post={r.Post_Mean:.3f}  "
                            f"Delta={r.Delta_Mean:.3f}  SD={0.0 if pd.isna(r.Delta_SD) else r.Delta_SD:.3f}\n"
                        )

        return True

    @staticmethod
    def export_school_audit(pre_df: pd.DataFrame,
                            post_df: pd.DataFrame,
                            out_dir: str,
                            filename_txt: str = "audit_school_selection.txt",
                            filename_csv: str = "audit_school_mismatches.csv") -> bool:
        
        if "school" not in pre_df.columns or "school" not in post_df.columns:
            return False

        schools_dir = os.path.join(out_dir, "schools")
        os.makedirs(schools_dir, exist_ok=True)

        def _norm_ay(x):
            if pd.isna(x):
                return ""
            s = str(x).strip()
            if s.endswith(".0"):
                s = s[:-2]
            s = re.sub(r"\s+", "", s)
            return s

        def _prep(d: pd.DataFrame) -> pd.DataFrame:
            dd = d.copy()
            dd["AY_norm"] = dd.get("AY", "").map(_norm_ay)
            dd["grp"] = dd.get("is_etw", False).map(lambda v: "ETW" if bool(v) else "CTRL")
            for c in ("classe","sezione"):
                if c not in dd.columns:
                    dd[c] = ""
                dd[c] = dd[c].fillna("").astype(str)
            dd["school"] = dd["school"].fillna("").astype(str).str.strip()
            dd["key"] = dd["grp"] + "_" + dd["classe"] + "_" + dd["sezione"] + "_" + dd["AY_norm"]
            return dd

        pre = _prep(pre_df)
        post = _prep(post_df)

        blank_pre = int((pre["school"] == "").sum())
        blank_post = int((post["school"] == "").sum())

        
        pre_multi = pre.groupby("key")["school"].nunique()
        post_multi = post.groupby("key")["school"].nunique()
        n_pre_multi = int((pre_multi > 1).sum())
        n_post_multi = int((post_multi > 1).sum())

        pre_one = pre[pre["school"] != ""].drop_duplicates("key")[["key","grp","classe","sezione","AY_norm","school"]].rename(columns={"school":"school_pre"})
        post_one = post[post["school"] != ""].drop_duplicates("key")[["key","school"]].rename(columns={"school":"school_post"})
        mm = pre_one.merge(post_one, on="key", how="inner")
        mm = mm[mm["school_pre"] != mm["school_post"]].copy()
        mm = mm.sort_values(["grp","classe","sezione","AY_norm"])

        
        mm.to_csv(os.path.join(schools_dir, filename_csv), index=False)

        
        with open(os.path.join(schools_dir, filename_txt), "w", encoding="utf-8") as f:
            f.write("School-selection audit (diagnostic only)\n")
            f.write("Key = grp + classe + sezione + AY_norm (school excluded on purpose).\n\n")
            f.write(f"Blank school entries: pre={blank_pre}  post={blank_post}\n")
            f.write(f"Keys mapped to multiple schools within pre:  {n_pre_multi}\n")
            f.write(f"Keys mapped to multiple schools within post: {n_post_multi}\n")
            f.write(f"Pre vs post keys with different school selected: {len(mm)}\n\n")
            if len(mm):
                f.write("Examples (first 30):\n")
                for _, r in mm.head(30).iterrows():
                    f.write(f"- {r.key}: pre={r.school_pre}  post={r.school_post}\n")

        return True


    def _trim_outliers(self, df, ents):
        return df



    def export_questionnaires_without_outliers(
        self,
        pre_num: pd.DataFrame,
        post_num: pd.DataFrame,
        keep_uids: pd.Series,
        out_dir: str,
        pre_name: str = "QUESTIONARIO_PRE_no_outliers.csv",
        post_name: str = "QUESTIONARIO_POST_no_outliers.csv",
    ) -> None:
        os.makedirs(out_dir, exist_ok=True)
        pre_num.to_csv(os.path.join(out_dir, pre_name), index=False)
        post_num.to_csv(os.path.join(out_dir, post_name), index=False)
        return None

    def _trim_section(self, df, ents, n, scope, positive):
        return df


        
        if scope == "eTwinning":
            sub = df[df["is_etw"]]
        elif scope == "controllo":
            sub = df[~df["is_etw"]]
        else:
            sub = df

        col = "CompetenzaChiave_delta"
        
        if positive:
            idx = sub[col].nlargest(min(n, len(sub))).index
        else:
            idx = sub[col].nsmallest(min(n, len(sub))).index

        
        return df.drop(idx).reset_index(drop=True)

    
    def _normality(self, df, ents, out):
        rows = []
        for ent in ents:
            for grp, lbl in [(True, "etw"), (False, "ctrl")]:
                vals = df[f"{ent}_delta"][df["is_etw"] == grp].dropna().astype(float)
                if len(vals) < 8:
                    rows.append([ent, lbl, np.nan, np.nan]); continue
                w, p = shapiro_francia(vals)
                rows.append([ent, lbl, w, p])
        pd.DataFrame(rows, columns=["entity", "group", "W", "p"])          .to_csv(os.path.join(out, "normality_summary.csv"), index=False)

    
    def _welch_MW(self, df, ents, out):
        rows = []
        for ent in ents:
            etw  = df[f"{ent}_delta"][df["is_etw"]].dropna().astype(float)
            ctrl = df[f"{ent}_delta"][~df["is_etw"]].dropna().astype(float)
            if min(len(etw), len(ctrl)) < 3:
                rows.append([ent, len(etw), len(ctrl), "–", "–", "–", np.nan, np.nan, np.nan]); continue
            gauss = (shapiro_francia(etw)[1] > .05) and (shapiro_francia(ctrl)[1] > .05)
            if gauss:
                stat, p = stats.ttest_ind(etw, ctrl, equal_var=False)
                test = "Welch"
            else:
                stat, p = stats.mannwhitneyu(etw, ctrl, alternative="two-sided")
                test = "Mann-Whitney"
            g, lo, hi = (hedges_g_ci(etw, ctrl) if gauss else (np.nan, np.nan, np.nan))
            sig = "sig" if p < .05 else "n.s."
            rows.append([ent, len(etw), len(ctrl), test, stat, p, sig, g, lo, hi])
        pd.DataFrame(rows, columns=["entity", "N_etw", "N_ctrl", "test",
                                    "statistic", "p", "sig/NS",
                                    "g", "CI_low", "CI_high"])          .to_csv(os.path.join(out, "Welch_MW_summary.csv"), index=False)

    
    def _ttest_step(self, df, ents, out):
        thresholds = [0.20, 0.40, 0.60, 0.80]
        rec = []
        for ent in ents:
            for th in thresholds:
                subset = df[np.abs(df[f"{ent}_delta"]) >= th]
                etw  = subset[f"{ent}_delta"][subset["is_etw"]]
                ctrl = subset[f"{ent}_delta"][~subset["is_etw"]]
                if min(len(etw), len(ctrl)) < 3:
                    rec.append([ent, th, np.nan, np.nan, np.nan, "–"]); continue
                t, p = stats.ttest_ind(etw, ctrl, equal_var=False)
                sig = "sig" if p < .05 else "n.s."
                rec.append([ent, th, len(etw)+len(ctrl), t, p, sig])
        pd.DataFrame(rec, columns=["entity", "cutoff", "N", "t", "p", "sig/NS"])          .to_csv(os.path.join(out, "ttest_stepwise.csv"), index=False)

    
    def _ancova(self, df, ents, out):
        cov = []
        if self.cov_eta.get(): cov.append("eta")
        if self.cov_scu.get(): cov.append("C(school)")
        covs = " + ".join(cov) if cov else ""
        rows = []
        for ent in ents:
            form = f'{q(ent+"_post")} ~ is_etw + {q(ent+"_pre")}' + (" + " + covs if covs else "")
            need = [f"{ent}_post", f"{ent}_pre", "is_etw"] +                   [c.split("(")[-1].rstrip(")") for c in cov]
            dft  = df[need].dropna()
            if len(dft) < 5 or dft["is_etw"].nunique() < 2:
                rows.append([ent]+[np.nan]*9); continue

            mdl  = smf.ols(form, dft).fit()
            coef = "is_etw[T.True]" if "is_etw[T.True]" in mdl.params else "is_etw"
            b, se = mdl.params[coef], mdl.bse[coef]
            t, p  = b / se, mdl.pvalues[coef]

            eta   = partial_eta_squared(mdl, term="is_etw")   
            rows.append([ent, len(dft), b, se, t, p,
                         "sig" if p < .05 else "n.s.",
                         eta, np.nan, np.nan])                

        pd.DataFrame(rows, columns=["entity","N","beta","SE","t","p","sig/NS",
                                    "partial_eta2","CI_low","CI_high"])          .to_csv(os.path.join(out,"ancova_summary.csv"), index=False)


if __name__ == "__main__":
    App().mainloop()
