import os
os.environ['OMP_NUM_THREADS'] = '1'
os.environ['OPENBLAS_NUM_THREADS'] = '1'
os.environ['MKL_NUM_THREADS'] = '1'
os.environ['VECLIB_MAXIMUM_THREADS'] = '1'
import json, os, re, warnings, tkinter as tk
from dataclasses import dataclass, field
from tkinter import filedialog, messagebox
DEFAULT_PRE_CSV = '/Users/jacopocondo/Desktop/Indire/DATI FINALI/1 - questionari/entrata/QUESTIONARIO PER LA VALUTAZIONE DELLE COMPETENZE CHIAVE in entrata pt.5.csv'
DEFAULT_POST_CSV = '/Users/jacopocondo/Desktop/Indire/DATI FINALI/1 - questionari/uscita/QUESTIONARIO PER LA VALUTAZIONE DELLE COMPETENZE CHIAVE in uscita pt.5 Maggio.csv'
DEFAULT_MAPPING_JSON = '/Users/jacopocondo/Desktop/Indire/DATI FINALI/script/support material/QUANTITATIVO/COMP5 digitale/mapping_competenza_digitale.json'
DEFAULT_MAPPING_JSON_FALLBACK = ''
DEFAULT_OUT_DIR = '/Users/jacopocondo/Desktop/Indire/DATI FINALI/OUTPUT_ANALISI/COMP5_digitale'

def _resolve_path(primary: str, fallback: str | None=None) -> str:
    try:
        if primary and os.path.exists(primary):
            return primary
        if fallback and os.path.exists(fallback):
            return fallback
    except Exception:
        pass
    return primary if primary else fallback or ''
import os
os.environ['OMP_NUM_THREADS'] = '1'
os.environ['OPENBLAS_NUM_THREADS'] = '1'
os.environ['MKL_NUM_THREADS'] = '1'

def eta_boot_ci(df, formula, term='is_etw', n_boot=100, alpha=0.05):
    etas = []
    for _ in range(n_boot):
        d = df.sample(frac=1, replace=True)
        try:
            mdl = smf.ols(formula, d).fit()
            aov = anova_lm(mdl, typ=2)
            ss = aov.at[term, 'sum_sq']
            sse = aov.at['Residual', 'sum_sq']
            etas.append(ss / (ss + sse))
        except Exception:
            continue
    if len(etas) < 20:
        return (np.nan, np.nan)
    lo, hi = np.percentile(etas, [100 * alpha / 2, 100 * (1 - alpha / 2)])
    return (lo, hi)
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.patches import Patch
from scipy import stats
from scipy.stats import norm
import statsmodels.formula.api as smf
from statsmodels.stats.anova import anova_lm
from statsmodels.stats.multitest import multipletests
from sklearn.decomposition import PCA
from sklearn.manifold import MDS
import textwrap
warnings.filterwarnings('ignore', category=FutureWarning, module='pandas')
plt.rcParams['figure.dpi'] = 110
warnings.filterwarnings('ignore', message='This figure includes Axes that are not compatible with tight_layout')
warnings.filterwarnings('ignore', message='covariance of constraints', module='statsmodels')
warnings.filterwarnings('ignore', message='invalid value encountered in divide', module='statsmodels')

def hedges_g_ci(x, y, alpha=0.05):
    x, y = (np.asarray(x, float), np.asarray(y, float))
    n1, n2 = (len(x), len(y))
    if min(n1, n2) < 2:
        return (np.nan, np.nan, np.nan)
    m1, m2 = (x.mean(), y.mean())
    s1, s2 = (x.std(ddof=1), y.std(ddof=1))
    s_pooled = np.sqrt(((n1 - 1) * s1 ** 2 + (n2 - 1) * s2 ** 2) / (n1 + n2 - 2))
    d = (m1 - m2) / s_pooled
    J = 1 - 3 / (4 * (n1 + n2) - 9)
    g = d * J
    se = np.sqrt((n1 + n2) / (n1 * n2) + g ** 2 / (2 * (n1 + n2 - 2))) * J
    z = norm.ppf(1 - alpha / 2)
    return (g, g - z * se, g + z * se)

def partial_eta_squared(model, term='is_etw'):
    aov = anova_lm(model, typ=2)
    ss_eff = aov.loc[term, 'sum_sq']
    ss_err = aov.loc['Residual', 'sum_sq']
    return ss_eff / (ss_eff + ss_err)

def eta_boot_ci(df, formula, term='is_etw', n_boot=100, alpha=0.05):
    etas = []
    for _ in range(n_boot):
        d = df.sample(frac=1, replace=True)
        try:
            mdl = smf.ols(formula, d).fit()
            aov = anova_lm(mdl, typ=2)
            ss = aov.at[term, 'sum_sq']
            sse = aov.at['Residual', 'sum_sq']
            etas.append(ss / (ss + sse))
        except Exception:
            continue
    if len(etas) < 20:
        return (np.nan, np.nan)
    lo, hi = np.percentile(etas, [100 * alpha / 2, 100 * (1 - alpha / 2)])
    return (lo, hi)

def forest_plot(df, eff, lo, hi, *, title, path):
    df = df.sort_values(eff)
    y = np.arange(len(df))
    plt.figure(figsize=(6, 0.45 * len(df) + 1.5))
    plt.errorbar(df[eff], y, xerr=[df[eff] - df[lo], df[hi] - df[eff]], fmt='o', color='k', capsize=3)
    plt.axvline(0, color='grey', ls='--')
    plt.yticks(y, df['entity'])
    plt.xlabel('Hedges g (95 % CI)')
    plt.title(title)
    plt.tight_layout()
    plt.savefig(path)
    plt.close()
AREA_MAP: dict[str, list[str]] = {'Alfabetizzazione su informazioni e dati': ['Navigare, ricercare e filtrare dati, informazioni e contenuti digitali.', 'Valutare dati, informazioni e contenuti digitali.', 'Gestire dati, informazioni e contenuti digitali.'], 'Comunicazione e collaborazione': ['Interagire con gli altri attraverso le tecnologie.', 'Condividere informazioni attraverso le tecnologie digitali.', 'Esercitare la cittadinanza attraverso le tecnologie digitali.', 'Collaborare attraverso le tecnologie digitali.', 'Netiquette.', 'Gestire l’identità digitale.'], 'Creazione di contenuti digitali': ['Sviluppare contenuti digitali.', 'Integrare e rielaborare contenuti digitali.', 'Copyright e licenze.', 'Programmazione.'], 'Sicurezza': ['Proteggere i dispositivi.', 'Proteggere i dati personali e la privacy.', 'Proteggere la salute e il benessere.', 'Proteggere l’ambiente.'], 'Risolvere problemi': ['Risolvere problemi tecnici.', 'Individuare bisogni e risposte tecnologiche.', 'Utilizzare in modo creativo le tecnologie digitali.', 'Individuare i divari di competenza digitale.']}
AREA_COLOR: dict[str, str] = {'Alfabetizzazione su informazioni e dati': 'navy', 'Comunicazione e collaborazione': 'darkorange', 'Creazione di contenuti digitali': 'darkgreen', 'Sicurezza': 'purple', 'Risolvere problemi': 'teal'}
AGGR_LABEL: str = 'Competenza digitale'
SCHOOL_COLS: list[str] = [chr(c) for c in range(ord('J'), ord('X') + 1)]
SEZIONE_COLS: list[str] = ['AB', 'AC', 'AD', 'AE', 'AF', 'AG', 'AH', 'AI', 'AJ', 'AK', 'AL', 'AM', 'AN', 'AO', 'AP', 'AQ', 'AR', 'AS', 'AT', 'AU', 'AV']
INVERTED_START_COLS: list[str] = ['BO', 'CN', 'CT', 'DI', 'DN', 'KP', 'MU', 'HE', 'LL', 'LQ', 'JI']
ACCENT_SUB: dict[str, str] = {'Ã¨': 'è', 'Ã©': 'é', 'Ã': 'à', 'Ã¹': 'ù', 'â€™': '’'}

def fix_enc(s: str) -> str:
    if not isinstance(s, str):
        return s
    return re.sub('|'.join(map(re.escape, ACCENT_SUB)), lambda m: ACCENT_SUB[m.group(0)], s)

def excel_col(i: int) -> str:
    out = ''
    while True:
        i, r = divmod(i, 26)
        out = chr(r + 65) + out
        if i == 0:
            break
        i -= 1
    return out
gen_cols = lambda n: [excel_col(i) for i in range(n)]
q = lambda name: f'Q("{name}")'

@dataclass
class DataLoader:
    path: str
    inverted: list[str]
    school_map: dict = field(init=False)

    def load_raw(self):
        df = pd.read_csv(self.path, header=None, dtype=str)
        df.columns = gen_cols(df.shape[1])
        desc = df.iloc[1].apply(fix_enc)
        self.school_map = {c: desc[c] for c in SCHOOL_COLS}
        df = df.drop(index=[0, 1], errors='ignore').reset_index(drop=True).apply(lambda c: c.map(fix_enc))
        return df

    def _score(self, block, rev):

        def s(r):
            first = r.first_valid_index()
            if first is None:
                return np.nan
            v = block.columns.get_loc(first) + 1
            return 6 - v if rev else v
        return block.apply(s, axis=1)

    def likert_numeric(self, df, mapping):
        scored = {}
        for q, info in mapping.items():
            start = info['start_col']
            rev = start in self.inverted
            i0 = df.columns.get_loc(start)
            scored[q] = self._score(df.iloc[:, i0:i0 + 5], rev)
        out = pd.DataFrame(scored)
        out['school'] = df.apply(lambda r: next((self.school_map[c] for c in SCHOOL_COLS if pd.notna(r[c])), 'SCONOSCIUTA'), axis=1)
        out['is_etw'] = df['AW'].eq('Si').fillna(False)
        out['AY'] = df['AY']

        def _sezione(row):
            for col in SEZIONE_COLS:
                v = str(row.get(col, '')).strip()
                if v and v.lower() not in ('nan', 'none'):
                    return v
            return np.nan
        out['sezione'] = df.apply(_sezione, axis=1)

        def _classe(row):
            for col, lbl in (('Y', 'prima'), ('Z', 'seconda'), ('AA', 'terza')):
                v = str(row.get(col, '')).strip().lower()
                if v in ('prima', 'seconda', 'terza'):
                    return lbl
            return np.nan
        out['classe'] = df.apply(_classe, axis=1)
        out['eta'] = out['classe'].map({'prima': 15, 'seconda': 16, 'terza': 17})
        return out

@dataclass
class Analyzer:
    pre: pd.DataFrame
    post: pd.DataFrame
    mapping: dict
    SCALE = 7 / 4
    TICKS = list(range(1, 9))
    TICKLAB = ['A1', 'A2', 'B1', 'B2', 'C1', 'C2', 'D1', 'D2']

    @staticmethod
    def _impute(df):
        if 'is_etw' not in df.columns:
            return df
        out = df.copy()
        prepost = [c for c in df.columns if c.endswith('_pre') or c.endswith('_post')]
        for grp, sub in df.groupby('is_etw'):
            for col in prepost:
                m = sub[col].astype(float).mean(skipna=True)
                if m == m:
                    out.loc[sub.index, col] = sub[col].astype(float).fillna(m)
        return out

    def process(self):

        def _norm_ay(x):
            if pd.isna(x):
                return ''
            s = str(x).strip()
            if s.endswith('.0'):
                s = s[:-2]
            s = re.sub('\\s+', '', s)
            return s
        for d in (self.pre, self.post):
            d['AY_norm'] = d['AY'].map(_norm_ay)
            d['grp'] = d['is_etw'].map(lambda v: 'ETW' if bool(v) else 'CTL')
            for c in ('school', 'classe', 'sezione'):
                if c in d.columns:
                    d[c] = d[c].fillna('').astype(str)
                else:
                    d[c] = ''
            d['UID'] = d['school'] + '_' + d['grp'] + '_' + d['classe'] + '_' + d['sezione'] + '_' + d['AY_norm']
        self.pre, self.post = (d.drop_duplicates('UID') for d in (self.pre, self.post))
        cmp_map = {}
        for q, info in self.mapping.items():
            for c in info['competencies']:
                cmp_map.setdefault(c, []).append(q)

        def agg(df):
            res = {c: df[qs].astype(float).mean(axis=1) for c, qs in cmp_map.items()}
            return pd.concat([pd.DataFrame(res), df[['UID', 'school', 'is_etw', 'eta', 'classe', 'sezione']]], axis=1)
        m = pd.merge(agg(self.pre), agg(self.post), on='UID', suffixes=('_pre', '_post'))
        for col in ('is_etw', 'school', 'classe', 'sezione', 'eta'):
            m[col] = m.pop(f'{col}_pre') if f'{col}_pre' in m else m.pop(f'{col}_post')
        m.drop(columns=[c for c in m.columns if c.endswith(('_pre', '_post')) and c.split('_')[0] in ('is_etw', 'school', 'classe', 'sezione', 'eta')], inplace=True, errors='ignore')
        prepost = [c for c in m.columns if c.endswith(('_pre', '_post'))]
        m[prepost] = m[prepost].apply(pd.to_numeric, errors='coerce')
        m[prepost] = 1 + (m[prepost] - 1) * self.SCALE
        entities = list(cmp_map.keys())
        for c in cmp_map:
            m[f'{c}_delta'] = m[f'{c}_post'] - m[f'{c}_pre']
        for area, comps in AREA_MAP.items():
            entities.append(area)
            m[f'{area}_pre'] = m[[f'{c}_pre' for c in comps]].mean(axis=1)
            m[f'{area}_post'] = m[[f'{c}_post' for c in comps]].mean(axis=1)
            m[f'{area}_delta'] = m[f'{area}_post'] - m[f'{area}_pre']
        entities.insert(0, AGGR_LABEL)
        m[f'{AGGR_LABEL}_pre'] = m[[f'{a}_pre' for a in AREA_MAP]].mean(axis=1)
        m[f'{AGGR_LABEL}_post'] = m[[f'{a}_post' for a in AREA_MAP]].mean(axis=1)
        m[f'{AGGR_LABEL}_delta'] = m[f'{AGGR_LABEL}_post'] - m[f'{AGGR_LABEL}_pre']
        return (self._impute(m), entities)

    @staticmethod
    def export_group_means(df: pd.DataFrame, entities: list[str], out_dir: str, filename: str='means_pre_post_delta_by_group.csv') -> pd.DataFrame:
        overall_col = 'CompetenzaChiave'
        if overall_col not in entities:
            try:
                if 'AGGR_LABEL' in globals() and (AGGR_LABEL in entities or f'{AGGR_LABEL}_pre' in df.columns):
                    overall_col = AGGR_LABEL
            except Exception:
                pass
        area_names = list(AREA_MAP.keys())
        specific_order = []
        for comps in AREA_MAP.values():
            specific_order.extend(comps)
        ordered = []
        seen = set()

        def _add(x):
            if x in entities and x not in seen:
                ordered.append(x)
                seen.add(x)
        _add(overall_col)
        for a in area_names:
            _add(a)
        for s in specific_order:
            _add(s)
        for e in entities:
            if e not in seen:
                ordered.append(e)
                seen.add(e).add(e)

        def _measure_label(m: str) -> str:
            if m == overall_col:
                return 'Key competence (overall)'
            if m in area_names:
                return f'Area: {m}'
            return f'Specific competence: {m}'
        group_specs = [(True, 'eTwinning'), (False, 'Control')]
        group_order = {'eTwinning': 0, 'Control': 1}
        measure_order = {m: i for i, m in enumerate(ordered)}
        rows = []
        for mname in ordered:
            pre_col = f'{mname}_pre'
            post_col = f'{mname}_post'
            delta_col = f'{mname}_delta'
            if pre_col not in df.columns or post_col not in df.columns:
                continue
            for flag, grp_label in group_specs:
                cols = [pre_col, post_col] + ([delta_col] if delta_col in df.columns else [])
                sub = df.loc[df['is_etw'] == flag, cols].copy()
                sub = sub.dropna(subset=[pre_col, post_col])
                n = int(sub.shape[0])
                if n == 0:
                    pre_m = post_m = delta_m = float('nan')
                else:
                    pre_m = float(sub[pre_col].mean())
                    post_m = float(sub[post_col].mean())
                    if delta_col in sub.columns:
                        delta_m = float(sub[delta_col].mean())
                    else:
                        delta_m = float((sub[post_col] - sub[pre_col]).mean())
                rows.append({'Measure': _measure_label(mname), 'Group': grp_label, 'N': n, 'Pre_M': pre_m, 'Post_M': post_m, 'Delta_M': delta_m, '__m_order': measure_order.get(mname, 10 ** 9), '__g_order': group_order.get(grp_label, 10 ** 9)})
        out_df = pd.DataFrame(rows)
        if not out_df.empty:
            for c in ('Pre_M', 'Post_M', 'Delta_M'):
                if c in out_df.columns:
                    out_df[c] = out_df[c].round(3)
            out_df = out_df.sort_values(['__m_order', '__g_order'], kind='mergesort').drop(columns=['__m_order', '__g_order'])
        if out_dir:
            os.makedirs(out_dir, exist_ok=True)
            out_df.to_csv(os.path.join(out_dir, filename), index=False)
        return out_df

    @staticmethod
    @staticmethod
    def export_school_deltas(df: pd.DataFrame, out_dir: str, overall_col: str='CompetenzaChiave', school_col: str='school', group_col: str='is_etw', uid_col: str='UID', folder_name: str='schools') -> None:
        try:
            if school_col not in df.columns:
                print('[WARN] schools export skipped: no school column found.')
                return
            pre_col = f'{overall_col}_pre'
            post_col = f'{overall_col}_post'
            delta_col = f'{overall_col}_delta'
            if pre_col not in df.columns or post_col not in df.columns:
                candidates = []
                try:
                    if 'AGGR_LABEL' in globals():
                        candidates.append(AGGR_LABEL)
                except Exception:
                    pass
                if 'CompetenzaChiave' not in candidates:
                    candidates.append('CompetenzaChiave')
                found = False
                for cand in candidates:
                    pc = f'{cand}_pre'
                    poc = f'{cand}_post'
                    if pc in df.columns and poc in df.columns:
                        overall_col = cand
                        pre_col, post_col, delta_col = (pc, poc, f'{cand}_delta')
                        found = True
                        break
                if not found:
                    print('[WARN] schools export skipped: missing overall pre/post columns.')
                    return
            cols = [uid_col, school_col, group_col, pre_col, post_col]
            if delta_col in df.columns:
                cols.append(delta_col)
            work = df[cols].copy()
            work = work.dropna(subset=[pre_col, post_col])
            if delta_col in work.columns:
                work['_delta'] = pd.to_numeric(work[delta_col], errors='coerce')
            else:
                work['_delta'] = pd.to_numeric(work[post_col], errors='coerce') - pd.to_numeric(work[pre_col], errors='coerce')
            work[school_col] = work[school_col].fillna('').astype(str).str.strip()
            unknown_mask = (work[school_col] == '') | (work[school_col].str.upper() == 'SCONOSCIUTA')
            n_unknown = int(unknown_mask.sum())
            work = work.loc[~unknown_mask].copy()
            if work.empty:
                print(f'[WARN] schools export skipped: no valid school entries after filtering (unknown rows={n_unknown}).')
                return

            def _sanitize_filename(s: str) -> str:
                s = re.sub('\\s+', '_', str(s).strip())
                s = re.sub('[^\\w\\-_\\.]', '_', s, flags=re.UNICODE)
                s = re.sub('_+', '_', s)
                return s[:180] if len(s) > 180 else s
            schools_dir = os.path.join(out_dir, folder_name)
            os.makedirs(schools_dir, exist_ok=True)
            work['_group'] = work[group_col].fillna(False).map(lambda v: 'ETW' if bool(v) else 'CTRL')
            summ = work.groupby([school_col, '_group'], dropna=False).agg(N=(uid_col, 'count'), Pre_Mean=(pre_col, 'mean'), Post_Mean=(post_col, 'mean'), Delta_Mean=('_delta', 'mean'), Delta_SD=('_delta', 'std')).reset_index().rename(columns={school_col: 'school', '_group': 'group'})
            summ = summ.sort_values(['school', 'group'], kind='mergesort')
            csv_path = os.path.join(schools_dir, 'summary_deltas_by_school.csv')
            summ.to_csv(csv_path, index=False)
            txt_path = os.path.join(schools_dir, 'summary_deltas_by_school.txt')
            with open(txt_path, 'w', encoding='utf-8') as f:
                f.write('Delta (POST-PRE) of overall competence by School × Group (complete-case UID-matched)\n')
                f.write(f'Overall column base: {overall_col}\n')
                f.write(f'Excluded unknown schools (blank/SCONOSCIUTA): {n_unknown}\n\n')
                for school in sorted(summ['school'].unique(), key=lambda x: str(x).lower()):
                    f.write(f'== {school} ==\n')
                    sub = summ[summ['school'] == school].copy()
                    for _, r in sub.iterrows():
                        sd = '' if pd.isna(r['Delta_SD']) else f"{r['Delta_SD']:.4f}"
                        f.write(f"  {r['group']:>4} | N={int(r['N']):>4} | PRE={r['Pre_Mean']:.4f} | POST={r['Post_Mean']:.4f} | Δ={r['Delta_Mean']:.4f} | SD={sd}\n")
                    f.write('\n')
            detail_cols = [uid_col, pre_col, post_col, '_delta', group_col]
            for school, wsub in work.groupby(school_col, dropna=False):
                safe = _sanitize_filename(school)
                per_csv = os.path.join(schools_dir, f'{safe}__deltas.csv')
                per_txt = os.path.join(schools_dir, f'{safe}__deltas.txt')
                wsub_out = wsub[detail_cols].copy()
                wsub_out.rename(columns={group_col: 'is_etw', '_delta': 'delta'}, inplace=True)
                wsub_out.to_csv(per_csv, index=False)
                agg2 = wsub.groupby('_group').agg(N=(uid_col, 'count'), Pre_Mean=(pre_col, 'mean'), Post_Mean=(post_col, 'mean'), Delta_Mean=('_delta', 'mean'), Delta_SD=('_delta', 'std')).reset_index().rename(columns={'_group': 'group'}).sort_values('group', kind='mergesort')
                with open(per_txt, 'w', encoding='utf-8') as f:
                    f.write(f'School: {school}\n')
                    f.write('Aggregate (complete-case UID-matched):\n')
                    for _, r in agg2.iterrows():
                        sd = '' if pd.isna(r['Delta_SD']) else f"{r['Delta_SD']:.4f}"
                        f.write(f"  {r['group']:>4} | N={int(r['N']):>4} | PRE={r['Pre_Mean']:.4f} | POST={r['Post_Mean']:.4f} | Δ={r['Delta_Mean']:.4f} | SD={sd}\n")
                    f.write('\nSample of UID-level rows (first 10):\n')
                    f.write(wsub_out.head(10).to_string(index=False))
                    f.write('\n')
            audit_path = os.path.join(schools_dir, 'audit_school_selection.txt')
            with open(audit_path, 'w', encoding='utf-8') as f:
                f.write('School export audit\n')
                f.write(f'- total complete-case rows considered: {int(df[[pre_col, post_col]].dropna().shape[0])}\n')
                f.write(f'- rows excluded due to unknown school: {n_unknown}\n')
                f.write(f'- rows exported: {int(work.shape[0])}\n\n')
                f.write('Small-N warnings (N<5):\n')
                small = summ[summ['N'] < 5]
                if small.empty:
                    f.write('  (none)\n')
                else:
                    for _, r in small.iterrows():
                        f.write(f"  {r['school']} | {r['group']} | N={int(r['N'])}\n")
            print(f'[OK] schools exported to: {schools_dir}')
        except Exception as e:
            print(f'[WARN] schools export failed: {e}')
            return

    def _safe_kde(vals):
        try:
            return stats.gaussian_kde(vals) if len(np.unique(vals)) > 1 else None
        except Exception:
            return None

    @classmethod
    def kde_pair(cls, df, col, title, path):
        delta = 'delta' in col
        xlim = (-1.75, 1.75) if delta else (1, 8)
        ticks = np.linspace(*xlim, 5) if delta else cls.TICKS
        fig, axs = plt.subplots(2, 1, sharex=True, figsize=(5, 4))
        for ax, flag, lbl, color in zip(axs, [True, False], ['eTwinning', 'Controllo'], ['tab:blue', 'red']):
            vals = df[col][df['is_etw'] == flag].dropna().astype(float)
            kde = cls._safe_kde(vals)
            xs = np.linspace(*xlim, 300)
            if kde is not None:
                ax.plot(xs, kde(xs), color=color)
            ax.set_title(lbl, fontsize=8)
            ax.set_xlim(*xlim)
            ax.set_xticks(ticks)
            if not delta:
                ax.set_xticklabels(cls.TICKLAB)
        fig.suptitle(title)
        fig.tight_layout()
        plt.savefig(path)
        plt.close()

    @classmethod
    def bar_delta(cls, df, col, title, path):
        mean_etw = df.loc[df['is_etw'], col].mean()
        mean_ctrl = df.loc[~df['is_etw'], col].mean()
        means = pd.Series([mean_etw, mean_ctrl], index=['eTwinning', 'Controllo'])
        max_abs = abs(means).max()
        if pd.isna(max_abs):
            return
        ylim = (-max_abs * 1.25, max_abs * 1.25)
        plt.figure(figsize=(4, 4))
        means.plot(kind='bar', color=['tab:blue', 'red'])
        plt.ylim(ylim)
        plt.ylabel('Δ medio (scala 1-8)')
        plt.title(title)
        plt.grid(ls=':', axis='y', alpha=0.6)
        plt.tight_layout()
        plt.savefig(path, dpi=300)
        plt.close()

    @classmethod
    def box_plot(cls, df, col, title, path):
        data = [df[col][df['is_etw'] == flag].dropna().astype(float) for flag in (True, False)]
        plt.figure(figsize=(4, 4))
        plt.boxplot(data, positions=[1, 2], patch_artist=True, boxprops=dict(color='k', facecolor='w'), medianprops=dict(color='k', linewidth=2), whiskerprops=dict(color='k'), capprops=dict(color='k'))
        for i, vals in enumerate(data, 1):
            plt.scatter(np.random.normal(i, 0.04, len(vals)), vals, s=12, alpha=0.5, color='tab:blue' if i == 1 else 'red')
        if 'delta' not in col:
            plt.ylim(1, 8)
            plt.yticks(cls.TICKS, cls.TICKLAB)
        else:
            vmin, vmax = (np.nanmin(np.hstack(data)), np.nanmax(np.hstack(data)))
            pad = max((vmax - vmin) * 0.05, 0.1)
            plt.ylim(vmin - pad, vmax + pad)
            plt.ylabel('Δ livello (1-8)')
        plt.title(title)
        plt.tight_layout()
        plt.savefig(path)
        plt.close()

    @classmethod
    def violin_plot(cls, df, col, title, path):
        data = [df[col][df['is_etw'] == flag].dropna().astype(float) for flag in (True, False)]
        fig, ax = plt.subplots(figsize=(4, 4))
        parts = ax.violinplot(data, positions=[1, 2], showmeans=False, showextrema=False, widths=0.8)
        for pc, color in zip(parts['bodies'], ['tab:blue', 'red']):
            pc.set_facecolor(color)
            pc.set_alpha(0.4)
        ax.boxplot(data, positions=[1, 2], widths=0.18, showcaps=False, boxprops=dict(color='k'), medianprops=dict(color='k'))
        for i, vals in enumerate(data, 1):
            ax.scatter(np.random.normal(i, 0.04, len(vals)), vals, s=10, alpha=0.6, color='tab:blue' if i == 1 else 'red')
        ax.set_xticks([1, 2])
        ax.set_xticklabels(['eTwin', 'Ctrl'])
        if 'delta' in col:
            ax.set_ylabel('Δ livello (1-8)')
        else:
            ax.set_ylabel('Livello (1-8)')
            ax.set_yticks(cls.TICKS)
            ax.set_yticklabels(cls.TICKLAB)
        ax.set_title(title)
        fig.tight_layout()
        plt.savefig(path)
        plt.close()

    @classmethod
    def dist_bar(cls, df: pd.DataFrame, col: str, title: str, path: str) -> None:
        fig, axs = plt.subplots(2, 1, figsize=(4, 4), sharex=True, gridspec_kw={'hspace': 0.3})
        for ax, flag, lbl, color in zip(axs, (True, False), ('eTwinning', 'Controllo'), ('tab:blue', 'red')):
            vals = df[col][df['is_etw'] == flag].dropna().round().astype(int)
            pct = vals.value_counts(normalize=True).sort_index().reindex(range(1, 9), fill_value=0) * 100
            bars = ax.bar(pct.index, pct.values, color=color, alpha=0.6)
            ax.set_ylim(0, max(pct.values.max(), 5) * 1.2)
            ax.set_ylabel('%')
            ax.set_title(lbl, fontsize=9)
            for b, v in zip(bars, pct.values):
                if v:
                    ax.text(b.get_x() + b.get_width() / 2, b.get_height() + 1, f'{v:.1f}%', ha='center', va='bottom', fontsize=8)
        axs[-1].set_xticks(cls.TICKS)
        axs[-1].set_xticklabels(cls.TICKLAB)
        fig.suptitle(title.replace('Likert', 'livelli'))
        fig.tight_layout()
        plt.savefig(path, dpi=300)
        plt.close(fig)

    @staticmethod
    def _radar(ax, labels, vals_etw, vals_ctrl, title):
        import textwrap
        n = len(labels)
        angles = np.linspace(0, 2 * np.pi, n, endpoint=False).tolist()
        angles += angles[:1]
        vmin, vmax = (np.nanmin([*vals_etw, *vals_ctrl]), np.nanmax([*vals_etw, *vals_ctrl]))
        shift = -vmin if vmin < 0 else 0
        v_etw = (np.array(vals_etw) + shift).tolist() + [shift + vals_etw[0]]
        v_ctrl = (np.array(vals_ctrl) + shift).tolist() + [shift + vals_ctrl[0]]
        ax.set_ylim(0, (vmax + shift) * 1.05)
        ax.set_theta_offset(np.pi / 2)
        ax.set_theta_direction(-1)
        ax.grid(color='lightgray', ls=':', lw=0.8)
        yticks = np.linspace(shift + vmin, shift + vmax, 5)
        ax.set_yticks(yticks)
        ax.set_yticklabels([f'{t - shift:.2f}' for t in yticks], fontsize=7)
        wrapped = [textwrap.fill(lab, width=22, break_long_words=False) for lab in labels]
        ax.set_xticks(angles[:-1])
        ax.set_xticklabels(wrapped, fontsize=8)
        ax.plot(angles, v_etw, color='tab:blue', lw=1.3)
        ax.fill(angles, v_etw, color='tab:blue', alpha=0.8)
        ax.plot(angles, v_ctrl, color='red', lw=1.3)
        ax.fill(angles, v_ctrl, color='red', alpha=0.8)
        ax.set_title(title, pad=25, fontsize=13)

    @classmethod
    def radar_area(cls, df: pd.DataFrame, out_dir: str) -> None:
        import textwrap
        labs = list(AREA_MAP.keys())
        etw = [df[f'{a}_delta'][df['is_etw']].mean() for a in labs]
        ctrl = [df[f'{a}_delta'][~df['is_etw']].mean() for a in labs]
        fig, ax = plt.subplots(figsize=(6, 6), subplot_kw={'projection': 'polar'})
        cls._radar(ax, labs, etw, ctrl, '')
        ax.set_title('Δ medio per area', pad=35)
        for tick, area in zip(ax.get_xticklabels(), labs):
            tick.set_color(AREA_COLOR[area])
            tick.set_text(textwrap.fill(tick.get_text(), 18))
        ax.legend(handles=[Patch(color='tab:blue', label='eTwinning'), Patch(color='red', label='Controllo')], loc='upper right', bbox_to_anchor=(1.12, 1.12))
        fig.tight_layout()
        fig.savefig(os.path.join(out_dir, 'radar_5aree.png'), dpi=300)
        plt.close(fig)

    @classmethod
    def radar_comp(cls, df: pd.DataFrame, out_dir: str) -> None:
        import textwrap
        labs = sum(AREA_MAP.values(), [])
        etw = [df[f'{c}_delta'][df['is_etw']].mean() for c in labs]
        ctrl = [df[f'{c}_delta'][~df['is_etw']].mean() for c in labs]
        fig, ax = plt.subplots(figsize=(8, 8), subplot_kw={'projection': 'polar'})
        cls._radar(ax, labs, etw, ctrl, '')
        ax.set_title(f'Δ medio competenze ({len(labs)})', pad=40)
        for tick, comp in zip(ax.get_xticklabels(), labs):
            area = next((a for a, lst in AREA_MAP.items() if comp in lst))
            tick.set_color(AREA_COLOR[area])
            tick.set_text(textwrap.fill(comp, 22))
        ax.legend(handles=[Patch(color='tab:blue', label='eTwinning'), Patch(color='red', label='Controllo')], loc='upper right', bbox_to_anchor=(1.18, 1.18))
        fig.tight_layout()
        fig.savefig(os.path.join(out_dir, 'radar_11comp.png'), dpi=300)
        plt.close(fig)

    @classmethod
    def run_pca(cls, df, ents, out_dir, dim=2):
        delta_cols = [f'{e}_delta' for e in ents]
        X = df[delta_cols].dropna()
        labels = df.loc[X.index, 'is_etw']
        pca = PCA().fit(X)
        evr = pca.explained_variance_ratio_
        cum = np.cumsum(evr)
        plt.figure(figsize=(4, 3))
        plt.bar(range(1, len(evr) + 1), evr * 100, color='grey')
        plt.plot(range(1, len(cum) + 1), cum * 100, marker='o')
        plt.axhline(80, color='r', ls='--')
        plt.ylabel('% varianza')
        plt.xlabel('Componenti')
        plt.title('Scree plot PCA')
        plt.tight_layout()
        plt.savefig(os.path.join(out_dir, 'pca_scree.png'))
        plt.close()
        n_comp = max(dim, np.where(cum >= 0.8)[0][0] + 1)
        pca = PCA(n_components=n_comp).fit(X)
        comp = pca.transform(X)
        fig = plt.figure(figsize=(5, 4))
        if dim == 3:
            from mpl_toolkits.mplot3d import Axes3D
            ax = fig.add_subplot(111, projection='3d')
            for flag, color, lbl in [(True, 'tab:blue', 'eTwinning'), (False, 'red', 'Controllo')]:
                pts = comp[labels == flag]
                ax.scatter(pts[:, 0], pts[:, 1], pts[:, 2], label=lbl, alpha=0.7, s=30, color=color)
            ax.set_zlabel('PC3')
        else:
            ax = fig.add_subplot(111)
            for flag, color, lbl in [(True, 'tab:blue', 'eTwinning'), (False, 'red', 'Controllo')]:
                pts = comp[labels == flag]
                ax.scatter(pts[:, 0], pts[:, 1], label=lbl, alpha=0.7, s=30, color=color)
        ax.set_xlabel('PC1')
        ax.set_ylabel('PC2')
        ax.legend()
        plt.title(f'PCA scatter ({dim}D)')
        plt.tight_layout()
        plt.savefig(os.path.join(out_dir, f'pca_scatter_{dim}d.png'))
        plt.close()
        load_df = pd.DataFrame(pca.components_.T, index=delta_cols, columns=[f'PC{i + 1}' for i in range(n_comp)])
        load_df.to_csv(os.path.join(out_dir, 'pca_loadings.csv'))

    @classmethod
    def run_mds(cls, df, ents, out_dir, dim=2):
        delta_cols = [f'{e}_delta' for e in ents]
        X = df[delta_cols].dropna()
        labels = df.loc[X.index, 'is_etw']
        mds = MDS(n_components=dim, dissimilarity='euclidean', random_state=1)
        coords = mds.fit_transform(X)
        fig = plt.figure(figsize=(5, 4))
        if dim == 3:
            from mpl_toolkits.mplot3d import Axes3D
            ax = fig.add_subplot(111, projection='3d')
            for flag, color, lbl in [(True, 'tab:blue', 'eTwinning'), (False, 'red', 'Controllo')]:
                pts = coords[labels == flag]
                ax.scatter(pts[:, 0], pts[:, 1], pts[:, 2], label=lbl, color=color, alpha=0.7, s=30)
            ax.set_zlabel('MDS3')
        else:
            ax = fig.add_subplot(111)
            for flag, color, lbl in [(True, 'tab:blue', 'eTwinning'), (False, 'red', 'Controllo')]:
                pts = coords[labels == flag]
                ax.scatter(pts[:, 0], pts[:, 1], label=lbl, color=color, alpha=0.7, s=30)
        ax.set_xlabel('MDS1')
        ax.set_ylabel('MDS2')
        ax.legend()
        plt.title(f'MDS scatter ({dim}D)')
        plt.tight_layout()
        plt.savefig(os.path.join(out_dir, f'mds_scatter_{dim}d.png'))
        plt.close()

def shapiro_francia(x):
    x = np.asarray(x, float)
    x = x[~np.isnan(x)]
    if len(x) < 5 or len(x) > 5000:
        stat, p = stats.normaltest(x, nan_policy='omit')
    else:
        stat, p = stats.shapiro(x)
    return (stat, p)

class App(tk.Tk):

    def __init__(self):
        super().__init__()
        self.title('eTwinning Analyzer')
        self.geometry('1080x580')
        self._build()

    def _build(self):
        self.paths = {k: tk.StringVar() for k in ('pre', 'post', 'map', 'out')}
        self.paths['pre'].set(DEFAULT_PRE_CSV)
        self.paths['post'].set(DEFAULT_POST_CSV)
        self.paths['map'].set(_resolve_path(DEFAULT_MAPPING_JSON, DEFAULT_MAPPING_JSON_FALLBACK))
        self.paths['out'].set(DEFAULT_OUT_DIR)
        rows = [('CSV entrata', 'pre'), ('CSV uscita', 'post'), ('Mapping JSON', 'map'), ('Cartella output', 'out')]
        for i, (lab, k) in enumerate(rows):
            tk.Label(self, text=lab).grid(row=i, column=0, sticky='w', padx=6, pady=6)
            tk.Entry(self, textvariable=self.paths[k], width=80).grid(row=i, column=1, padx=4)
            tk.Button(self, text='…', command=lambda k=k: self._pick(k)).grid(row=i, column=2)
        self.do_norm = tk.BooleanVar(value=True)
        self.do_t = tk.BooleanVar(value=True)
        self.do_anc = tk.BooleanVar(value=True)
        self.t_step = tk.BooleanVar()
        opts = [('Test normalità', self.do_norm, 0), ('Welch / MW t-test', self.do_t, 1), ('ANCOVA', self.do_anc, 2), ('t-test step-wise', self.t_step, 3)]
        for txt, var, col in opts:
            tk.Checkbutton(self, text=txt, var=var).grid(row=4, column=col, sticky='w', padx=6)
        tk.Label(self, text='Riduzione dimensioni:').grid(row=5, column=0, sticky='e', padx=6)
        self.do_pca = tk.BooleanVar()
        self.do_mds = tk.BooleanVar()
        tk.Checkbutton(self, text='PCA', var=self.do_pca).grid(row=5, column=1, sticky='w')
        tk.Checkbutton(self, text='MDS', var=self.do_mds).grid(row=5, column=2, sticky='w')
        self.dim = tk.IntVar(value=2)
        for i, d in enumerate((2, 3)):
            tk.Radiobutton(self, text=f'{d}D', variable=self.dim, value=d).grid(row=5, column=3 + i, sticky='w')
        tk.Label(self, text='Covariate ANCOVA:').grid(row=6, column=0, sticky='e', padx=6)
        self.cov_eta = tk.BooleanVar()
        self.cov_scu = tk.BooleanVar()
        tk.Checkbutton(self, text='Età', var=self.cov_eta).grid(row=6, column=1, sticky='w')
        tk.Checkbutton(self, text='Scuola', var=self.cov_scu).grid(row=6, column=2, sticky='w')
        self.opt_fdr = tk.BooleanVar(value=True)
        self.opt_eta_boot = tk.BooleanVar()
        self.opt_cluster = tk.BooleanVar()
        self.opt_mixed = tk.BooleanVar()
        self.opt_two_stage = tk.BooleanVar()
        tk.Label(self, text='Opzioni avanzate:').grid(row=6, column=3, sticky='e', padx=6)
        tk.Checkbutton(self, text='FDR (q-values)', var=self.opt_fdr).grid(row=6, column=4, sticky='w')
        tk.Checkbutton(self, text='CI bootstrap η²', var=self.opt_eta_boot).grid(row=6, column=5, sticky='w')
        tk.Checkbutton(self, text='SE cluster-robust', var=self.opt_cluster).grid(row=6, column=6, sticky='w')
        tk.Checkbutton(self, text='Mixed-effects', var=self.opt_mixed).grid(row=6, column=7, sticky='w')
        tk.Checkbutton(self, text='Analisi due-stadi (scuola)', var=self.opt_two_stage).grid(row=6, column=8, sticky='w')
        tk.Button(self, text='Processa', command=self._run).grid(row=9, column=0, columnspan=6, pady=20)

    def _pick(self, key):
        p = filedialog.askopenfilename() if key != 'out' else filedialog.askdirectory()
        if p:
            self.paths[key].set(p)

    def _audit_mapping(self, df_raw, mapping, out_dir):
        import pandas as _pd, numpy as _np, os as _os
        rows = []
        intervals = []
        for q, info in mapping.items():
            s = str(info.get('start_col', '')).strip().upper()
            if s in df_raw.columns:
                i0 = df_raw.columns.get_loc(s)
                ok = i0 + 4 < df_raw.shape[1]
                e = df_raw.columns[i0 + 4] if ok else ''
            else:
                ok = False
                e = ''
                i0 = None
            rows.append([q, s, e, ';'.join(info.get('competencies', [])), bool(ok)])
            if ok:
                intervals.append((q, i0, i0 + 4))
        _pd.DataFrame(rows, columns=['question', 'start', 'end', 'competencies', 'valid_5cols']).to_csv(_os.path.join(out_dir, 'mapping_audit.csv'), index=False)
        ov = []
        for i in range(len(intervals)):
            qi, a0, a1 = (intervals[i][0], intervals[i][1], intervals[i][2])
            for j in range(i + 1, len(intervals)):
                qj, b0, b1 = (intervals[j][0], intervals[j][1], intervals[j][2])
                if max(a0, b0) <= min(a1, b1):
                    ov.append([qi, qj, df_raw.columns[max(a0, b0)], df_raw.columns[min(a1, b1)]])
        if ov:
            _pd.DataFrame(ov, columns=['Q_i', 'Q_j', 'overlap_start', 'overlap_end']).to_csv(_os.path.join(out_dir, 'mapping_overlaps.csv'), index=False)

    def _detect_multi_marks(self, df_raw, mapping, out_dir):
        import pandas as _pd, numpy as _np, os as _os
        warn = []
        for q, info in mapping.items():
            s = str(info.get('start_col', '')).strip().upper()
            if s not in df_raw.columns:
                continue
            i0 = df_raw.columns.get_loc(s)
            blk = df_raw.iloc[:, i0:i0 + 5]

            def _is_marked(x):
                if x is None:
                    return False
                if isinstance(x, float) and _np.isnan(x):
                    return False
                return str(x).strip() != ''
            cnt = (blk.map(_is_marked) if hasattr(blk, 'map') else blk.applymap(_is_marked)).sum(axis=1)
            bad = cnt[cnt > 1]
            if not bad.empty:
                for idx, n in bad.items():
                    warn.append([q, int(idx), int(n)])
        if warn:
            _pd.DataFrame(warn, columns=['question', 'row_idx', 'n_marked']).to_csv(_os.path.join(out_dir, 'scoring_warnings_multimark.csv'), index=False)

    def _two_stage(self, df, ents, out):
        import numpy as _np, pandas as _pd, os as _os
        rows = []
        for ent in ents:
            d = df[[f'{ent}_delta', 'school', 'is_etw']].dropna()
            if d.empty:
                rows.append([ent, 0] + [_np.nan] * 6)
                continue
            grp = d.groupby(['school', 'is_etw'])[f'{ent}_delta'].mean().unstack()
            if grp is None or grp.shape[1] < 2:
                rows.append([ent, 0] + [_np.nan] * 6)
                continue
            inter = grp.dropna().dropna(axis=0, how='any')
            if inter.empty or inter.shape[0] < 2:
                rows.append([ent, 0] + [_np.nan] * 6)
                continue
            diff = inter[True] - inter[False]
            n = len(diff)
            m = diff.mean()
            s = diff.std(ddof=1)
            if n < 2 or s == 0 or _np.isnan(s):
                rows.append([ent, n, m, _np.nan, _np.nan, _np.nan, _np.nan, _np.nan])
                continue
            from scipy.stats import t as _t
            t = m / (s / _np.sqrt(n))
            p = 2 * (1 - _t.cdf(abs(t), df=n - 1))
            z = _t.ppf(0.975, df=n - 1)
            lo, hi = (m - z * s / _np.sqrt(n), m + z * s / _np.sqrt(n))
            rows.append([ent, n, m, s, t, p, lo, hi])
        _pd.DataFrame(rows, columns=['entity', 'N_schools', 'mean_diff', 'SD', 't', 'p', 'CI_low', 'CI_high']).to_csv(_os.path.join(out, 'two_stage_school_summary.csv'), index=False)

    @staticmethod
    def _attach_uid(df_scored: pd.DataFrame) -> pd.DataFrame:

        def _norm_ay(x):
            if pd.isna(x):
                return ''
            s = str(x).strip()
            if s.endswith('.0'):
                s = s[:-2]
            s = re.sub('\\s+', '', s)
            return s
        d = df_scored.copy()
        d['AY_norm'] = d['AY'].map(_norm_ay)
        d['grp'] = d['is_etw'].map(lambda v: 'ETW' if bool(v) else 'CTL')
        for c in ('school', 'classe', 'sezione'):
            if c not in d.columns:
                d[c] = ''
            d[c] = d[c].fillna('').astype(str)
        d['UID'] = d['school'] + '_' + d['grp'] + '_' + d['classe'] + '_' + d['sezione'] + '_' + d['AY_norm']
        return d

    @staticmethod
    def _load_full_raw_csv(path: str) -> pd.DataFrame:
        return pd.read_csv(path, header=None, dtype=str)

    def _run(self):
        if not all((self.paths[k].get() for k in self.paths)):
            messagebox.showerror('Errore', 'Seleziona tutti i file')
            return
        mapping = json.load(open(self.paths['map'].get(), encoding='utf-8'))
        pre_ld = DataLoader(self.paths['pre'].get(), INVERTED_START_COLS)
        post_ld = DataLoader(self.paths['post'].get(), INVERTED_START_COLS)
        pre_raw = pre_ld.load_raw()
        post_raw = post_ld.load_raw()
        out_dir = self.paths['out'].get()
        os.makedirs(out_dir, exist_ok=True)
        self._audit_mapping(pre_raw, mapping, out_dir)
        self._detect_multi_marks(pre_raw, mapping, out_dir)
        pre_num = pre_ld.likert_numeric(pre_raw, mapping)
        post_num = post_ld.likert_numeric(post_raw, mapping)
        merged, ents = Analyzer(pre_num, post_num, mapping).process()
        out_dir = self.paths['out'].get()
        os.makedirs(out_dir, exist_ok=True)
        merged.isna().mean().sort_values(ascending=False).to_csv(os.path.join(out_dir, 'missingness_summary.csv'))
        merged.to_csv(os.path.join(out_dir, 'results_competenza_digitale.csv'), index=False)
        try:
            Analyzer.export_group_means(merged, ents, out_dir)
        except Exception:
            pass
        try:
            Analyzer.export_school_deltas(merged, out_dir, overall_col=AGGR_LABEL if 'AGGR_LABEL' in globals() else 'CompetenzaChiave')
        except Exception as e:
            print(f'[WARN] schools export failed: {e}')
        for ent in ents:
            Analyzer.kde_pair(merged, f'{ent}_pre', f'{ent} – PRE', os.path.join(out_dir, f'{ent}_pre_kde.png'))
            Analyzer.kde_pair(merged, f'{ent}_post', f'{ent} – POST', os.path.join(out_dir, f'{ent}_post_kde.png'))
            Analyzer.kde_pair(merged, f'{ent}_delta', f'{ent} – Δ', os.path.join(out_dir, f'{ent}_delta_kde.png'))
            Analyzer.bar_delta(merged, f'{ent}_delta', f'Δ medio – {ent}', os.path.join(out_dir, f'{ent}_delta_bar.png'))
            Analyzer.box_plot(merged, f'{ent}_delta', f'{ent} – Δ box-plot', os.path.join(out_dir, f'{ent}_delta_box.png'))
            Analyzer.violin_plot(merged, f'{ent}_delta', f'{ent} – Δ raincloud', os.path.join(out_dir, f'{ent}_delta_violin.png'))
            Analyzer.dist_bar(merged, f'{ent}_pre', f'{ent} – distribuzione livelli PRE', os.path.join(out_dir, f'{ent}_pre_dist.png'))
            Analyzer.dist_bar(merged, f'{ent}_post', f'{ent} – distribuzione livelli POST', os.path.join(out_dir, f'{ent}_post_dist.png'))
        Analyzer.radar_area(merged, out_dir)
        Analyzer.radar_comp(merged, out_dir)
        dim = self.dim.get()
        if self.do_pca.get():
            Analyzer.run_pca(merged, ents, out_dir, dim=dim)
        if self.do_mds.get():
            Analyzer.run_mds(merged, ents, out_dir, dim=dim)
        if self.opt_two_stage.get():
            self._two_stage(merged, ents, out_dir)
        if self.do_norm.get():
            self._normality(merged, ents, out_dir)
        if self.do_t.get():
            self._welch_MW(merged, ents, out_dir)
        if self.t_step.get():
            self._ttest_step(merged, ents, out_dir)
        if self.do_anc.get():
            self._ancova(merged, ents, out_dir)

    def _normality(self, df, ents, out):
        rows = []
        for ent in ents:
            for grp, lbl in [(True, 'etw'), (False, 'ctrl')]:
                vals = df[f'{ent}_delta'][df['is_etw'] == grp].dropna().astype(float)
                if len(vals) < 8:
                    rows.append([ent, lbl, np.nan, np.nan])
                    continue
                w, p = shapiro_francia(vals)
                rows.append([ent, lbl, w, p])
        pd.DataFrame(rows, columns=['entity', 'group', 'stat', 'p']).to_csv(os.path.join(out, 'normality_summary.csv'), index=False)

    def _welch_MW(self, df, ents, out):
        rows = []
        rows_compact = []
        pvals = []
        for ent in ents:
            etw = df[f'{ent}_delta'][df['is_etw']].dropna().astype(float)
            ctrl = df[f'{ent}_delta'][~df['is_etw']].dropna().astype(float)
            if min(len(etw), len(ctrl)) < 3:
                rows.append([ent, len(etw), len(ctrl), '–', '–', np.nan, '–', np.nan, np.nan, np.nan, np.nan, np.nan, np.nan])
                pvals.append(np.nan)
                continue
            gauss = shapiro_francia(etw)[1] > 0.05 and shapiro_francia(ctrl)[1] > 0.05
            if gauss:
                stat, p = stats.ttest_ind(etw, ctrl, equal_var=False)
                test = 'Welch'
                g, lo, hi = hedges_g_ci(etw, ctrl)
                rb, rb_lo, rb_hi = (np.nan, np.nan, np.nan)
            else:
                res = stats.mannwhitneyu(etw, ctrl, alternative='two-sided')
                stat, p = (float(res.statistic), float(res.pvalue))
                test = 'Mann-Whitney'
                g, lo, hi = (np.nan, np.nan, np.nan)
                n1, n2 = (len(etw), len(ctrl))
                rb = 2 * stat / (n1 * n2) - 1
                rng = np.random.default_rng(42)
                boots = []
                for _ in range(400):
                    s1 = etw.sample(frac=1, replace=True, random_state=rng.integers(0, 1 << 32))
                    s2 = ctrl.sample(frac=1, replace=True, random_state=rng.integers(0, 1 << 32))
                    U = stats.mannwhitneyu(s1, s2, alternative='two-sided').statistic
                    boots.append(2 * U / (len(s1) * len(s2)) - 1)
                rb_lo, rb_hi = (np.nanpercentile(boots, 2.5), np.nanpercentile(boots, 97.5)) if boots else (np.nan, np.nan)
            sig = 'sig' if p < 0.05 else 'n.s.'
            rows.append([ent, len(etw), len(ctrl), test, stat, p, sig, g, lo, hi, rb, rb_lo, rb_hi])
            pvals.append(p)
        import pandas as pd
        from statsmodels.stats.multitest import multipletests
        df_out = pd.DataFrame(rows, columns=['entity', 'N_etw', 'N_ctrl', 'test', 'statistic', 'p', 'sig/NS', 'g', 'CI_low', 'CI_high', 'rank_biserial', 'rb_CI_low', 'rb_CI_high'])
        if self.opt_fdr.get():
            df_out['q'] = multipletests(df_out['p'].values, method='fdr_bh')[1]
        df_out.to_csv(os.path.join(out, 'Welch_MW_summary.csv'), index=False)

    def _ttest_step(self, df, ents, out):
        thresholds = [0.2, 0.4, 0.6, 0.8]
        rec = []
        for ent in ents:
            for th in thresholds:
                subset = df[np.abs(df[f'{ent}_delta']) >= th]
                etw = subset[f'{ent}_delta'][subset['is_etw']]
                ctrl = subset[f'{ent}_delta'][~subset['is_etw']]
                if min(len(etw), len(ctrl)) < 3:
                    rec.append([ent, th, np.nan, np.nan, np.nan, '–'])
                    continue
                t, p = stats.ttest_ind(etw, ctrl, equal_var=False)
                sig = 'sig' if p < 0.05 else 'n.s.'
                rec.append([ent, th, len(etw) + len(ctrl), t, p, sig])
        pd.DataFrame(rec, columns=['entity', 'cutoff', 'N', 't', 'p', 'sig/NS']).to_csv(os.path.join(out, 'ttest_stepwise.csv'), index=False)

    def _ancova(self, df, ents, out):
        cov = []
        if self.cov_eta.get():
            cov.append('eta')
        if self.cov_scu.get():
            cov.append('C(school)')
        covs = ' + '.join(cov) if cov else ''
        rows = []
        pvals = []
        rows_compact = []
        for ent in ents:
            form = f"{q(ent + '_post')} ~ is_etw + {q(ent + '_pre')}" + (' + ' + covs if covs else '')
            use_mixed = self.opt_mixed.get()
            use_cluster = self.opt_cluster.get() and (not use_mixed)
            need_school = self.cov_scu.get() or use_cluster or use_mixed
            need = [f'{ent}_post', f'{ent}_pre', 'is_etw']
            if need_school:
                need.append('school')
            if self.cov_eta.get():
                need.append('eta')
            need = list(dict.fromkeys(need))
            dft = df[need].dropna()
            too_few = len(dft) < 5
            two_grps = dft['is_etw'].nunique(dropna=True) >= 2
            if need_school:
                if 'school' in dft.columns:
                    n_sch = dft['school'].nunique(dropna=True) if not isinstance(dft['school'], pd.DataFrame) else dft['school'].iloc[:, 0].nunique(dropna=True)
                else:
                    n_sch = 0
                ok_school = n_sch >= 1
            else:
                ok_school = True
            if too_few or not two_grps or (not ok_school):
                rows.append([ent] + [np.nan] * 11)
                pvals.append(np.nan)
                continue
            try:
                if use_mixed:
                    mdl = smf.mixedlm(form, dft, groups=dft['school']).fit(reml=True, method='lbfgs')
                    coef = 'is_etw[T.True]' if 'is_etw[T.True]' in mdl.params.index else 'is_etw'
                    b, se = (mdl.params.get(coef, np.nan), mdl.bse.get(coef, np.nan))
                    t = b / se if se and se == se and (se != 0) else np.nan
                    p = mdl.pvalues.get(coef, np.nan)
                    eta = np.nan
                    ci_lo = np.nan
                    ci_hi = np.nan
                    model = 'mixed'
                else:
                    if use_cluster:
                        mdl = smf.ols(form, dft).fit(cov_type='cluster', cov_kwds={'groups': dft['school']})
                        model = 'cluster'
                    else:
                        mdl = smf.ols(form, dft).fit()
                        model = 'ols'
                    coef = 'is_etw[T.True]' if 'is_etw[T.True]' in getattr(mdl, 'params', {}) else 'is_etw'
                    b = mdl.params.get(coef, np.nan) if hasattr(mdl, 'params') else np.nan
                    se = mdl.bse.get(coef, np.nan) if hasattr(mdl, 'bse') else np.nan
                    t = b / se if se and se == se and (se != 0) else np.nan
                    p = mdl.pvalues.get(coef, np.nan) if hasattr(mdl, 'pvalues') else np.nan
                    try:
                        aov = anova_lm(mdl, typ=2)
                        ss = aov.at['is_etw', 'sum_sq']
                        sse = aov.at['Residual', 'sum_sq']
                        eta = ss / (ss + sse)
                    except Exception:
                        eta = np.nan
                    if self.opt_eta_boot.get() and np.isfinite(eta):
                        try:
                            lo, hi = eta_boot_ci(dft, form, term='is_etw', n_boot=300, alpha=0.05)
                            ci_lo, ci_hi = (lo, hi)
                        except Exception:
                            ci_lo = ci_hi = np.nan
                    else:
                        ci_lo = ci_hi = np.nan
            except Exception:
                b = se = t = p = eta = ci_lo = ci_hi = np.nan
                model = 'ols'
            sig = 'sig' if p is not None and p == p and (p < 0.05) else 'n.s.'
            rows.append([ent, len(dft), b, se, t, p, sig, eta, ci_lo, ci_hi, model])
            try:
                aov = anova_lm(mdl, typ=2)
                term = 'is_etw' if 'is_etw' in aov.index else next((i for i in aov.index if 'is_etw' in str(i)), None)
                if term is None:
                    raise KeyError('is_etw term not found')
                Fv = float(aov.loc[term, 'F'])
                pcol = 'PR(>F)' if 'PR(>F)' in aov.columns else 'Pr(>F)' if 'Pr(>F)' in aov.columns else None
                pF = float(aov.loc[term, pcol]) if pcol else float(p)
            except Exception:
                Fv, pF = (t ** 2, float(p))
            rows_compact.append([ent, Fv, pF, '✔' if pF < 0.05 else ''])
            pvals.append(p)
        df_out = pd.DataFrame(rows, columns=['entity', 'N', 'beta', 'SE', 't', 'p', 'sig/NS', 'partial_eta2', 'CI_low', 'CI_high', 'model'])
        if self.opt_fdr.get():
            from statsmodels.stats.multitest import multipletests
            df_out['q'] = multipletests(df_out['p'].values, method='fdr_bh')[1]
        df_out.to_csv(os.path.join(out, 'ancova_summary.csv'), index=False)
        pd.DataFrame(rows_compact, columns=['oggetto', 'F', 'p', 'significativo']).to_csv(os.path.join(out, 'ancova_summary_compact.csv'), index=False)
if __name__ == '__main__':
    App().mainloop()
