"""Customer retention model on UCI Online Retail II — reproduction script.

DataSense8, https://datasense8.com/case-studies/customer-retention-model/
Data: Chen, D. (2012). Online Retail II [Dataset]. UCI Machine Learning Repository.
https://doi.org/10.24432/C5CG6D — licensed CC BY 4.0.

Run:
    pip install pandas openpyxl scikit-learn
    python customer-retention-model.py                 # downloads the UCI zip next to the script
    python customer-retention-model.py --xlsx path/to/online_retail_II.xlsx --out results

Outputs (CSV: separator ';', decimal comma, UTF-8 with BOM — opens in Excel with a European locale):
    customer-retention-features.csv  one row per customer: RFM features at the cut-off date,
                                     whether the customer bought again in the next 90 days,
                                     out-of-fold model probability
    customer-retention-segments.csv  the same, aggregated by RFM segment
    customer-retention-summary.json  model metrics and dataset facts
"""
# Комментарии в коде — на русском (правило владельца); docstring выше — для читателя сайта, по-английски.
import argparse
import io
import json
import os
import urllib.request
import zipfile
from datetime import date

import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, roc_auc_score
from sklearn.model_selection import StratifiedKFold, cross_val_predict

UCI_ZIP = 'https://archive.ics.uci.edu/static/public/502/online+retail+ii.zip'
SHEET = 'Year 2010-2011'
CUTOFF = pd.Timestamp('2011-09-01')   # признаки — по сделкам до этой даты
HORIZON_DAYS = 90                     # метка — была ли покупка в следующие 90 дней (до 30.11.2011)
SEED = 42
# Служебные коды: почта, комиссии, ручные правки — не товары (как в расчётах блога)
NON_PRODUCT = {'POST', 'D', 'M', 'C2', 'DOT', 'BANK CHARGES', 'AMAZONFEE', 'CRUK', 'PADS', 'S', 'B',
               'ADJUST', 'ADJUST2', 'TEST001', 'TEST002'}
SEGMENTS = ['Champions', 'Loyal', "Can't lose", 'At risk', 'Potential loyalists', 'Need attention',
            'New', 'Promising', 'About to sleep', 'Hibernating']


def get_xlsx(path):
    # Нет файла — качаем архив UCI (~45 МБ) и достаём xlsx рядом с указанным путём
    if os.path.exists(path):
        return path
    print('downloading', UCI_ZIP)
    with urllib.request.urlopen(UCI_ZIP) as resp:
        blob = resp.read()
    with zipfile.ZipFile(io.BytesIO(blob)) as z:
        name = next(n for n in z.namelist() if n.lower().endswith('.xlsx'))
        with open(path, 'wb') as f:
            f.write(z.read(name))
    return path


def load_sales(xlsx):
    # Чтение листа Excel занимает около минуты
    df = pd.read_excel(xlsx, sheet_name=SHEET, dtype={'Invoice': str, 'StockCode': str})
    df['Invoice'] = df['Invoice'].astype(str)
    df['StockCode'] = df['StockCode'].astype(str).str.upper()
    df['Revenue'] = df['Quantity'] * df['Price']
    rows = len(df)
    # Продажи: без отмен (счета на «C»), без нулевых и отрицательных строк, без служебных кодов и дублей
    s = df[(~df['Invoice'].str.startswith('C')) & (df['Quantity'] > 0) & (df['Price'] > 0)
           & (~df['StockCode'].isin(NON_PRODUCT))].drop_duplicates()
    s = s.dropna(subset=['Customer ID']).copy()
    s['Customer ID'] = s['Customer ID'].astype(int)
    return s, rows


def segment(r_, f_):
    # Классическая сетка R×F (10 сегментов) — та же, что в посте про RFM
    if r_ == 5 and f_ >= 4: return 'Champions'
    if r_ >= 3 and f_ >= 4: return 'Loyal'
    if r_ >= 4 and f_ >= 2: return 'Potential loyalists'
    if r_ == 5 and f_ == 1: return 'New'
    if r_ == 4 and f_ == 1: return 'Promising'
    if r_ == 3 and f_ == 3: return 'Need attention'
    if r_ == 3 and f_ <= 2: return 'About to sleep'
    if r_ <= 2 and f_ == 5: return "Can't lose"
    if r_ <= 2 and f_ >= 3: return 'At risk'
    return 'Hibernating'


def features(sales):
    past = sales[sales['InvoiceDate'] < CUTOFF]
    future = sales[(sales['InvoiceDate'] >= CUTOFF)
                   & (sales['InvoiceDate'] < CUTOFF + pd.Timedelta(days=HORIZON_DAYS))]
    g = past.groupby('Customer ID')
    x = pd.DataFrame({
        'last': g['InvoiceDate'].max(),
        'first': g['InvoiceDate'].min(),
        'frequency': g['Invoice'].nunique(),
        'monetary': g['Revenue'].sum(),
        'products': g['StockCode'].nunique(),
        'country': g['Country'].agg(lambda c: c.mode().iloc[0]),
    })
    x['recency_days'] = (CUTOFF - x['last'].dt.normalize()).dt.days
    x['tenure_days'] = (CUTOFF - x['first'].dt.normalize()).dt.days
    x['avg_order_value'] = x['monetary'] / x['frequency']
    x['uk'] = (x['country'] == 'United Kingdom').astype(int)
    # Квинтили по рангу: у частоты много одинаковых значений, ранг разводит их равномерно
    x['r_score'] = pd.qcut(x['recency_days'].rank(method='first'), 5, labels=[5, 4, 3, 2, 1]).astype(int)
    x['f_score'] = pd.qcut(x['frequency'].rank(method='first'), 5, labels=[1, 2, 3, 4, 5]).astype(int)
    x['m_score'] = pd.qcut(x['monetary'].rank(method='first'), 5, labels=[1, 2, 3, 4, 5]).astype(int)
    x['segment'] = [segment(a, b) for a, b in zip(x['r_score'], x['f_score'])]
    x['returned_90d'] = x.index.isin(future['Customer ID'].unique()).astype(int)
    return x


def main():
    ap = argparse.ArgumentParser(description=__doc__.split('\n')[0])
    here = os.path.dirname(os.path.abspath(__file__))
    ap.add_argument('--xlsx', default=os.path.join(here, 'online_retail_II.xlsx'))
    ap.add_argument('--out', default=here)
    a = ap.parse_args()
    os.makedirs(a.out, exist_ok=True)

    sales, rows = load_sales(get_xlsx(a.xlsx))
    x = features(sales)
    cols = ['recency_days', 'frequency', 'monetary', 'avg_order_value', 'tenure_days', 'products', 'uk']
    y = x['returned_90d']

    # Вероятность возврата — вне выборки обучения: 5 фолдов, каждый клиент оценён моделью, его не видевшей
    rf = RandomForestClassifier(n_estimators=300, min_samples_leaf=5, random_state=SEED, n_jobs=-1)
    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)
    p = cross_val_predict(rf, x[cols], y, cv=cv, method='predict_proba')[:, 1]
    x['p_return'] = p
    auc = roc_auc_score(y, p)
    acc = accuracy_score(y, (p >= 0.5).astype(int))
    base_auc = roc_auc_score(y, -x['recency_days'])  # без модели: «кто купил недавно, тот вернётся»
    imp = rf.fit(x[cols], y).feature_importances_

    # Признаки по клиентам
    out = x.reset_index().rename(columns={'Customer ID': 'customer_id'})
    out['monetary'] = out['monetary'].round(2)
    out['avg_order_value'] = out['avg_order_value'].round(2)
    out['p_return'] = out['p_return'].round(3)
    out = out[['customer_id', 'country', 'uk', 'recency_days', 'frequency', 'monetary', 'avg_order_value',
               'tenure_days', 'products', 'r_score', 'f_score', 'm_score', 'segment', 'returned_90d', 'p_return']]
    csv = dict(sep=';', decimal=',', encoding='utf-8-sig', index=False)
    out.to_csv(os.path.join(a.out, 'customer-retention-features.csv'), **csv)

    # Агрегат по сегментам
    seg = x.groupby('segment').agg(customers=('frequency', 'size'), revenue=('monetary', 'sum'),
                                   returned=('returned_90d', 'mean'), predicted=('p_return', 'mean'),
                                   median_recency_days=('recency_days', 'median'),
                                   median_orders=('frequency', 'median'))
    seg = seg.reindex([s for s in SEGMENTS if s in seg.index])
    seg_out = pd.DataFrame({
        'segment': seg.index,
        'customers': seg['customers'].astype(int).values,
        'customer_share_pct': (100 * seg['customers'] / len(x)).round(1).values,
        'revenue_share_pct': (100 * seg['revenue'] / x['monetary'].sum()).round(1).values,
        'median_recency_days': seg['median_recency_days'].round(0).astype(int).values,
        'median_orders': seg['median_orders'].round(0).astype(int).values,
        'returned_90d_pct': (100 * seg['returned']).round(1).values,
        'model_predicted_pct': (100 * seg['predicted']).round(1).values,
    })
    seg_out.to_csv(os.path.join(a.out, 'customer-retention-segments.csv'), **csv)

    summary = {
        'source': 'UCI Online Retail II (Chen, 2012), sheet Year 2010-2011',
        'doi': 'https://doi.org/10.24432/C5CG6D',
        'license': 'CC BY 4.0',
        'rows': int(rows),
        'from': sales['InvoiceDate'].min().strftime('%Y-%m-%d'),
        'to': sales['InvoiceDate'].max().strftime('%Y-%m-%d'),
        'cutoff': CUTOFF.strftime('%Y-%m-%d'),
        'horizonDays': HORIZON_DAYS,
        'customers': int(len(x)),
        'returnedShare': round(100 * float(y.mean()), 1),
        'auc': round(float(auc), 3),
        'accuracy': round(100 * float(acc), 1),
        'baselineAuc': round(float(base_auc), 3),
        'features': cols,
        'importance': {c: round(float(v), 3) for c, v in sorted(zip(cols, imp), key=lambda t: -t[1])},
        'segments': seg_out.to_dict('records'),
        'generated': date.today().isoformat(),
    }
    for rec in summary['segments']:
        for k, v in rec.items():
            if isinstance(v, np.generic):
                rec[k] = v.item()
    with open(os.path.join(a.out, 'customer-retention-summary.json'), 'w', encoding='utf-8') as f:
        json.dump(summary, f, ensure_ascii=False, indent=1)
    print(f"customers {summary['customers']}, returned {summary['returnedShare']}%, "
          f"ROC-AUC {summary['auc']} (recency only {summary['baselineAuc']}), accuracy {summary['accuracy']}%")


if __name__ == '__main__':
    main()
