Feature Engineering · Notebook 6 companion articleFeature Engineering for Text Data

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import re

from sklearn.feature_extraction.text import (
    CountVectorizer, TfidfVectorizer, HashingVectorizer, ENGLISH_STOP_WORDS
)
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import (
    cross_val_score, StratifiedKFold, RepeatedStratifiedKFold, train_test_split
)
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.preprocessing import StandardScaler
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.metrics import roc_auc_score, classification_report, confusion_matrix
from sklearn.utils.murmurhash import murmurhash3_bytes_s32

plt.style.use('seaborn-v0_8-whitegrid')
pd.set_option('display.width', 200)
pd.set_option('display.max_columns', 50)

# One split object reused everywhere so every score in this notebook is comparable.
CV5 = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

print('Ready.')

sms = pd.read_csv('datasets/sms_spam.csv')
sms['is_spam'] = (sms['label'] == 'spam').astype(int)

print('shape:', sms.shape)
print()
print(sms['label'].value_counts())
print()
print(f"spam rate: {sms['is_spam'].mean():.4f}")
print()
sms.head(5)

for lab in ['ham', 'spam']:
    print(f'--- {lab.upper()} ---')
    for m in sms.loc[sms['label'] == lab, 'message'].head(3):
        print('  ', m[:110])
    print()

majority = 1 - sms['is_spam'].mean()
print(f'Always-predict-ham accuracy: {majority:.4f}')
print()
print(f'A model can be {majority:.1%} accurate on this dataset while never once')
print('catching a single spam message.')
print('Accuracy is the wrong dial to watch. Through this notebook we report:')
print('  ROC AUC   - how well the model ranks spam above ham overall')
print('  F1 (spam) - the balance of precision and recall on the class we care about')
print('  recall    - the share of actual spam we caught')

n_dup_rows = sms.duplicated(subset='message').sum()
repeated = sms['message'].value_counts()
repeated = repeated[repeated > 1]

print(f'rows                          : {len(sms)}')
print(f'duplicate message rows        : {n_dup_rows}')
print(f'distinct messages that repeat : {len(repeated)}')
print()
flagged = sms.assign(dup=sms.duplicated(subset='message', keep=False))
print('duplicate rate by class:')
print(flagged.groupby('label')['dup'].mean().round(4))
print()
print('duplicated rows by class (absolute):')
print(flagged[flagged['dup']].groupby('label').size())
print()
print('most repeated messages:')
for m, c in repeated.head(5).items():
    print(f'  x{c:<3} {m[:72]}')

def quick_auc(data, vec=None):
    # One TF-IDF + logistic regression pass, scored by ROC AUC.
    vec = vec if vec is not None else TfidfVectorizer()
    pipe = Pipeline([('vec', vec), ('clf', LogisticRegression(max_iter=2000))])
    return cross_val_score(pipe, data['message'].values, data['is_spam'].values,
                           cv=CV5, scoring='roc_auc', n_jobs=1).mean()

sms_dedup = sms.drop_duplicates(subset='message').reset_index(drop=True)

auc_with = quick_auc(sms)
auc_without = quick_auc(sms_dedup)

print(f'with duplicates   n={len(sms):5d}   AUC {auc_with:.4f}')
print(f'deduplicated      n={len(sms_dedup):5d}   AUC {auc_without:.4f}')
print(f'inflation                        {auc_with - auc_without:+.4f}')

sms = sms_dedup
X_text = sms['message'].values
y = sms['is_spam'].values

print(f'working set: {sms.shape}, spam rate {y.mean():.4f}')

toy = [
    "The pizza was great and the service was great",
    "The pizza was cold and the service was slow",
    "Great service, great food, will come back",
    "Cold food, slow service, never again",
]

toy_cv = CountVectorizer()
toy_dtm = toy_cv.fit_transform(toy)

n_cells_toy = toy_dtm.shape[0] * toy_dtm.shape[1]
print('vocabulary:', list(toy_cv.get_feature_names_out()))
print(f'\nmatrix shape: {toy_dtm.shape}  ({toy_dtm.shape[0]} documents x {toy_dtm.shape[1]} words)')
print(f'cells: {n_cells_toy},  non-zero: {toy_dtm.nnz},  zero: {n_cells_toy - toy_dtm.nnz}')

pd.DataFrame(toy_dtm.toarray(),
             columns=toy_cv.get_feature_names_out(),
             index=[f'doc{i+1}' for i in range(len(toy))])

pair = [
    'the food was good but the service was not',
    'the food was not good but the service was',
]

pair_cv = CountVectorizer().fit(pair)
P = pair_cv.transform(pair).toarray()

print('vocabulary:', list(pair_cv.get_feature_names_out()))
print('sentence 1:', P[0])
print('sentence 2:', P[1])
print()
print('Vectors identical?', bool((P[0] == P[1]).all()))

pair_bi = CountVectorizer(ngram_range=(1, 2)).fit(pair)
B = pair_bi.transform(pair).toarray()

print('Identical with bigrams?', bool((B[0] == B[1]).all()))

differing = [name for name, a, b in zip(pair_bi.get_feature_names_out(), B[0], B[1]) if a != b]
print('features that now separate them:', differing)

count_vec = CountVectorizer()
dtm = count_vec.fit_transform(X_text)

n_docs, n_terms = dtm.shape
n_cells = n_docs * n_terms
density = dtm.nnz / n_cells

print(f'documents        : {n_docs:,}')
print(f'vocabulary       : {n_terms:,}')
print(f'cells in matrix  : {n_cells:,}')
print(f'non-zero cells   : {dtm.nnz:,}')
print(f'density          : {density:.5%}')
print(f'average words stored per message: {dtm.nnz / n_docs:.1f}')
print()
dense_mb = n_cells * 8 / 1e6
sparse_mb = (dtm.data.nbytes + dtm.indices.nbytes + dtm.indptr.nbytes) / 1e6
print(f'as a dense float64 array : {dense_mb:,.1f} MB')
print(f'as the sparse matrix     : {sparse_mb:,.2f} MB')
print(f'ratio                    : {dense_mb / sparse_mb:,.0f}x')

# Two different counts, and mixing them up is a genuine trap.
#   term_freq : how many times a word occurs in total
#   doc_freq  : how many documents contain it at least once
# min_df and max_df both work on DOCUMENT frequency, not total occurrences.
term_freq = np.asarray(dtm.sum(axis=0)).ravel()
doc_freq = np.asarray((dtm > 0).sum(axis=0)).ravel()

order = np.argsort(term_freq)[::-1]
terms_sorted = count_vec.get_feature_names_out()[order]
freq_sorted = term_freq[order]

once = int((term_freq == 1).sum())

print(f'vocabulary size             : {n_terms:,}')
print(f'terms occurring exactly once: {once:,}  ({once / n_terms:.1%})')
print()
print(f'top 10 terms cover  {freq_sorted[:10].sum() / freq_sorted.sum():.1%} of all word occurrences')
print(f'top 100 terms cover {freq_sorted[:100].sum() / freq_sorted.sum():.1%} of all word occurrences')
print()
print('most frequent:', ', '.join(f'{t}({f})' for t, f in zip(terms_sorted[:10], freq_sorted[:10])))
print()
print('the two counts disagree, which is why the distinction matters:')
for cutoff in [2, 3, 5]:
    print(f'  min_df={cutoff}: keeps {int((doc_freq >= cutoff).sum()):,} terms by document frequency, '
          f'but {int((term_freq >= cutoff).sum()):,} if you wrongly used total occurrences')

sizes = [250, 500, 1000, 2000, 3000, 4000, len(X_text)]
vocab_growth = [len(CountVectorizer().fit(X_text[:n]).vocabulary_) for n in sizes]

fig, axes = plt.subplots(1, 3, figsize=(16, 4.4))

ax = axes[0]
ax.spy(dtm[:120, :400], markersize=0.7, aspect='auto', color='#2b6cb0')
ax.set_title('Document-term matrix, top-left corner\n(120 messages x 400 terms)', fontsize=10)
ax.set_xlabel('vocabulary term'); ax.set_ylabel('message')

ax = axes[1]
ax.loglog(np.arange(1, len(freq_sorted) + 1), freq_sorted, color='#2b6cb0', lw=1.6)
for cutoff, colour in [(2, '#dd6b20'), (3, '#c53030')]:
    # doc_freq, not term_freq: min_df counts documents, not occurrences.
    n_kept = int((doc_freq >= cutoff).sum())
    ax.axvline(n_kept, color=colour, ls='--', lw=1.3,
               label=f'min_df={cutoff} keeps {n_kept:,}')
ax.set_title('Word frequency is deeply unequal\n(log-log)', fontsize=10)
ax.set_xlabel('term rank'); ax.set_ylabel('total occurrences')
ax.legend(fontsize=8)

ax = axes[2]
ax.plot(sizes, vocab_growth, 'o-', color='#2b6cb0')
ax.set_title('Vocabulary keeps growing with corpus size\n(it never settles)', fontsize=10)
ax.set_xlabel('documents used'); ax.set_ylabel('distinct terms')

plt.tight_layout()
plt.show()

print('vocabulary growth:')
for n, v in zip(sizes, vocab_growth):
    print(f'  {n:5,} documents -> {v:6,} terms')

def score_vec(vec, metric='roc_auc'):
    # Vectoriser and classifier live in one Pipeline so the vocabulary is
    # rebuilt from the training rows of every fold. Section 6 explains why.
    pipe = Pipeline([('vec', vec), ('clf', LogisticRegression(max_iter=3000))])
    return cross_val_score(pipe, X_text, y, cv=CV5, scoring=metric, n_jobs=1).mean()


print('min_df / max_df sweep (TF-IDF):')
print(f"{'setting':30}{'vocabulary':>12}{'ROC AUC':>10}")
for kwargs in [dict(), dict(min_df=2), dict(min_df=3), dict(min_df=5),
               dict(min_df=2, max_df=0.5)]:
    vocab = len(TfidfVectorizer(**kwargs).fit(X_text).vocabulary_)
    label = ', '.join(f'{k}={v}' for k, v in kwargs.items()) or 'defaults'
    print(f'{label:30}{vocab:12,}{score_vec(TfidfVectorizer(**kwargs)):10.4f}')

print('n-gram range (TF-IDF, min_df=2):')
print(f"{'range':12}{'vocabulary':>12}{'ROC AUC':>10}{'F1 (spam)':>12}")
for ng in [(1, 1), (1, 2), (1, 3)]:
    v = TfidfVectorizer(ngram_range=ng, min_df=2)
    vocab = len(v.fit(X_text).vocabulary_)
    print(f'{str(ng):12}{vocab:12,}'
          f'{score_vec(TfidfVectorizer(ngram_range=ng, min_df=2)):10.4f}'
          f'{score_vec(TfidfVectorizer(ngram_range=ng, min_df=2), "f1"):12.4f}')

toy_tfidf = TfidfVectorizer()
toy_tfidf.fit(toy)

idf_table = (pd.DataFrame({'term': toy_tfidf.get_feature_names_out(),
                           'idf': toy_tfidf.idf_.round(3)})
             .sort_values('idf')
             .reset_index(drop=True))

print('IDF on the four toy reviews:')
print(idf_table.to_string(index=False))
print()
print("'service' appears in all 4 documents and sits exactly on the floor of 1.0.")
print("'never' and 'again' appear in 1 document each, so they get the highest weight.")
print()
n_toy, df_service = len(toy), 4
print('checking the formula by hand for "service":')
print(f'  ln((1 + {n_toy}) / (1 + {df_service})) + 1 = {np.log((1 + n_toy) / (1 + df_service)) + 1:.4f}')
print(f'  the textbook log(N / df) would give   = {np.log(n_toy / df_service):.4f}')

print('Real corpus, TF-IDF with min_df=2:')
tfidf_full = TfidfVectorizer(min_df=2).fit(X_text)
names = np.array(tfidf_full.get_feature_names_out())
idf = tfidf_full.idf_
o = np.argsort(idf)

print('\n  lowest IDF (common, pushed down):')
print('   ', ', '.join(f'{names[i]} ({idf[i]:.2f})' for i in o[:10]))
print('\n  highest IDF (rare, pushed up):')
print('   ', ', '.join(f'{names[i]} ({idf[i]:.2f})' for i in o[-6:]))

auc_count = score_vec(CountVectorizer())
auc_tfidf = score_vec(TfidfVectorizer())

print(f'CountVectorizer   AUC {auc_count:.4f}')
print(f'TfidfVectorizer   AUC {auc_tfidf:.4f}')
print(f'difference            {auc_tfidf - auc_count:+.4f}')

# Fitted on the whole corpus deliberately. We are reading the coefficients to
# understand the data, not estimating performance, so there is no held-out claim
# to protect here. Section 8 covers when this would not be acceptable.
M = tfidf_full.transform(X_text)
inspect_model = LogisticRegression(max_iter=3000).fit(M, y)
coef = inspect_model.coef_[0]
rank = np.argsort(coef)

print('most spam-ish terms:')
print('  ', ', '.join(f'{names[i]} ({coef[i]:+.2f})' for i in rank[::-1][:12]))
print('\nmost ham-ish terms:')
print('  ', ', '.join(f'{names[i]} ({coef[i]:+.2f})' for i in rank[:12]))

print(f'sklearn English stopword list: {len(ENGLISH_STOP_WORDS)} words')
print()
check = ['not', 'no', 'never', 'nor', 'none', 'cannot', 'without', 'against', 'but', 'very']
for w in check:
    print(f'  {w!r:10} in the stopword list: {w in ENGLISH_STOP_WORDS}')

def clean_text(text, stopwords=ENGLISH_STOP_WORDS):
    # The textbook recipe: lowercase, drop punctuation and digits, drop stopwords.
    text = text.lower()
    text = re.sub(r'[^\w\s]', ' ', text)
    text = re.sub(r'\d+', ' ', text)
    tokens = [t for t in text.split() if t not in stopwords]
    return ' '.join(tokens)


pairs = [
    ("The movie was not good at all.",   "The movie was good."),
    ("I would never buy this again.",    "I would buy this again."),
    ("This is not worth the money.",     "This is worth the money."),
]

for negative, positive in pairs:
    cn, cp = clean_text(negative), clean_text(positive)
    print(f'  {negative!r}')
    print(f'    -> {cn!r}')
    print(f'  {positive!r}')
    print(f'    -> {cp!r}')
    print(f'    identical after cleaning: {cn == cp}')
    print()

rev = pd.read_csv('datasets/review_sentiment.csv')
print('shape:', rev.shape)
print(rev['sentiment'].value_counts().to_dict())
print(rev['source'].value_counts().to_dict())
print()

X_rev, y_rev = rev['text'].values, rev['sentiment'].values

NEGATION = {'not', 'no', 'never', 'nor', 'nothing', 'none', 'cannot', 'without'}
has_negation = rev['text'].str.lower().str.findall(r"[a-z']+").apply(
    lambda ws: bool(NEGATION & set(ws)))

print(f'sentences containing a negation stopword: {has_negation.sum()} / {len(rev)} '
      f'({has_negation.mean():.1%})')
print(f'positive rate among those sentences : {y_rev[has_negation.values].mean():.3f}')
print(f'positive rate overall               : {y_rev.mean():.3f}')

KEEP_THESE = {'not', 'no', 'never', 'nor', 'nothing', 'none', 'cannot',
              'without', 'against', 'but', 'very', 'too'}
CURATED = ENGLISH_STOP_WORDS - KEEP_THESE

rcv = RepeatedStratifiedKFold(n_splits=5, n_repeats=6, random_state=42)

def score_sentiment(**vec_kwargs):
    pipe = Pipeline([('vec', TfidfVectorizer(**vec_kwargs)),
                     ('clf', LogisticRegression(max_iter=2000))])
    s = cross_val_score(pipe, X_rev, y_rev, cv=rcv, scoring='accuracy', n_jobs=1)
    return s.mean(), s.std()

print(f'full stopword list: {len(ENGLISH_STOP_WORDS)} words')
print(f'curated list      : {len(CURATED)} words  ({len(KEEP_THESE)} put back)')
print()
print('sentiment accuracy over 30 folds:')
print(f"{'strategy':40}{'accuracy':>10}{'sd':>9}")
for label, kwargs in [
    ('keep every word',                       dict()),
    ('remove sklearn stopwords',              dict(stop_words='english')),
    ('remove stopwords, keep negation',       dict(stop_words=list(CURATED))),
]:
    m, sd = score_sentiment(**kwargs)
    print(f'{label:40}{m:10.4f}{sd:9.4f}')

print('SMS spam, same comparison (5-fold AUC):')
print(f'  keep every word          {score_vec(TfidfVectorizer()):.4f}')
print(f'  remove sklearn stopwords {score_vec(TfidfVectorizer(stop_words="english")):.4f}')

Xtr, Xte, ytr, yte = train_test_split(X_text, y, test_size=0.25,
                                      stratify=y, random_state=42)

# The wrong way: vocabulary built from every document, test included.
vec_leaky = TfidfVectorizer(min_df=2).fit(X_text)
model_leaky = LogisticRegression(max_iter=2000).fit(vec_leaky.transform(Xtr), ytr)
auc_leaky = roc_auc_score(yte, model_leaky.predict_proba(vec_leaky.transform(Xte))[:, 1])

# The right way: vocabulary built from training documents only.
vec_clean = TfidfVectorizer(min_df=2).fit(Xtr)
model_clean = LogisticRegression(max_iter=2000).fit(vec_clean.transform(Xtr), ytr)
auc_clean = roc_auc_score(yte, model_clean.predict_proba(vec_clean.transform(Xte))[:, 1])

only_from_test = set(vec_leaky.vocabulary_) - set(vec_clean.vocabulary_)

print(f'vocabulary fit on ALL data : {len(vec_leaky.vocabulary_):5,} terms   test AUC {auc_leaky:.4f}')
print(f'vocabulary fit on TRAIN    : {len(vec_clean.vocabulary_):5,} terms   test AUC {auc_clean:.4f}')
print(f'difference                                        {auc_leaky - auc_clean:+.4f}')
print()
print(f'terms in the model only because test data was included: {len(only_from_test):,}')
print('  examples:', ', '.join(sorted(only_from_test)[:8]))

print('how much of the vocabulary comes from test documents, by split size:')
print(f"{'test_size':>10}{'train rows':>12}{'train vocab':>13}{'from test':>11}{'share':>9}")
for test_size in [0.25, 0.50, 0.75, 0.90]:
    Xa, _, _, _ = train_test_split(X_text, y, test_size=test_size,
                                   stratify=y, random_state=42)
    v_all = TfidfVectorizer(min_df=2).fit(X_text)
    v_tr = TfidfVectorizer(min_df=2).fit(Xa)
    from_test = len(set(v_all.vocabulary_) - set(v_tr.vocabulary_))
    print(f'{test_size:10.2f}{len(Xa):12,}{len(v_tr.vocabulary_):13,}'
          f'{from_test:11,}{from_test / len(v_all.vocabulary_):9.1%}')

# Everything before this point in the notebook already used a Pipeline for exactly
# this reason: cross_val_score refits every step on each fold's training rows.
pipe = Pipeline([
    ('vec', TfidfVectorizer(min_df=2)),
    ('clf', LogisticRegression(max_iter=3000)),
])

print('Inside cross_val_score, this pipeline rebuilds the vocabulary 5 times,')
print('once per fold, each time from that fold\'s training documents only.')
print()
print(f'AUC {cross_val_score(pipe, X_text, y, cv=CV5, scoring="roc_auc", n_jobs=1).mean():.4f}')

class StyleFeatures(BaseEstimator, TransformerMixin):
    # Turns raw strings into a dense array of writing-style statistics.
    # Stateless: fit does nothing, so there is nothing here that can leak.
    names = ['n_chars', 'n_words', 'avg_word_len', 'digit_ratio', 'upper_ratio',
             'n_exclam', 'n_currency', 'has_url', 'n_longnum']

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        rows = []
        for text in X:
            words = text.split()
            length = max(len(text), 1)
            rows.append([
                len(text),
                len(words),
                np.mean([len(w) for w in words]) if words else 0.0,
                sum(c.isdigit() for c in text) / length,
                sum(c.isupper() for c in text) / length,
                text.count('!'),
                sum(text.count(c) for c in '£$€'),
                float(bool(re.search(r'http|www\.', text, re.I))),
                len(re.findall(r'\d{5,}', text)),          # phone numbers, short codes
            ])
        return np.asarray(rows, dtype=float)

    def get_feature_names_out(self, input_features=None):
        return np.array(self.names)


style_matrix = pd.DataFrame(StyleFeatures().transform(X_text),
                            columns=StyleFeatures.names)

by_class = style_matrix.assign(is_spam=y).groupby('is_spam').mean().T
by_class.columns = ['ham', 'spam']
by_class['ratio'] = (by_class['spam'] / by_class['ham'].replace(0, np.nan)).round(1)
print('mean value by class:')
print(by_class.round(3).to_string())

# A mean of 0.998 does not imply "one per message". Check the actual shape.
print('\nshare of messages containing at least one long digit run:')
for label, mask in [('ham', y == 0), ('spam', y == 1)]:
    col = style_matrix.loc[mask, 'n_longnum']
    print(f'  {label:5} {(col > 0).mean():7.2%}   ({int((col > 0).sum()):,} of {int(mask.sum()):,})')
print('\ncount distribution for spam:')
print(style_matrix.loc[y == 1, 'n_longnum'].astype(int).value_counts().sort_index().to_string())

corr = (style_matrix.assign(is_spam=y).corr()['is_spam']
        .drop('is_spam').sort_values())

fig, axes = plt.subplots(1, 3, figsize=(16, 4.4))

ax = axes[0]
colours = ['#c53030' if v < 0 else '#2b6cb0' for v in corr.values]
ax.barh(corr.index, corr.values, color=colours)
ax.set_title('Correlation of each style feature with is_spam', fontsize=10)
ax.set_xlabel('correlation')

ax = axes[1]
bins = np.linspace(0, 300, 45)
ax.hist(style_matrix.loc[y == 0, 'n_chars'], bins=bins, alpha=0.65,
        label='ham', color='#2b6cb0', density=True)
ax.hist(style_matrix.loc[y == 1, 'n_chars'], bins=bins, alpha=0.65,
        label='spam', color='#c53030', density=True)
ax.set_title('Message length separates the classes', fontsize=10)
ax.set_xlabel('characters'); ax.set_ylabel('density'); ax.legend()

ax = axes[2]
bins = np.linspace(0, 0.4, 45)
ax.hist(style_matrix.loc[y == 0, 'digit_ratio'], bins=bins, alpha=0.65,
        label='ham', color='#2b6cb0', density=True)
ax.hist(style_matrix.loc[y == 1, 'digit_ratio'], bins=bins, alpha=0.65,
        label='spam', color='#c53030', density=True)
ax.set_title('Digit ratio separates them even better', fontsize=10)
ax.set_xlabel('fraction of characters that are digits'); ax.legend()

plt.tight_layout()
plt.show()

print('correlation with is_spam:')
print(corr.sort_values(ascending=False).round(3).to_string())

style_pipe = Pipeline([('style', StyleFeatures()), ('scale', StandardScaler())])

def evaluate(estimator, label):
    row = {'model': label}
    for metric in ['roc_auc', 'f1', 'recall', 'precision']:
        row[metric] = cross_val_score(estimator, X_text, y, cv=CV5,
                                      scoring=metric, n_jobs=1).mean()
    return row


results = [
    evaluate(Pipeline([('v', style_pipe), ('c', LogisticRegression(max_iter=3000))]),
             'style only (9 columns)'),
    evaluate(Pipeline([('v', TfidfVectorizer(min_df=2)), ('c', LogisticRegression(max_iter=3000))]),
             'word TF-IDF (3,802 columns)'),
]

print(pd.DataFrame(results).round(4).to_string(index=False))

rf_style = Pipeline([('style', StyleFeatures()),
                     ('clf', RandomForestClassifier(n_estimators=300,
                                                    random_state=42, n_jobs=1))])
print('Random forest on the same 9 style features:')
print(f"  ROC AUC {cross_val_score(rf_style, X_text, y, cv=CV5, scoring='roc_auc', n_jobs=1).mean():.4f}")
print(f"  F1      {cross_val_score(rf_style, X_text, y, cv=CV5, scoring='f1', n_jobs=1).mean():.4f}")

word_tfidf = TfidfVectorizer(min_df=2)
char_tfidf = TfidfVectorizer(analyzer='char_wb', ngram_range=(2, 4), min_df=2)

def combo(*branches):
    return Pipeline([
        ('features', FeatureUnion(list(branches))),
        ('clf', LogisticRegression(max_iter=3000)),
    ])

table = [
    evaluate(Pipeline([('v', style_pipe), ('c', LogisticRegression(max_iter=3000))]),
             'style only'),
    evaluate(Pipeline([('v', word_tfidf), ('c', LogisticRegression(max_iter=3000))]),
             'word TF-IDF'),
    evaluate(combo(('w', word_tfidf), ('s', style_pipe)),
             'word TF-IDF + style'),
]
print(pd.DataFrame(table).round(4).to_string(index=False))

print(f"{'analyzer':10}{'range':10}{'vocabulary':>12}{'ROC AUC':>10}{'F1':>9}")
for analyzer, ng in [('word', (1, 1)), ('word', (1, 2)),
                     ('char_wb', (2, 4)), ('char_wb', (3, 5))]:
    v = TfidfVectorizer(analyzer=analyzer, ngram_range=ng, min_df=2)
    vocab = len(v.fit(X_text).vocabulary_)
    est = Pipeline([('v', TfidfVectorizer(analyzer=analyzer, ngram_range=ng, min_df=2)),
                    ('c', LogisticRegression(max_iter=3000))])
    auc = cross_val_score(est, X_text, y, cv=CV5, scoring='roc_auc', n_jobs=1).mean()
    f1 = cross_val_score(est, X_text, y, cv=CV5, scoring='f1', n_jobs=1).mean()
    print(f'{analyzer:10}{str(ng):10}{vocab:12,}{auc:10.4f}{f1:9.4f}')

# alternate_sign=False keeps every value non-negative, which some downstream
# steps require. The default True lets colliding terms cancel on average, and on
# this corpus the two settings land within 0.002 of each other.
print('HashingVectorizer, no vocabulary stored anywhere:')
print(f"{'n_features':>12}{'collided terms':>17}{'ROC AUC':>10}")

vocab_terms = list(CountVectorizer().fit(X_text).vocabulary_.keys())

for n_features in [2 ** 10, 2 ** 12, 2 ** 14, 2 ** 18]:
    est = Pipeline([('v', HashingVectorizer(n_features=n_features,
                                            alternate_sign=False, norm='l2')),
                    ('c', LogisticRegression(max_iter=3000))])
    auc = cross_val_score(est, X_text, y, cv=CV5, scoring='roc_auc', n_jobs=1).mean()

    # How many of the real vocabulary terms share a column with another term?
    buckets = np.array([murmurhash3_bytes_s32(t.encode('utf-8'), 0) % n_features
                        for t in vocab_terms])
    collided = len(vocab_terms) - len(np.unique(buckets))
    print(f'{n_features:12,}{collided:11,} ({collided / len(vocab_terms):4.0%}){auc:10.4f}')

print()
# Fresh throwaway vectoriser: fitting the shared `word_tfidf` here would leave it
# fitted on the full corpus, which is the exact habit section 8 argued against.
reference_vocab = len(TfidfVectorizer(min_df=2).fit(X_text).vocabulary_)
print(f'For reference, TF-IDF stores a {reference_vocab:,}-term dictionary and scores 0.9894.')

final_table = [
    evaluate(Pipeline([('v', style_pipe), ('c', LogisticRegression(max_iter=3000))]), 'style only'),
    evaluate(Pipeline([('v', word_tfidf), ('c', LogisticRegression(max_iter=3000))]), 'word TF-IDF'),
    evaluate(Pipeline([('v', char_tfidf), ('c', LogisticRegression(max_iter=3000))]), 'char TF-IDF'),
    evaluate(combo(('w', word_tfidf), ('s', style_pipe)), 'word + style'),
    evaluate(combo(('c', char_tfidf), ('s', style_pipe)), 'char + style'),
    evaluate(combo(('w', word_tfidf), ('c', char_tfidf), ('s', style_pipe)), 'word + char + style'),
]
print('5-fold cross-validation, deduplicated SMS:')
print(pd.DataFrame(final_table).round(4).to_string(index=False))

summary = pd.DataFrame(final_table).set_index('model')

fig, axes = plt.subplots(1, 2, figsize=(15, 4.6))

ax = axes[0]
x = np.arange(len(summary))
width = 0.27
for offset, metric, colour in [(-width, 'roc_auc', '#2b6cb0'),
                               (0.0, 'f1', '#dd6b20'),
                               (width, 'recall', '#c53030')]:
    ax.bar(x + offset, summary[metric], width, label=metric, color=colour)
ax.set_xticks(x)
ax.set_xticklabels(summary.index, rotation=30, ha='right', fontsize=8)
ax.set_ylim(0.70, 1.005)
ax.set_ylabel('score')
ax.set_title('AUC looks flat while F1 and recall move a lot', fontsize=10)
ax.legend(fontsize=8)

ax = axes[1]
base = summary.loc['word TF-IDF']
delta = (summary - base).drop(index='word TF-IDF')
for metric, colour, marker in [('roc_auc', '#2b6cb0', 'o'),
                               ('f1', '#dd6b20', 's'),
                               ('recall', '#c53030', '^')]:
    ax.plot(delta.index, delta[metric], marker + '-', color=colour, label=metric)
ax.axhline(0, color='#666666', lw=1)
ax.set_xticks(range(len(delta)))
ax.set_xticklabels(delta.index, rotation=30, ha='right', fontsize=8)
ax.set_ylabel('change vs word TF-IDF')
ax.set_title('Same experiments, measured as a change from the word baseline', fontsize=10)
ax.legend(fontsize=8)

plt.tight_layout()
plt.show()

print('spread of each metric across the six models:')
for m in ['roc_auc', 'f1', 'recall']:
    print(f'  {m:8} min {summary[m].min():.4f}  max {summary[m].max():.4f}'
          f'  spread {summary[m].max() - summary[m].min():.4f}')

print(f'Hold-out set: {len(Xte)} messages, {int(yte.sum())} of them spam')
print()

candidates = {
    'char + style': combo(('c', char_tfidf), ('s', style_pipe)),
    'word + char + style': combo(('w', word_tfidf), ('c', char_tfidf), ('s', style_pipe)),
}

fitted = {}
rows = []
for name, est in candidates.items():
    est.fit(Xtr, ytr)
    fitted[name] = est
    pred = est.predict(Xte)
    proba = est.predict_proba(Xte)[:, 1]
    rows.append({
        'model': name,
        'test AUC': roc_auc_score(yte, proba),
        'spam recall': (pred[yte == 1] == 1).mean(),
        'spam precision': (yte[pred == 1] == 1).mean(),
        'false positives': int(((pred == 1) & (yte == 0)).sum()),
    })

print(pd.DataFrame(rows).round(4).to_string(index=False))

best_name = 'word + char + style'
pred = fitted[best_name].predict(Xte)

print(f'{best_name} on the hold-out:')
print()
print(classification_report(yte, pred, target_names=['ham', 'spam'], digits=4))
print('confusion matrix:')
print(pd.DataFrame(confusion_matrix(yte, pred),
                   index=['actual ham', 'actual spam'],
                   columns=['predicted ham', 'predicted spam']))