Day 065 — スタッキング（Stacking Ensemble）— OOF予測でメタモデルを学習する

📚 背景知識（読んでから問題へ）

スタッキングとは？

複数のモデル（ベースモデル）の予測結果を「新しい特徴量」として、さらに別のモデル（メタモデル）に学習させる 二段構造のアンサンブル手法です。
単純な多数決・平均と違い、「どのモデルのどの予測を信頼するか」をデータから自動学習します。

単純アンサンブルとの違い

🏗️ スタッキングの2層構造

🔄 OOF（Out-of-Fold）予測フロー

🌈 モデルの多様性 — なぜ異なるアーキテクチャを選ぶか

📊 スタッキングによる精度向上の目安

Titanic（n=5000）での典型的な結果

🗂️ データスキーマ（Titanic 風 n=5,000）

📝 問題

セットアップコード（最初に実行）

📋 コピーimport numpy as np
import pandas as pd
import xgboost as xgb
import lightgbm as lgb
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold, cross_val_score, train_test_split
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score
import warnings
warnings.filterwarnings('ignore')

# Titanic 風データ生成（n=5000）
np.random.seed(42)
n = 5000
data = pd.DataFrame({
    'Pclass':     np.random.choice([1, 2, 3], n, p=[0.24, 0.21, 0.55]),
    'Sex':        np.random.choice(['male', 'female'], n, p=[0.65, 0.35]),
    'Age':        np.random.normal(29.7, 14.5, n).clip(1, 80),
    'SibSp':      np.random.choice(range(9), n, p=[0.68,0.23,0.04,0.02,0.01,0.01,0.005,0.005,0.01]),
    'Parch':      np.random.choice(range(7), n, p=[0.76,0.13,0.09,0.005,0.005,0.005,0.005]),
    'Fare':       np.abs(np.random.normal(32, 50, n)),
    'Embarked':   np.random.choice(['S', 'C', 'Q'], n, p=[0.72, 0.19, 0.09]),
    'Cabin_type': np.random.choice(['A','B','C','D','E','None'], n,
                                   p=[0.05,0.08,0.12,0.1,0.07,0.58]),
})
data['Survived'] = (
    (data['Sex'] == 'female') * 0.55 + (data['Pclass'] == 1) * 0.25 +
    (data['Age'] < 16) * 0.2 + np.random.normal(0, 0.12, n)
) > 0.4
data['Survived'] = data['Survived'].astype(int)

for col in ['Sex', 'Embarked', 'Cabin_type']:
    data[col] = LabelEncoder().fit_transform(data[col])

features = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Cabin_type']
X = data[features].values
y = data['Survived'].values

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
print("データ準備完了:", X_train.shape, X_test.shape)

問1 — OOF予測生成関数の実装

以下の関数を完成させよ。

📋 コピーdef generate_oof_predictions(model, X, y, X_test, cv, model_name=""):
    """
    OOF予測を生成する関数
    Returns:
        oof_preds: shape (n_train,) — fold-out予測の集合
        test_preds: shape (n_test,) — 全foldの予測の平均
    """
    oof_preds  = np.zeros(len(X))  # (1) ここを埋める
    test_preds = []                # (2) ここを埋める

    for fold, (tr_idx, val_idx) in enumerate(cv.split(X, y)):
        # (3) ここを実装: 学習 → OOF予測 → test予測
        pass

    # (4) test_preds をfold平均にして返す
    return oof_preds, np.mean(test_preds, axis=0)

問2 — 3モデルのOOF予測を生成

📋 コピー# XGBoost
xgb_model = xgb.XGBClassifier(
    n_estimators=300, max_depth=6, learning_rate=0.05,
    subsample=0.8, colsample_bytree=0.8,
    random_state=42, eval_metric='logloss'
)
oof_xgb, test_xgb = generate_oof_predictions(
    xgb_model, X_train, y_train, X_test, cv, "XGB"
)

# LightGBM (ここを実装)

# LogisticRegression with StandardScaler (ここを実装)
# ヒント: Pipeline([('scaler', StandardScaler()), ('clf', LogisticRegression(...))])

問3 — メタモデルの学習と評価

📋 コピー# OOF予測を新しい特徴量として積み重ねる
X_meta_train = np.column_stack([oof_xgb, oof_lgb, oof_lr])
X_meta_test  = np.column_stack([test_xgb, test_lgb, test_lr])

# メタモデル（LogisticRegression）で学習してCV精度を評価せよ
# ここを実装

問4 — 精度比較表を完成させよ

問5 — 考察（自分の言葉で答えよ）

1. スタッキングはなぜ単体モデルより精度が上がることが多いのか？（2〜3文で）
2. ベースモデルが全て XGBoost だった場合、スタッキングの効果はどう変わるか？

💡 ヒント

📋 コピーdef generate_oof_predictions(model, X, y, X_test, cv, model_name=""):
    oof_preds  = np.zeros(len(X))
    test_preds = []

    for fold, (tr_idx, val_idx) in enumerate(cv.split(X, y)):
        X_tr, X_val = X[tr_idx], X[val_idx]
        y_tr        = y[tr_idx]

        model.fit(X_tr, y_tr)
        oof_preds[val_idx] = model.predict_proba(X_val)[:, 1]
        test_preds.append(model.predict_proba(X_test)[:, 1])
        print(f"  {model_name} fold {fold+1}/5 done")

    return oof_preds, np.mean(test_preds, axis=0)

# LightGBM
lgb_model = lgb.LGBMClassifier(
    n_estimators=300, num_leaves=63, learning_rate=0.05,
    min_child_samples=20, random_state=42, verbose=-1
)
oof_lgb, test_lgb = generate_oof_predictions(lgb_model, X_train, y_train, X_test, cv, "LGB")

📋 コピー# LogisticRegression + StandardScaler（Pipeline）
lr_pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('clf',    LogisticRegression(C=1.0, random_state=42, max_iter=1000))
])
oof_lr, test_lr = generate_oof_predictions(lr_pipe, X_train, y_train, X_test, cv, "LR")

# メタモデル
X_meta_train = np.column_stack([oof_xgb, oof_lgb, oof_lr])  # (n_train, 3)
X_meta_test  = np.column_stack([test_xgb, test_lgb, test_lr])  # (n_test, 3)

meta_model = LogisticRegression(C=1.0, random_state=42)
cv_meta = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)
meta_score = cross_val_score(meta_model, X_meta_train, y_train,
                             cv=cv_meta, scoring='accuracy').mean()
print(f"Stacking CV: {meta_score:.4f}")

meta_model.fit(X_meta_train, y_train)
test_score = accuracy_score(y_test, meta_model.predict(X_meta_test))
print(f"Stacking Test: {test_score:.4f}")

✅ 模範解答

📋 コピーimport numpy as np
import pandas as pd
import xgboost as xgb
import lightgbm as lgb
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold, cross_val_score, train_test_split
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score
import warnings
warnings.filterwarnings('ignore')

# ── データ準備 ──
np.random.seed(42)
n = 5000
data = pd.DataFrame({
    'Pclass':     np.random.choice([1, 2, 3], n, p=[0.24, 0.21, 0.55]),
    'Sex':        np.random.choice(['male', 'female'], n, p=[0.65, 0.35]),
    'Age':        np.random.normal(29.7, 14.5, n).clip(1, 80),
    'SibSp':      np.random.choice(range(9), n, p=[0.68,0.23,0.04,0.02,0.01,0.01,0.005,0.005,0.01]),
    'Parch':      np.random.choice(range(7), n, p=[0.76,0.13,0.09,0.005,0.005,0.005,0.005]),
    'Fare':       np.abs(np.random.normal(32, 50, n)),
    'Embarked':   np.random.choice(['S', 'C', 'Q'], n, p=[0.72, 0.19, 0.09]),
    'Cabin_type': np.random.choice(['A','B','C','D','E','None'], n,
                                   p=[0.05,0.08,0.12,0.1,0.07,0.58]),
})
data['Survived'] = (
    (data['Sex'] == 'female') * 0.55 + (data['Pclass'] == 1) * 0.25 +
    (data['Age'] < 16) * 0.2 + np.random.normal(0, 0.12, n)
) > 0.4
data['Survived'] = data['Survived'].astype(int)

for col in ['Sex', 'Embarked', 'Cabin_type']:
    data[col] = LabelEncoder().fit_transform(data[col])

features = ['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'Cabin_type']
X = data[features].values
y = data['Survived'].values

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

# ── OOF予測生成関数 ──
def generate_oof_predictions(model, X, y, X_test, cv, model_name=""):
    oof_preds  = np.zeros(len(X))
    test_preds = []
    for fold, (tr_idx, val_idx) in enumerate(cv.split(X, y)):
        X_tr, X_val = X[tr_idx], X[val_idx]
        y_tr        = y[tr_idx]
        model.fit(X_tr, y_tr)
        oof_preds[val_idx] = model.predict_proba(X_val)[:, 1]
        test_preds.append(model.predict_proba(X_test)[:, 1])
        print(f"  {model_name} fold {fold+1}/5 done")
    return oof_preds, np.mean(test_preds, axis=0)

# ── Level 0: ベースモデルOOF予測 ──
print("=" * 60)
print("Level 0: ベースモデルのOOF予測生成")
print("=" * 60)

# XGBoost
xgb_model = xgb.XGBClassifier(
    n_estimators=300, max_depth=6, learning_rate=0.05,
    subsample=0.8, colsample_bytree=0.8,
    random_state=42, eval_metric='logloss'
)
print("XGBoost:")
oof_xgb, test_xgb = generate_oof_predictions(xgb_model, X_train, y_train, X_test, cv, "XGB")
single_xgb = accuracy_score(y_train, (oof_xgb > 0.5).astype(int))
print(f"  XGBoost OOF Accuracy: {single_xgb:.4f}")

# LightGBM
lgb_model = lgb.LGBMClassifier(
    n_estimators=300, num_leaves=63, learning_rate=0.05,
    min_child_samples=20, random_state=42, verbose=-1
)
print("LightGBM:")
oof_lgb, test_lgb = generate_oof_predictions(lgb_model, X_train, y_train, X_test, cv, "LGB")
single_lgb = accuracy_score(y_train, (oof_lgb > 0.5).astype(int))
print(f"  LightGBM OOF Accuracy: {single_lgb:.4f}")

# LogisticRegression
lr_pipe = Pipeline([
    ('scaler', StandardScaler()),
    ('clf',    LogisticRegression(C=1.0, random_state=42, max_iter=1000))
])
print("LogisticRegression:")
oof_lr, test_lr = generate_oof_predictions(lr_pipe, X_train, y_train, X_test, cv, "LR")
single_lr = accuracy_score(y_train, (oof_lr > 0.5).astype(int))
print(f"  LR OOF Accuracy: {single_lr:.4f}")

# ── Level 1: メタモデル ──
print("\n" + "=" * 60)
print("Level 1: メタモデル（LogisticRegression）")
print("=" * 60)

X_meta_train = np.column_stack([oof_xgb, oof_lgb, oof_lr])
X_meta_test  = np.column_stack([test_xgb, test_lgb, test_lr])

cv_meta = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)
meta_model = LogisticRegression(C=1.0, random_state=42)
meta_cv_score = cross_val_score(meta_model, X_meta_train, y_train,
                                cv=cv_meta, scoring='accuracy').mean()
print(f"Stacking CV Accuracy: {meta_cv_score:.4f}")

meta_model.fit(X_meta_train, y_train)
stacking_test_acc = accuracy_score(y_test, meta_model.predict(X_meta_test))

# ── 精度比較表 ──
print("\n" + "=" * 60)
print("精度比較表")
print("=" * 60)

xgb_model.fit(X_train, y_train)
lgb_model.fit(X_train, y_train)
lr_pipe.fit(X_train, y_train)

rows = [
    ("XGBoost",          single_xgb, accuracy_score(y_test, xgb_model.predict(X_test))),
    ("LightGBM",         single_lgb, accuracy_score(y_test, lgb_model.predict(X_test))),
    ("LogisticRegression", single_lr, accuracy_score(y_test, lr_pipe.predict(X_test))),
    ("Stacking",         meta_cv_score, stacking_test_acc),
]
print(f"{'モデル':<22} {'OOF Accuracy':<16} {'Test Accuracy'}")
print("-" * 55)
for name, oof, test in rows:
    print(f"  {name:<20} {oof:.4f}          {test:.4f}")

# ── メタモデルの係数 ──
coefs = meta_model.coef_[0]
print("\nメタモデルの係数（各ベースモデルの貢献度）:")
for lab, coef in zip(["XGBoost", "LightGBM", "LR"], coefs):
    bar = "█" * int(abs(coef) * 20)
    print(f"  {lab:<18}: {coef:+.4f}  {bar}")

🪜 Step-by-Step 解説

1なぜ OOF が必要か — 過学習の罠

📋 コピー# NG: trainで学習したモデルのtrain予測（過学習した値）をメタ入力にする
xgb_model.fit(X_train, y_train)
bad_preds = xgb_model.predict_proba(X_train)[:, 1]  # スコアが高すぎる！
# → メタモデルが過学習した予測を信頼してしまい、test時に性能が落ちる

# OK: fold-outで予測（見ていないデータへの予測）
oof_preds[val_idx] = model.predict_proba(X_val)[:, 1]
# → 各サンプルは「自分を含まないモデル」が予測 → 汎化性能を正確に反映

2OOF配列の構造を理解する

📋 コピーoof_preds = np.zeros(len(X_train))  # n_train個のゼロ配列

# fold 1: val_idx=[0, 1, ..., 799] → この部分が埋まる
# fold 2: val_idx=[800, ..., 1599] → この部分が埋まる
# fold 3: val_idx=[1600, ..., 2399] → ...
# fold 4: val_idx=[2400, ..., 3199] → ...
# fold 5: val_idx=[3200, ..., 3999] → ...

# 全fold終了後:
# oof_preds = [pred_0, pred_1, ..., pred_3999]
# 全サンプルが「自分を含まないモデル」によって予測されている

3test予測は fold 平均を取る

📋 コピーtest_preds = []
for fold, (tr_idx, val_idx) in enumerate(cv.split(X_train, y_train)):
    model.fit(X_train[tr_idx], y_train[tr_idx])
    test_preds.append(model.predict_proba(X_test)[:, 1])
    # 5回のfoldで5つのモデルがtestを予測

final_test_preds = np.mean(test_preds, axis=0)
# shape: (n_test,) — 5モデルの平均 → バギング効果で安定

4メタ特徴量の形

📋 コピー# ベースモデルが3つ → メタ学習の特徴量は3列
X_meta_train = np.column_stack([oof_xgb, oof_lgb, oof_lr])
# shape: (4000, 3)  —— 「各モデルが各サンプルをどう予測したか」を表す

X_meta_test = np.column_stack([test_xgb, test_lgb, test_lr])
# shape: (1000, 3)  —— testデータの場合も同様

5メタモデルの係数でモデル貢献度を確認

📋 コピーmeta_model.fit(X_meta_train, y_train)
print(meta_model.coef_)
# 例: [[0.8, 0.9, 0.4]]  → XGB:0.8, LGB:0.9, LR:0.4
# → LightGBM の予測をより強く信頼していることがわかる

📐 数学・統計の補足（文系向け）

飲食店の例えで理解するスタッキング

なぜ多様性が精度を上げるのか

🏆 Kaggleでの実践的な使い方

📋 コピー# Kaggle 本番パターン: sklearn の StackingClassifier
from sklearn.ensemble import StackingClassifier

estimators = [
    ('xgb', xgb.XGBClassifier(n_estimators=300, random_state=42, eval_metric='logloss')),
    ('lgb', lgb.LGBMClassifier(n_estimators=300, random_state=42, verbose=-1)),
    ('lr',  Pipeline([('sc', StandardScaler()), ('clf', LogisticRegression(C=1.0))])),
]

stacking_clf = StackingClassifier(
    estimators=estimators,
    final_estimator=LogisticRegression(C=1.0),
    cv=5,
    passthrough=False,  # ベースモデルの予測のみをメタ入力とする
    n_jobs=-1
)
score = cross_val_score(stacking_clf, X_train, y_train, cv=5, scoring='accuracy').mean()
print(f"sklearn Stacking CV: {score:.4f}")

🚫 よくある誤解・ミス

🚀 次のステップ

• 発展: Level 2スタッキング（メタモデルの出力をさらに別モデルで学習）、sklearn.ensemble.StackingClassifier の活用、パススルー（passthrough=True）で元特徴量もメタ学習に含める
• 次回予告（Day 066）: ブレンディング（Blending）と アンサンブル実践 — 加重平均・多数決・スタッキングの選択基準

📋 自己評価（解いた後に記入）