実践ML評価: 精度、再現率、AUCで正確性を超える

                     ┌─────────────────────────────────────────┐
                     │            Actual Reality               │
                     │    Positive (1)   │    Negative (0)     │
┌─────────┬──────────┼───────────────────┼─────────────────────┤
│ Model   │ Pos (1)  │ True Pos (TP)     │ False Pos (FP)      │
│ Predict ├──────────┼───────────────────┼─────────────────────┤
│         │ Neg (0)  │ False Neg (FN)    │ True Neg (TN)       │
└─────────┴──────────┴───────────────────┴─────────────────────┘

Low Threshold (e.g. 0.1) ───────────► High Recall, Low Precision (Catches everything, high noise)
High Threshold (e.g. 0.8) ──────────► High Precision, Low Recall (Only fires on sure things)

ROC Curve:   Plot of True Positive Rate (Recall) vs False Positive Rate (FPR)
PR Curve:    Plot of Precision vs Recall

from sklearn.metrics import roc_auc_score, average_precision_score

# ROC-AUC is misleadingly high on imbalanced datasets!
roc = roc_auc_score(y_true, y_pred_prob)          # e.g., 0.985 (looks amazing)

# PR-AUC (Average Precision) reflects real-world rare class performance
pr_auc = average_precision_score(y_true, y_pred_prob) # e.g., 0.620 (reveals true difficulty)

from sklearn.calibration import CalibratedClassifierCV, calibration_curve
from sklearn.metrics import brier_score_loss
import lightgbm as lgb

# Train raw classifier
base_model = lgb.LGBMClassifier()
base_model.fit(X_train, y_train)

# Calculate Brier Score (lower is better; 0 = perfect calibration)
raw_brier = brier_score_loss(y_test, base_model.predict_proba(X_test)[:, 1])

# Calibrate using Isotonic Regression or Platt Scaling (Sigmoid)
calibrated_model = CalibratedClassifierCV(
    estimator=base_model,
    method='isotonic', # or 'sigmoid' for smaller datasets (< 1000 samples)
    cv='prefit'
)
calibrated_model.fit(X_val, y_val)

calibrated_brier = brier_score_loss(y_test, calibrated_model.predict_proba(X_test)[:, 1])
print(f"Brier score improved from {raw_brier:.4f} to {calibrated_brier:.4f}")

import numpy as np
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.metrics import precision_recall_curve

# Generate synthetic imbalanced dataset (1% positive class)
X, y = make_classification(
    n_samples=50_000, n_features=20, weights=[0.99, 0.01], random_state=42
)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, stratify=y)

# Fit model
clf = HistGradientBoostingClassifier(class_weight='balanced', random_state=42)
clf.fit(X_train, y_train)

# Predict continuous probabilities
y_probs = clf.predict_proba(X_test)[:, 1]

# Calculate Precision-Recall curve
precisions, recalls, thresholds = precision_recall_curve(y_test, y_probs)

# Define Business Cost Function
COST_FALSE_NEGATIVE = 100.0  # Missed fraud cost
COST_FALSE_POSITIVE = 5.0    # User friction cost

total_costs = []
for t in thresholds:
    y_pred = (y_probs >= t).astype(int)
    fn = np.sum((y_test == 1) & (y_pred == 0))
    fp = np.sum((y_test == 0) & (y_pred == 1))
    cost = (fn * COST_FALSE_NEGATIVE) + (fp * COST_FALSE_POSITIVE)
    total_costs.append(cost)

# Find optimal threshold minimizing business loss
best_idx = np.argmin(total_costs)
optimal_threshold = thresholds[best_idx]

print(f"Default 0.5 Threshold Cost:  ${total_costs[np.abs(thresholds - 0.5).argmin()]:,.2f}")
print(f"Optimized Threshold ({optimal_threshold:.3f}) Cost: ${total_costs[best_idx]:,.2f}")