Model Bias Analysis

Once a model is trained, we must assess whether it exhibits bias in its predictions. This chapter covers techniques for analyzing trained models to detect and quantify bias across demographic groups.

Introduction to Model Analysis

Even with clean data, models can learn and amplify biases through the training process. Systematic model analysis is essential to ensure fair AI systems that benefit all users equally.

Disaggregated Performance Evaluation

The foundation of model bias detection is evaluating performance separately for each demographic group rather than only looking at aggregate metrics.

# Disaggregated Evaluation Framework
import numpy as np
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score

def disaggregated_evaluation(model, X_test, y_test, protected_attr):
    """
    Evaluate model performance separately for each group
    """
    groups = np.unique(protected_attr)
    y_pred = model.predict(X_test)

    print("DISAGGREGATED PERFORMANCE EVALUATION")
    print("=" * 80)
    print(f"{'Group':<15} {'Accuracy':<12} {'Precision':<12} {'Recall':<12} {'F1 Score':<12}")
    print("-" * 80)

    results = {}
    for group in groups:
        mask = protected_attr == group

        acc = accuracy_score(y_test[mask], y_pred[mask])
        prec = precision_score(y_test[mask], y_pred[mask], zero_division=0)
        rec = recall_score(y_test[mask], y_pred[mask], zero_division=0)
        f1 = f1_score(y_test[mask], y_pred[mask], zero_division=0)

        results[group] = {'accuracy': acc, 'precision': prec, 'recall': rec, 'f1': f1}
        print(f"{group:<15} {acc:<12.3f} {prec:<12.3f} {rec:<12.3f} {f1:<12.3f}")

    # Calculate disparities
    print("\n" + "=" * 80)
    print("PERFORMANCE DISPARITIES")
    print("-" * 80)

    for metric in ['accuracy', 'precision', 'recall', 'f1']:
        values = [r[metric] for r in results.values()]
        disparity = max(values) - min(values)
        status = "⚠️ HIGH" if disparity > 0.1 else "✓ OK"
        print(f"{metric.capitalize():<15} Disparity: {disparity:.3f} {status}")

    return results

Confusion Matrix Analysis by Group

Examining confusion matrices for each group reveals patterns in how errors are distributed.

# Group-Specific Confusion Matrices
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt

def confusion_matrix_by_group(y_true, y_pred, protected_attr):
    """
    Generate confusion matrices for each demographic group
    """
    groups = np.unique(protected_attr)

    print("CONFUSION MATRICES BY GROUP")
    print("=" * 60)

    for group in groups:
        mask = protected_attr == group
        cm = confusion_matrix(y_true[mask], y_pred[mask])

        tn, fp, fn, tp = cm.ravel() if cm.size == 4 else (0, 0, 0, 0)

        print(f"\n{group}:")
        print(f"  True Negatives:  {tn:>6}  |  False Positives: {fp:>6}")
        print(f"  False Negatives: {fn:>6}  |  True Positives:  {tp:>6}")

        # Calculate rates
        tpr = tp / (tp + fn) if (tp + fn) > 0 else 0
        fpr = fp / (fp + tn) if (fp + tn) > 0 else 0
        tnr = tn / (tn + fp) if (tn + fp) > 0 else 0
        fnr = fn / (fn + tp) if (fn + tp) > 0 else 0

        print(f"  TPR: {tpr:.3f}  FPR: {fpr:.3f}  TNR: {tnr:.3f}  FNR: {fnr:.3f}")

    return True

Decision Threshold Analysis

For probabilistic classifiers, the decision threshold can significantly impact fairness. Different thresholds may be needed for different groups to achieve fairness criteria.

# Threshold Optimization for Fairness
def analyze_thresholds(model, X_test, y_test, protected_attr, metric='tpr'):
    """
    Analyze how different thresholds affect fairness
    """
    y_proba = model.predict_proba(X_test)[:, 1]
    groups = np.unique(protected_attr)

    thresholds = np.linspace(0, 1, 101)
    results = {group: [] for group in groups}

    for threshold in thresholds:
        y_pred = (y_proba >= threshold).astype(int)

        for group in groups:
            mask = protected_attr == group

            if metric == 'tpr':
                tp = np.sum((y_test[mask] == 1) & (y_pred[mask] == 1))
                fn = np.sum((y_test[mask] == 1) & (y_pred[mask] == 0))
                value = tp / (tp + fn) if (tp + fn) > 0 else 0
            elif metric == 'fpr':
                fp = np.sum((y_test[mask] == 0) & (y_pred[mask] == 1))
                tn = np.sum((y_test[mask] == 0) & (y_pred[mask] == 0))
                value = fp / (fp + tn) if (fp + tn) > 0 else 0

            results[group].append(value)

    # Find optimal threshold for fairness
    min_disparity = float('inf')
    best_threshold = 0.5

    for i, threshold in enumerate(thresholds):
        values = [results[group][i] for group in groups]
        disparity = max(values) - min(values)
        if disparity < min_disparity:
            min_disparity = disparity
            best_threshold = threshold

    print(f"Optimal threshold for {metric.upper()} parity: {best_threshold:.3f}")
    print(f"Minimum disparity: {min_disparity:.3f}")

    return best_threshold, results

Feature Importance and SHAP Analysis

Understanding which features drive predictions helps identify potential sources of bias.

# Feature Importance Analysis
def analyze_feature_importance_by_group(model, X_test, protected_attr, feature_names):
    """
    Analyze if feature importance differs across groups
    """
    from sklearn.inspection import permutation_importance

    groups = np.unique(protected_attr)

    print("FEATURE IMPORTANCE BY GROUP")
    print("=" * 80)

    for group in groups:
        mask = protected_attr == group
        X_group = X_test[mask]
        y_group = y_test[mask]

        # Calculate permutation importance
        perm_importance = permutation_importance(
            model, X_group, y_group, n_repeats=10, random_state=42
        )

        print(f"\n{group} - Top 5 Most Important Features:")
        indices = perm_importance.importances_mean.argsort()[-5:][::-1]

        for idx in indices:
            importance = perm_importance.importances_mean[idx]
            print(f"  {feature_names[idx]:<30} {importance:.4f}")

    print("\n⚠️ Look for features that are important for one group but not others")
    print("   This may indicate differential treatment")

Calibration Analysis

A well-calibrated model's probability predictions should match observed frequencies across all groups.

# Calibration Analysis by Group
from sklearn.calibration import calibration_curve

def calibration_analysis(model, X_test, y_test, protected_attr, n_bins=10):
    """
    Assess calibration separately for each group
    """
    y_proba = model.predict_proba(X_test)[:, 1]
    groups = np.unique(protected_attr)

    print("CALIBRATION ANALYSIS")
    print("=" * 80)

    calibration_errors = {}

    for group in groups:
        mask = protected_attr == group

        prob_true, prob_pred = calibration_curve(
            y_test[mask],
            y_proba[mask],
            n_bins=n_bins,
            strategy='quantile'
        )

        # Calculate calibration error (ECE - Expected Calibration Error)
        calibration_error = np.mean(np.abs(prob_true - prob_pred))
        calibration_errors[group] = calibration_error

        print(f"\n{group}:")
        print(f"  Expected Calibration Error: {calibration_error:.4f}")

        print(f"  {'Predicted':<12} {'Actual':<12} {'Difference':<12}")
        print(f"  {'-'*36}")
        for pred, true in zip(prob_pred, prob_true):
            diff = abs(pred - true)
            print(f"  {pred:<12.3f} {true:<12.3f} {diff:<12.3f}")

    # Check for calibration disparities
    errors = list(calibration_errors.values())
    disparity = max(errors) - min(errors)

    print(f"\nCalibration error disparity: {disparity:.4f}")
    if disparity > 0.05:
        print("⚠️ Significant calibration disparity detected")

    return calibration_errors

Subgroup Analysis

Examine performance on specific subgroups that may be particularly vulnerable to bias.

# Subgroup Performance Analysis
def subgroup_analysis(model, X_test, y_test, protected_attrs):
    """
    Analyze performance across intersectional subgroups
    """
    # Create subgroup identifiers
    subgroups = X_test[protected_attrs].apply(
        lambda x: '_'.join(x.astype(str)), axis=1
    )

    unique_subgroups = subgroups.unique()
    y_pred = model.predict(X_test)

    print("SUBGROUP ANALYSIS")
    print("=" * 80)
    print(f"{'Subgroup':<30} {'Count':<10} {'Accuracy':<12} {'Precision':<12} {'Recall':<12}")
    print("-" * 80)

    subgroup_results = []

    for subgroup in unique_subgroups:
        mask = subgroups == subgroup
        count = mask.sum()

        if count < 10:  # Skip very small subgroups
            continue

        acc = accuracy_score(y_test[mask], y_pred[mask])
        prec = precision_score(y_test[mask], y_pred[mask], zero_division=0)
        rec = recall_score(y_test[mask], y_pred[mask], zero_division=0)

        subgroup_results.append({
            'subgroup': subgroup,
            'count': count,
            'accuracy': acc,
            'precision': prec,
            'recall': rec
        })

        print(f"{subgroup:<30} {count:<10} {acc:<12.3f} {prec:<12.3f} {rec:<12.3f}")

    # Identify worst-performing subgroups
    sorted_results = sorted(subgroup_results, key=lambda x: x['accuracy'])

    print("\n⚠️ Worst Performing Subgroups:")
    for result in sorted_results[:3]:
        print(f"  {result['subgroup']}: Accuracy = {result['accuracy']:.3f}")

    return subgroup_results

Counterfactual Fairness Testing

Test whether changing protected attributes (while keeping other features constant) changes predictions.

# Counterfactual Fairness Test
def counterfactual_test(model, X_test, protected_attr_col, protected_attr_values):
    """
    Test if predictions change when only protected attribute changes
    """
    X_counterfactual = X_test.copy()

    original_predictions = model.predict_proba(X_test)[:, 1]
    counterfactual_predictions = {}

    for value in protected_attr_values:
        X_counterfactual[protected_attr_col] = value
        counterfactual_predictions[value] = model.predict_proba(X_counterfactual)[:, 1]

    # Calculate how often predictions change
    changes = 0
    total = len(X_test)

    for i in range(total):
        preds = [counterfactual_predictions[v][i] for v in protected_attr_values]
        if max(preds) - min(preds) > 0.1:  # Significant change
            changes += 1

    change_rate = changes / total

    print(f"Counterfactual change rate: {change_rate:.3f}")
    if change_rate > 0.1:
        print("⚠️ Model is sensitive to protected attribute")
        print("   This suggests potential discrimination")

    return change_rate

Bias Amplification Detection

Check if the model amplifies biases present in the training data.

# Bias Amplification Analysis
def detect_bias_amplification(train_data, model, X_test, protected_attr):
    """
    Compare bias in training data vs model predictions
    """
    # Measure bias in training data
    train_positive_rates = {}
    for group in train_data[protected_attr].unique():
        mask = train_data[protected_attr] == group
        train_positive_rates[group] = train_data[mask]['label'].mean()

    # Measure bias in predictions
    y_pred = model.predict(X_test)
    test_protected = X_test[protected_attr]

    pred_positive_rates = {}
    for group in test_protected.unique():
        mask = test_protected == group
        pred_positive_rates[group] = y_pred[mask].mean()

    print("BIAS AMPLIFICATION ANALYSIS")
    print("=" * 80)
    print(f"{'Group':<15} {'Training %':<15} {'Prediction %':<15} {'Change':<15}")
    print("-" * 80)

    for group in train_positive_rates.keys():
        train_rate = train_positive_rates.get(group, 0) * 100
        pred_rate = pred_positive_rates.get(group, 0) * 100
        change = pred_rate - train_rate

        status = "⚠️ AMPLIFIED" if abs(change) > 5 else "✓ OK"
        print(f"{group:<15} {train_rate:<15.2f} {pred_rate:<15.2f} {change:+.2f}% {status}")

    return train_positive_rates, pred_positive_rates

Comprehensive Model Audit

# Complete Model Bias Audit
def comprehensive_model_audit(model, X_train, y_train, X_test, y_test,
                              protected_attrs, feature_names):
    """
    Perform complete bias audit of trained model
    """
    print("\n" + "=" * 80)
    print("COMPREHENSIVE MODEL BIAS AUDIT")
    print("=" * 80)

    # 1. Disaggregated Performance
    print("\n1. DISAGGREGATED PERFORMANCE")
    print("-" * 80)
    for attr in protected_attrs:
        disaggregated_evaluation(model, X_test, y_test, X_test[attr])

    # 2. Confusion Matrix Analysis
    print("\n2. CONFUSION MATRIX ANALYSIS")
    print("-" * 80)
    for attr in protected_attrs:
        confusion_matrix_by_group(y_test, model.predict(X_test), X_test[attr])

    # 3. Calibration Check
    print("\n3. CALIBRATION ANALYSIS")
    print("-" * 80)
    for attr in protected_attrs:
        calibration_analysis(model, X_test, y_test, X_test[attr])

    # 4. Subgroup Analysis
    if len(protected_attrs) > 1:
        print("\n4. INTERSECTIONAL SUBGROUP ANALYSIS")
        print("-" * 80)
        subgroup_analysis(model, X_test, y_test, protected_attrs)

    # Final Recommendations
    print("\n" + "=" * 80)
    print("AUDIT SUMMARY AND RECOMMENDATIONS")
    print("=" * 80)
    print("1. Review all performance disparities greater than 10%")
    print("2. Investigate groups with high error rates")
    print("3. Check for bias amplification")
    print("4. Consider threshold adjustments for fairness")
    print("5. Implement mitigation techniques if necessary")
    print("\n弘益人間 - Ensure your model serves all users fairly")
    print("=" * 80)

    return True