Note
Click here to download the full example code
Compare DupleBalance with Decomposition-based McIL Methods¶
In this example, we compare the duplebalance.DupleBalanceClassifier
and other decomposition + binaryIL multi-class imbalanced learning methods.
print(__doc__)
RANDOM_STATE = 42
Preparation¶
First, we will import necessary packages and generate an example multi-class imbalanced dataset.
from duplebalance import DupleBalanceClassifier
from duplebalance import DecompositionBasedClassifier
from duplebalance.base import sort_dict_by_key
import pandas as pd
from collections import Counter
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
Make a 5-class imbalanced classification task
X, y = make_classification(n_classes=5, class_sep=1, # 3-class
weights=[0.05, 0.05, 0.15, 0.25, 0.5], n_informative=10, n_redundant=1, flip_y=0,
n_features=20, n_clusters_per_class=1, n_samples=2000, random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42)
origin_distr = sort_dict_by_key(Counter(y_train))
test_distr = sort_dict_by_key(Counter(y_test))
print('Original training dataset shape %s' % origin_distr)
print('Original test dataset shape %s' % test_distr)
# Initialize results list
all_results = []
all_results_columns = ['Method', 'Score', '#Base Estimators', '#Training Samples']
Out:
Original training dataset shape {0: 60, 1: 42, 2: 131, 3: 257, 4: 510}
Original test dataset shape {0: 40, 1: 58, 2: 169, 3: 243, 4: 490}
Train a DupleBalance Classifier¶
# Train a DupleBalanceClassifier
ensemble_init_kwargs = {
'n_estimators': 5,
'random_state': RANDOM_STATE,
}
clf = DupleBalanceClassifier(**ensemble_init_kwargs).fit(
X_train, y_train,
perturb_alpha='auto',
sample_weight=None,
eval_datasets={'test': (X_test, y_test)},
train_verbose={
'granularity': 1,
'print_distribution': True,
'print_metrics': True,
},
)
y_pred_proba = clf.predict_proba(X_test)
score = roc_auc_score(y_test, y_pred_proba, **{'average': 'weighted', 'multi_class': 'ovo'})
print ("DupleBalance {} | Balanced AUROC: {:.3f} | #Training Samples: {:d}".format(
ensemble_init_kwargs['n_estimators'], score, sum(clf.estimators_n_training_samples_)
))
all_results.append(
['DupleBalance', score, len(clf.estimators_), sum(clf.estimators_n_training_samples_)]
)
Out:
0%| | 0/21 [00:00<?, ?it/s]
'perturb_alpha' == 'auto', auto tuning: 0%| | 0/21 [00:00<?, ?it/s]
'perturb_alpha' == 'auto', auto tuning: 5%|## | 1/21 [00:00<00:03, 5.06it/s]
'perturb_alpha' == 'auto', auto tuning: 10%|#### | 2/21 [00:00<00:03, 4.87it/s]
'perturb_alpha' == 'auto', auto tuning: 14%|######1 | 3/21 [00:00<00:03, 4.79it/s]
'perturb_alpha' == 'auto', auto tuning: 19%|########1 | 4/21 [00:00<00:03, 4.74it/s]
'perturb_alpha' == 'auto', auto tuning: 24%|##########2 | 5/21 [00:01<00:03, 4.69it/s]
'perturb_alpha' == 'auto', auto tuning: 29%|############2 | 6/21 [00:01<00:03, 4.60it/s]
'perturb_alpha' == 'auto', auto tuning: 33%|##############3 | 7/21 [00:01<00:03, 4.57it/s]
'perturb_alpha' == 'auto', auto tuning: 38%|################3 | 8/21 [00:01<00:02, 4.54it/s]
'perturb_alpha' == 'auto', auto tuning: 43%|##################4 | 9/21 [00:01<00:02, 4.52it/s]
'perturb_alpha' == 'auto', auto tuning: 48%|#################### | 10/21 [00:02<00:02, 4.49it/s]
'perturb_alpha' == 'auto', auto tuning: 52%|###################### | 11/21 [00:02<00:02, 4.46it/s]
'perturb_alpha' == 'auto', auto tuning: 57%|######################## | 12/21 [00:02<00:02, 4.46it/s]
'perturb_alpha' == 'auto', auto tuning: 62%|########################## | 13/21 [00:02<00:01, 4.46it/s]
'perturb_alpha' == 'auto', auto tuning: 67%|############################ | 14/21 [00:03<00:01, 4.41it/s]
'perturb_alpha' == 'auto', auto tuning: 71%|############################## | 15/21 [00:03<00:01, 4.40it/s]
'perturb_alpha' == 'auto', auto tuning: 76%|################################ | 16/21 [00:03<00:01, 4.34it/s]
'perturb_alpha' == 'auto', auto tuning: 81%|################################## | 17/21 [00:03<00:00, 4.36it/s]
'perturb_alpha' == 'auto', auto tuning: 86%|#################################### | 18/21 [00:04<00:00, 4.37it/s]
'perturb_alpha' == 'auto', auto tuning: 90%|###################################### | 19/21 [00:04<00:00, 4.34it/s]
'perturb_alpha' == 'auto', auto tuning: 95%|######################################## | 20/21 [00:04<00:00, 4.38it/s]
'perturb_alpha' == 'auto', auto tuning: 100%|##########################################| 21/21 [00:04<00:00, 4.38it/s]
'perturb_alpha' == 'auto', auto tuning: 100%|##########################################| 21/21 [00:04<00:00, 4.46it/s]
The perturb_alpha will be set to 0.650 with 0.925 balanced AUROC (validation score)
┏━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ ┃ ┃ Data: train ┃ Data: test ┃
┃ #Estimators ┃ Class Distribution ┃ Metric ┃ Metric ┃
┃ ┃ ┃ balance-roc-auc ┃ balance-roc-auc ┃
┣━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━━━┫
┃ 1 ┃ {0: 60, 1: 42, 2: 131, 3: 257, 4: 510} ┃ 0.813 ┃ 0.767 ┃
┃ 2 ┃ {0: 96, 1: 83, 2: 150, 3: 244, 4: 434} ┃ 0.910 ┃ 0.846 ┃
┃ 3 ┃ {0: 132, 1: 123, 2: 167, 3: 230, 4: 356} ┃ 0.964 ┃ 0.905 ┃
┃ 4 ┃ {0: 167, 1: 163, 2: 185, 3: 217, 4: 280} ┃ 0.980 ┃ 0.926 ┃
┃ 5 ┃ {0: 204, 1: 204, 2: 204, 3: 204, 4: 204} ┃ 0.989 ┃ 0.952 ┃
┣━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━━━╋━━━━━━━━━━━━━━━━━┫
┃ final ┃ {0: 204, 1: 204, 2: 204, 3: 204, 4: 204} ┃ 0.989 ┃ 0.952 ┃
┗━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━━━┛
DupleBalance 5 | Balanced AUROC: 0.952 | #Training Samples: 5047
Train Decomposition + Binary IL Classifiers
# Train all decomposition + binary imbalanced learning McIL methods
ALL_DECOMP = ['ova', 'ovo', 'ecoc']
ALL_BINARY = ['clean', 'enn', 'oneside', 'tomeklink', 'smote', 'border', 'oups', 'ans', 'ccr', 'gazzah', 'smotersb', 'smotetomek']
for decomposition in ALL_DECOMP:
for binary_il in ALL_BINARY:
# print (f"Training decomposition {decomposition} + {binary_il} ...")
clf = DecompositionBasedClassifier(
binary_il=binary_il,
decomposition=decomposition,
random_state=RANDOM_STATE,
).fit(X_train, y_train)
y_pred_proba = clf.predict_proba(X_test)
score = roc_auc_score(y_test, y_pred_proba, **{'average': 'weighted', 'multi_class': 'ovo'})
print ("Decomp: {:<5s} | BinaryIL: {:<10s} | Balanced AUROC: {:.3f} | #Training Samples: {:d}".format(
decomposition, binary_il, score, sum(clf.estimators_n_training_samples_)
))
all_results.append(
[f'{decomposition}+{binary_il}', score, len(clf.estimators_), sum(clf.estimators_n_training_samples_)]
)
print ('\n')
Out:
Decomp: ova | BinaryIL: clean | Balanced AUROC: 0.779 | #Training Samples: 4640
Decomp: ova | BinaryIL: enn | Balanced AUROC: 0.753 | #Training Samples: 4085
Decomp: ova | BinaryIL: oneside | Balanced AUROC: 0.788 | #Training Samples: 4025
Decomp: ova | BinaryIL: tomeklink | Balanced AUROC: 0.789 | #Training Samples: 4880
Decomp: ova | BinaryIL: smote | Balanced AUROC: 0.793 | #Training Samples: 12750
Decomp: ova | BinaryIL: border | Balanced AUROC: 0.783 | #Training Samples: 12750
Decomp: ova | BinaryIL: oups | Balanced AUROC: 0.764 | #Training Samples: 12880
Decomp: ova | BinaryIL: ans | Balanced AUROC: 0.788 | #Training Samples: 10855
Decomp: ova | BinaryIL: ccr | Balanced AUROC: 0.793 | #Training Samples: 12960
Decomp: ova | BinaryIL: gazzah | Balanced AUROC: 0.809 | #Training Samples: 6015
Decomp: ova | BinaryIL: smotersb | Balanced AUROC: 0.768 | #Training Samples: 7560
Decomp: ova | BinaryIL: smotetomek | Balanced AUROC: 0.796 | #Training Samples: 12750
Decomp: ovo | BinaryIL: clean | Balanced AUROC: 0.836 | #Training Samples: 3712
Decomp: ovo | BinaryIL: enn | Balanced AUROC: 0.792 | #Training Samples: 3268
Decomp: ovo | BinaryIL: oneside | Balanced AUROC: 0.786 | #Training Samples: 3292
Decomp: ovo | BinaryIL: tomeklink | Balanced AUROC: 0.821 | #Training Samples: 3904
Decomp: ovo | BinaryIL: smote | Balanced AUROC: 0.842 | #Training Samples: 10200
Decomp: ovo | BinaryIL: border | Balanced AUROC: 0.800 | #Training Samples: 10200
Decomp: ovo | BinaryIL: oups | Balanced AUROC: 0.816 | #Training Samples: 10288
Decomp: ovo | BinaryIL: ans | Balanced AUROC: 0.812 | #Training Samples: 8684
Decomp: ovo | BinaryIL: ccr | Balanced AUROC: 0.838 | #Training Samples: 10368
Decomp: ovo | BinaryIL: gazzah | Balanced AUROC: 0.831 | #Training Samples: 4812
Decomp: ovo | BinaryIL: smotersb | Balanced AUROC: 0.804 | #Training Samples: 6228
Decomp: ovo | BinaryIL: smotetomek | Balanced AUROC: 0.826 | #Training Samples: 10200
Could not find cached matrix for dense code for 5 classes, generating matrix...
Decomp: ecoc | BinaryIL: clean | Balanced AUROC: 0.818 | #Training Samples: 22272
Could not find cached matrix for dense code for 5 classes, generating matrix...
Decomp: ecoc | BinaryIL: enn | Balanced AUROC: 0.795 | #Training Samples: 19608
Could not find cached matrix for dense code for 5 classes, generating matrix...
Decomp: ecoc | BinaryIL: oneside | Balanced AUROC: 0.843 | #Training Samples: 19368
Could not find cached matrix for dense code for 5 classes, generating matrix...
Decomp: ecoc | BinaryIL: tomeklink | Balanced AUROC: 0.829 | #Training Samples: 23424
Could not find cached matrix for dense code for 5 classes, generating matrix...
Decomp: ecoc | BinaryIL: smote | Balanced AUROC: 0.875 | #Training Samples: 61200
Could not find cached matrix for dense code for 5 classes, generating matrix...
Decomp: ecoc | BinaryIL: border | Balanced AUROC: 0.862 | #Training Samples: 61200
Could not find cached matrix for dense code for 5 classes, generating matrix...
Decomp: ecoc | BinaryIL: oups | Balanced AUROC: 0.857 | #Training Samples: 61752
Could not find cached matrix for dense code for 5 classes, generating matrix...
Decomp: ecoc | BinaryIL: ans | Balanced AUROC: 0.853 | #Training Samples: 52104
Could not find cached matrix for dense code for 5 classes, generating matrix...
Decomp: ecoc | BinaryIL: ccr | Balanced AUROC: 0.889 | #Training Samples: 62208
Could not find cached matrix for dense code for 5 classes, generating matrix...
Decomp: ecoc | BinaryIL: gazzah | Balanced AUROC: 0.871 | #Training Samples: 28872
Could not find cached matrix for dense code for 5 classes, generating matrix...
Decomp: ecoc | BinaryIL: smotersb | Balanced AUROC: 0.871 | #Training Samples: 37344
Could not find cached matrix for dense code for 5 classes, generating matrix...
Decomp: ecoc | BinaryIL: smotetomek | Balanced AUROC: 0.868 | #Training Samples: 61200
Results Visualization¶
import matplotlib.pyplot as plt
import seaborn as sns
# sns.set_context('talk')
def get_decomposition_mask(decomposition, all_results):
mask = []
for method in all_results['Method'].values:
if method == 'DupleBalance':
mask.append(True)
elif method[:3] == decomposition:
mask.append(True)
elif method[:4] == decomposition:
mask.append(True)
else: mask.append(False)
return mask
all_results = pd.DataFrame(all_results, columns=all_results_columns)
figure, axes = plt.subplots(1, 3, figsize=(12,6))
for decomposition, ax in zip(['ova', 'ovo', 'ecoc'], axes.flatten()):
results_vis = all_results[get_decomposition_mask(decomposition, all_results)]
sns.scatterplot(
data=results_vis,
x='#Training Samples', y='Score', hue='Method', style='Method',
s=300, ax=ax,
)
for position, spine in ax.spines.items():
spine.set_color('black')
spine.set_linewidth(2)
ax.grid(color = 'black', linestyle='-.', alpha=0.3)
ax.set_ylabel('AUROC (macro)')
ax.legend(columnspacing=0.2,
borderaxespad=0.2,
handletextpad=0.2,
labelspacing=0.2,
handlelength=None,)
ax.set_title(f"DupleBalance versus {decomposition.upper()}")
plt.tight_layout()
plt.show()
Total running time of the script: ( 1 minutes 29.263 seconds)
Estimated memory usage: 39 MB