{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# Compare DupleBalance with Decomposition-based McIL Methods\n\nIn this example, we compare the :class:`duplebalance.DupleBalanceClassifier` \nand other decomposition + binaryIL multi-class imbalanced learning methods.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "print(__doc__)\n\nRANDOM_STATE = 42"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Preparation\nFirst, we will import necessary packages and generate an example\nmulti-class imbalanced dataset.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from duplebalance import DupleBalanceClassifier\nfrom duplebalance import DecompositionBasedClassifier\nfrom duplebalance.base import sort_dict_by_key\n\nimport pandas as pd\nfrom collections import Counter\nimport matplotlib.pyplot as plt\n\nfrom sklearn.datasets import make_classification\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Make a 5-class imbalanced classification task\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "X, y = make_classification(n_classes=5, class_sep=1, # 3-class\n    weights=[0.05, 0.05, 0.15, 0.25, 0.5], n_informative=10, n_redundant=1, flip_y=0,\n    n_features=20, n_clusters_per_class=1, n_samples=2000, random_state=0)\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42)\n\norigin_distr = sort_dict_by_key(Counter(y_train))\ntest_distr = sort_dict_by_key(Counter(y_test))\nprint('Original training dataset shape %s' % origin_distr)\nprint('Original test dataset shape %s' % test_distr)\n\n# Initialize results list \nall_results = []\nall_results_columns = ['Method', 'Score', '#Base Estimators', '#Training Samples']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Train a DupleBalance Classifier\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# Train a DupleBalanceClassifier\n\nensemble_init_kwargs = {\n    'n_estimators': 5,\n    'random_state': RANDOM_STATE,\n}\n\nclf = DupleBalanceClassifier(**ensemble_init_kwargs).fit(\n    X_train, y_train,\n    perturb_alpha='auto',\n    sample_weight=None,\n    eval_datasets={'test': (X_test, y_test)},\n    train_verbose={\n        'granularity': 1,\n        'print_distribution': True,\n        'print_metrics': True,\n    },\n)\ny_pred_proba = clf.predict_proba(X_test)\nscore = roc_auc_score(y_test, y_pred_proba, **{'average': 'weighted', 'multi_class': 'ovo'})\nprint (\"DupleBalance {} | Balanced AUROC: {:.3f} | #Training Samples: {:d}\".format(\n    ensemble_init_kwargs['n_estimators'], score, sum(clf.estimators_n_training_samples_)\n    ))\nall_results.append(\n    ['DupleBalance', score, len(clf.estimators_), sum(clf.estimators_n_training_samples_)]\n)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Train Decomposition + Binary IL Classifiers\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# Train all decomposition + binary imbalanced learning McIL methods\n\nALL_DECOMP = ['ova', 'ovo', 'ecoc']\nALL_BINARY = ['clean', 'enn', 'oneside', 'tomeklink', 'smote', 'border', 'oups', 'ans', 'ccr', 'gazzah', 'smotersb', 'smotetomek']\n\nfor decomposition in ALL_DECOMP:\n    for binary_il in ALL_BINARY:\n        # print (f\"Training decomposition {decomposition} + {binary_il} ...\")\n        clf = DecompositionBasedClassifier(\n            binary_il=binary_il,\n            decomposition=decomposition,\n            random_state=RANDOM_STATE,\n        ).fit(X_train, y_train)\n        y_pred_proba = clf.predict_proba(X_test)\n        score = roc_auc_score(y_test, y_pred_proba, **{'average': 'weighted', 'multi_class': 'ovo'})\n        print (\"Decomp: {:<5s} | BinaryIL: {:<10s} | Balanced AUROC: {:.3f} | #Training Samples: {:d}\".format(\n            decomposition, binary_il, score, sum(clf.estimators_n_training_samples_)\n            ))\n        all_results.append(\n            [f'{decomposition}+{binary_il}', score, len(clf.estimators_), sum(clf.estimators_n_training_samples_)]\n        )\n    print ('\\n')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Results Visualization\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\nimport seaborn as sns\n# sns.set_context('talk')\n\ndef get_decomposition_mask(decomposition, all_results):\n    mask = []\n    for method in all_results['Method'].values:\n        if method == 'DupleBalance':\n            mask.append(True)\n        elif method[:3] == decomposition:\n            mask.append(True)\n        elif method[:4] == decomposition:\n            mask.append(True)\n        else: mask.append(False)\n    return mask\n\nall_results = pd.DataFrame(all_results, columns=all_results_columns)\n\n\nfigure, axes = plt.subplots(1, 3, figsize=(12,6))\n\nfor decomposition, ax in zip(['ova', 'ovo', 'ecoc'], axes.flatten()):\n    \n    results_vis = all_results[get_decomposition_mask(decomposition, all_results)]\n    sns.scatterplot(\n        data=results_vis, \n        x='#Training Samples', y='Score', hue='Method', style='Method',\n        s=300, ax=ax,\n    )\n\n    for position, spine in ax.spines.items():\n        spine.set_color('black')\n        spine.set_linewidth(2)\n\n    ax.grid(color = 'black', linestyle='-.', alpha=0.3)\n    ax.set_ylabel('AUROC (macro)')\n    ax.legend(columnspacing=0.2,\n              borderaxespad=0.2,\n              handletextpad=0.2,\n              labelspacing=0.2,\n              handlelength=None,)\n    ax.set_title(f\"DupleBalance versus {decomposition.upper()}\")\n\nplt.tight_layout()\nplt.show()"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}