{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# Compare DupleBalance with Ad-hoc McIL Methods\n\nIn this example, we compare the :class:`duplebalance.DupleBalanceClassifier` \nand other ad-hoc multi-class imbalanced learning methods.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "print(__doc__)\n\nRANDOM_STATE = 42"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Preparation\nFirst, we will import necessary packages and generate an example\nmulti-class imbalanced dataset.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from duplebalance import DupleBalanceClassifier\nfrom duplebalance import AdhocMultiClassifier\nfrom duplebalance.base import sort_dict_by_key\n\nimport pandas as pd\nfrom collections import Counter\nimport matplotlib.pyplot as plt\n\nfrom sklearn.datasets import make_classification\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Make a 5-class imbalanced classification task\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "X, y = make_classification(n_classes=5, class_sep=1, # 3-class\n    weights=[0.05, 0.05, 0.15, 0.25, 0.5], n_informative=10, n_redundant=1, flip_y=0,\n    n_features=20, n_clusters_per_class=1, n_samples=2000, random_state=0)\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42)\n\norigin_distr = sort_dict_by_key(Counter(y_train))\ntest_distr = sort_dict_by_key(Counter(y_test))\nprint('Original training dataset shape %s' % origin_distr)\nprint('Original test dataset shape %s' % test_distr)\n\n# Initialize results list \nall_results = []\nall_results_columns = ['Method', 'Score', '#Base Estimators', '#Training Samples']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Train a DupleBalance Classifier\nTrain a DupleBalanceClassifier\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "n_estimators_list = [1, 5, 10, 25, 50]\n\nensemble_init_kwargs = {\n    'random_state': RANDOM_STATE,\n}\n\nfor n_estimators in n_estimators_list:\n    clf = DupleBalanceClassifier(\n        n_estimators=n_estimators,\n        **ensemble_init_kwargs\n    ).fit(\n        X_train, y_train,\n        perturb_alpha=0.7,\n        sample_weight=None,\n        eval_datasets={'test': (X_test, y_test)},\n        train_verbose=False,\n    )\n    y_pred_proba = clf.predict_proba(X_test)\n    score = roc_auc_score(y_test, y_pred_proba, **{'average': 'weighted', 'multi_class': 'ovo'})\n    print (\"DupleBalance {:<2d} | Balanced AUROC: {:.3f} | #Training Samples: {:d}\".format(\n        n_estimators, score, sum(clf.estimators_n_training_samples_)\n        ))\n    all_results.append(\n        ['DupleBalance', score, len(clf.estimators_), sum(clf.estimators_n_training_samples_)]\n    )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Train Ad-hoc McIL Classifiers\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# Train all ad-hoc McIL methods\n\nALL_ADHOC_METHOD = ['mdoboost', 'mdobagging', 'soupboost', 'soupbagging', 'mrrbagging', 'adacost', 'asymboost']\n\nfor n_estimators in n_estimators_list:\n    for method in ALL_ADHOC_METHOD:\n        clf = AdhocMultiClassifier(\n            method=method,\n            n_estimators=n_estimators,\n            **ensemble_init_kwargs\n        ).fit(X_train, y_train)\n        y_pred_proba = clf.predict_proba(X_test)\n        score = roc_auc_score(y_test, y_pred_proba, **{'average': 'weighted', 'multi_class': 'ovo'})\n        print (\"Ad-hoc method: {:<15s} {:<2d} | Balanced AUROC: {:.3f} | #Training Samples: {:d}\".format(\n            method, n_estimators, score, sum(clf.estimators_n_training_samples_)\n            ))\n        all_results.append(\n            [method, score, len(clf.estimators_), sum(clf.estimators_n_training_samples_)]\n        )\n    print ('\\n')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Results Visualization\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_context('talk')\n\nresults_vis = pd.DataFrame(all_results, columns=all_results_columns)\n\nfig = plt.figure(figsize=(10,6))\nax = sns.lineplot(\n    data=results_vis, \n    x='#Training Samples', y='Score', hue='Method', style='Method',\n    markers=True, err_style='bars', linewidth=4, markersize=12, alpha=0.9\n)\n\nfor position, spine in ax.spines.items():\n    spine.set_color('black')\n    spine.set_linewidth(2)\n\nax.grid(color = 'black', linestyle='-.', alpha=0.3)\nax.set_ylabel('AUROC (macro)')\nax.legend(columnspacing=0.2,\n          borderaxespad=0.2,\n          handletextpad=0.2,\n          labelspacing=0.2,\n          handlelength=None,)\nax.set_title(f\"DupleBalance versus Ad-hoc Methods\")\nplt.show()"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}