{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example: Data engines\n",
    "-----------------------\n",
    "\n",
    "This example shows how ATOM interacts with other data engines than pandas, for example [polars](https://pola.rs/).\n",
    "\n",
    "Import the breast cancer dataset from [sklearn.datasets](https://scikit-learn.org/stable/datasets/index.html#wine-dataset). This is a small and easy to train dataset whose goal is to predict whether a patient has breast cancer or not."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import packages\n",
    "import polars as pl\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "from atom import ATOMClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 30)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>mean radius</th><th>mean texture</th><th>mean perimeter</th><th>mean area</th><th>mean smoothness</th><th>mean compactness</th><th>mean concavity</th><th>mean concave points</th><th>mean symmetry</th><th>mean fractal dimension</th><th>radius error</th><th>texture error</th><th>perimeter error</th><th>area error</th><th>smoothness error</th><th>compactness error</th><th>concavity error</th><th>concave points error</th><th>symmetry error</th><th>fractal dimension error</th><th>worst radius</th><th>worst texture</th><th>worst perimeter</th><th>worst area</th><th>worst smoothness</th><th>worst compactness</th><th>worst concavity</th><th>worst concave points</th><th>worst symmetry</th><th>worst fractal dimension</th></tr><tr><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>17.99</td><td>10.38</td><td>122.8</td><td>1001.0</td><td>0.1184</td><td>0.2776</td><td>0.3001</td><td>0.1471</td><td>0.2419</td><td>0.07871</td><td>1.095</td><td>0.9053</td><td>8.589</td><td>153.4</td><td>0.006399</td><td>0.04904</td><td>0.05373</td><td>0.01587</td><td>0.03003</td><td>0.006193</td><td>25.38</td><td>17.33</td><td>184.6</td><td>2019.0</td><td>0.1622</td><td>0.6656</td><td>0.7119</td><td>0.2654</td><td>0.4601</td><td>0.1189</td></tr><tr><td>20.57</td><td>17.77</td><td>132.9</td><td>1326.0</td><td>0.08474</td><td>0.07864</td><td>0.0869</td><td>0.07017</td><td>0.1812</td><td>0.05667</td><td>0.5435</td><td>0.7339</td><td>3.398</td><td>74.08</td><td>0.005225</td><td>0.01308</td><td>0.0186</td><td>0.0134</td><td>0.01389</td><td>0.003532</td><td>24.99</td><td>23.41</td><td>158.8</td><td>1956.0</td><td>0.1238</td><td>0.1866</td><td>0.2416</td><td>0.186</td><td>0.275</td><td>0.08902</td></tr><tr><td>19.69</td><td>21.25</td><td>130.0</td><td>1203.0</td><td>0.1096</td><td>0.1599</td><td>0.1974</td><td>0.1279</td><td>0.2069</td><td>0.05999</td><td>0.7456</td><td>0.7869</td><td>4.585</td><td>94.03</td><td>0.00615</td><td>0.04006</td><td>0.03832</td><td>0.02058</td><td>0.0225</td><td>0.004571</td><td>23.57</td><td>25.53</td><td>152.5</td><td>1709.0</td><td>0.1444</td><td>0.4245</td><td>0.4504</td><td>0.243</td><td>0.3613</td><td>0.08758</td></tr><tr><td>11.42</td><td>20.38</td><td>77.58</td><td>386.1</td><td>0.1425</td><td>0.2839</td><td>0.2414</td><td>0.1052</td><td>0.2597</td><td>0.09744</td><td>0.4956</td><td>1.156</td><td>3.445</td><td>27.23</td><td>0.00911</td><td>0.07458</td><td>0.05661</td><td>0.01867</td><td>0.05963</td><td>0.009208</td><td>14.91</td><td>26.5</td><td>98.87</td><td>567.7</td><td>0.2098</td><td>0.8663</td><td>0.6869</td><td>0.2575</td><td>0.6638</td><td>0.173</td></tr><tr><td>20.29</td><td>14.34</td><td>135.1</td><td>1297.0</td><td>0.1003</td><td>0.1328</td><td>0.198</td><td>0.1043</td><td>0.1809</td><td>0.05883</td><td>0.7572</td><td>0.7813</td><td>5.438</td><td>94.44</td><td>0.01149</td><td>0.02461</td><td>0.05688</td><td>0.01885</td><td>0.01756</td><td>0.005115</td><td>22.54</td><td>16.67</td><td>152.2</td><td>1575.0</td><td>0.1374</td><td>0.205</td><td>0.4</td><td>0.1625</td><td>0.2364</td><td>0.07678</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 30)\n",
       "┌─────────────┬──────────────┬────────────────┬───────────┬───┬─────────────────┬──────────────────────┬────────────────┬─────────────────────────┐\n",
       "│ mean radius ┆ mean texture ┆ mean perimeter ┆ mean area ┆ … ┆ worst concavity ┆ worst concave points ┆ worst symmetry ┆ worst fractal dimension │\n",
       "│ ---         ┆ ---          ┆ ---            ┆ ---       ┆   ┆ ---             ┆ ---                  ┆ ---            ┆ ---                     │\n",
       "│ f64         ┆ f64          ┆ f64            ┆ f64       ┆   ┆ f64             ┆ f64                  ┆ f64            ┆ f64                     │\n",
       "╞═════════════╪══════════════╪════════════════╪═══════════╪═══╪═════════════════╪══════════════════════╪════════════════╪═════════════════════════╡\n",
       "│ 17.99       ┆ 10.38        ┆ 122.8          ┆ 1001.0    ┆ … ┆ 0.7119          ┆ 0.2654               ┆ 0.4601         ┆ 0.1189                  │\n",
       "│ 20.57       ┆ 17.77        ┆ 132.9          ┆ 1326.0    ┆ … ┆ 0.2416          ┆ 0.186                ┆ 0.275          ┆ 0.08902                 │\n",
       "│ 19.69       ┆ 21.25        ┆ 130.0          ┆ 1203.0    ┆ … ┆ 0.4504          ┆ 0.243                ┆ 0.3613         ┆ 0.08758                 │\n",
       "│ 11.42       ┆ 20.38        ┆ 77.58          ┆ 386.1     ┆ … ┆ 0.6869          ┆ 0.2575               ┆ 0.6638         ┆ 0.173                   │\n",
       "│ 20.29       ┆ 14.34        ┆ 135.1          ┆ 1297.0    ┆ … ┆ 0.4             ┆ 0.1625               ┆ 0.2364         ┆ 0.07678                 │\n",
       "└─────────────┴──────────────┴────────────────┴───────────┴───┴─────────────────┴──────────────────────┴────────────────┴─────────────────────────┘"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the data and convert to polars for demonstration purposes\n",
    "X, y = load_breast_cancer(return_X_y=True, as_frame=True)\n",
    "\n",
    "X = pl.from_pandas(X)\n",
    "y = pl.from_pandas(y)\n",
    "\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<< ================== ATOM ================== >>\n",
      "\n",
      "Configuration ==================== >>\n",
      "Algorithm task: Binary classification.\n",
      "Data engine: polars\n",
      "\n",
      "Dataset stats ==================== >>\n",
      "Shape: (569, 31)\n",
      "Train set size: 456\n",
      "Test set size: 113\n",
      "-------------------------------------\n",
      "Memory: 138.97 kB\n",
      "Scaled: False\n",
      "Outlier values: 167 (1.2%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Specify the data engine in the constructor\n",
    "# Note that atom accepts any dataframe-like object to create the dataset\n",
    "atom = ATOMClassifier(X, y, engine=\"polars\", verbose=2, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 30)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>mean radius</th><th>mean texture</th><th>mean perimeter</th><th>mean area</th><th>mean smoothness</th><th>mean compactness</th><th>mean concavity</th><th>mean concave points</th><th>mean symmetry</th><th>mean fractal dimension</th><th>radius error</th><th>texture error</th><th>perimeter error</th><th>area error</th><th>smoothness error</th><th>compactness error</th><th>concavity error</th><th>concave points error</th><th>symmetry error</th><th>fractal dimension error</th><th>worst radius</th><th>worst texture</th><th>worst perimeter</th><th>worst area</th><th>worst smoothness</th><th>worst compactness</th><th>worst concavity</th><th>worst concave points</th><th>worst symmetry</th><th>worst fractal dimension</th></tr><tr><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>13.48</td><td>20.82</td><td>88.4</td><td>559.2</td><td>0.1016</td><td>0.1255</td><td>0.1063</td><td>0.05439</td><td>0.172</td><td>0.06419</td><td>0.213</td><td>0.5914</td><td>1.545</td><td>18.52</td><td>0.005367</td><td>0.02239</td><td>0.03049</td><td>0.01262</td><td>0.01377</td><td>0.003187</td><td>15.53</td><td>26.02</td><td>107.3</td><td>740.4</td><td>0.161</td><td>0.4225</td><td>0.503</td><td>0.2258</td><td>0.2807</td><td>0.1071</td></tr><tr><td>18.31</td><td>20.58</td><td>120.8</td><td>1052.0</td><td>0.1068</td><td>0.1248</td><td>0.1569</td><td>0.09451</td><td>0.186</td><td>0.05941</td><td>0.5449</td><td>0.9225</td><td>3.218</td><td>67.36</td><td>0.006176</td><td>0.01877</td><td>0.02913</td><td>0.01046</td><td>0.01559</td><td>0.002725</td><td>21.86</td><td>26.2</td><td>142.2</td><td>1493.0</td><td>0.1492</td><td>0.2536</td><td>0.3759</td><td>0.151</td><td>0.3074</td><td>0.07863</td></tr><tr><td>17.93</td><td>24.48</td><td>115.2</td><td>998.9</td><td>0.08855</td><td>0.07027</td><td>0.05699</td><td>0.04744</td><td>0.1538</td><td>0.0551</td><td>0.4212</td><td>1.433</td><td>2.765</td><td>45.81</td><td>0.005444</td><td>0.01169</td><td>0.01622</td><td>0.008522</td><td>0.01419</td><td>0.002751</td><td>20.92</td><td>34.69</td><td>135.1</td><td>1320.0</td><td>0.1315</td><td>0.1806</td><td>0.208</td><td>0.1136</td><td>0.2504</td><td>0.07948</td></tr><tr><td>15.13</td><td>29.81</td><td>96.71</td><td>719.5</td><td>0.0832</td><td>0.04605</td><td>0.04686</td><td>0.02739</td><td>0.1852</td><td>0.05294</td><td>0.4681</td><td>1.627</td><td>3.043</td><td>45.38</td><td>0.006831</td><td>0.01427</td><td>0.02489</td><td>0.009087</td><td>0.03151</td><td>0.00175</td><td>17.26</td><td>36.91</td><td>110.1</td><td>931.4</td><td>0.1148</td><td>0.09866</td><td>0.1547</td><td>0.06575</td><td>0.3233</td><td>0.06165</td></tr><tr><td>8.95</td><td>15.76</td><td>58.74</td><td>245.2</td><td>0.09462</td><td>0.1243</td><td>0.09263</td><td>0.02308</td><td>0.1305</td><td>0.07163</td><td>0.3132</td><td>0.9789</td><td>3.28</td><td>16.94</td><td>0.01835</td><td>0.0676</td><td>0.09263</td><td>0.02308</td><td>0.02384</td><td>0.005601</td><td>9.414</td><td>17.07</td><td>63.34</td><td>270.0</td><td>0.1179</td><td>0.1879</td><td>0.1544</td><td>0.03846</td><td>0.1652</td><td>0.07722</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 30)\n",
       "┌─────────────┬──────────────┬────────────────┬───────────┬───┬─────────────────┬──────────────────────┬────────────────┬─────────────────────────┐\n",
       "│ mean radius ┆ mean texture ┆ mean perimeter ┆ mean area ┆ … ┆ worst concavity ┆ worst concave points ┆ worst symmetry ┆ worst fractal dimension │\n",
       "│ ---         ┆ ---          ┆ ---            ┆ ---       ┆   ┆ ---             ┆ ---                  ┆ ---            ┆ ---                     │\n",
       "│ f64         ┆ f64          ┆ f64            ┆ f64       ┆   ┆ f64             ┆ f64                  ┆ f64            ┆ f64                     │\n",
       "╞═════════════╪══════════════╪════════════════╪═══════════╪═══╪═════════════════╪══════════════════════╪════════════════╪═════════════════════════╡\n",
       "│ 13.48       ┆ 20.82        ┆ 88.4           ┆ 559.2     ┆ … ┆ 0.503           ┆ 0.2258               ┆ 0.2807         ┆ 0.1071                  │\n",
       "│ 18.31       ┆ 20.58        ┆ 120.8          ┆ 1052.0    ┆ … ┆ 0.3759          ┆ 0.151                ┆ 0.3074         ┆ 0.07863                 │\n",
       "│ 17.93       ┆ 24.48        ┆ 115.2          ┆ 998.9     ┆ … ┆ 0.208           ┆ 0.1136               ┆ 0.2504         ┆ 0.07948                 │\n",
       "│ 15.13       ┆ 29.81        ┆ 96.71          ┆ 719.5     ┆ … ┆ 0.1547          ┆ 0.06575              ┆ 0.3233         ┆ 0.06165                 │\n",
       "│ 8.95        ┆ 15.76        ┆ 58.74          ┆ 245.2     ┆ … ┆ 0.1544          ┆ 0.03846              ┆ 0.1652         ┆ 0.07722                 │\n",
       "└─────────────┴──────────────┴────────────────┴───────────┴───┴─────────────────┴──────────────────────┴────────────────┴─────────────────────────┘"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The data attributes return now polars types\n",
    "atom.X.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5,)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>target</th></tr><tr><td>i32</td></tr></thead><tbody><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>1</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5,)\n",
       "Series: 'target' [i32]\n",
       "[\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t1\n",
       "]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "atom.y.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training ========================= >>\n",
      "Models: LR\n",
      "Metric: f1\n",
      "\n",
      "\n",
      "Results for LogisticRegression:\n",
      "Fit ---------------------------------------------\n",
      "Train evaluation --> f1: 0.9913\n",
      "Test evaluation --> f1: 0.9861\n",
      "Time elapsed: 0.129s\n",
      "-------------------------------------------------\n",
      "Time: 0.129s\n",
      "\n",
      "\n",
      "Final results ==================== >>\n",
      "Total time: 0.132s\n",
      "-------------------------------------\n",
      "LogisticRegression --> f1: 0.9861\n"
     ]
    }
   ],
   "source": [
    "atom.run(\"LR\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (569,)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>target</th></tr><tr><td>i64</td></tr></thead><tbody><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>&hellip;</td></tr><tr><td>1</td></tr><tr><td>1</td></tr><tr><td>1</td></tr><tr><td>1</td></tr><tr><td>1</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>0</td></tr><tr><td>1</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (569,)\n",
       "Series: 'target' [i64]\n",
       "[\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t…\n",
       "\t1\n",
       "\t1\n",
       "\t1\n",
       "\t1\n",
       "\t1\n",
       "\t1\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t0\n",
       "\t1\n",
       "]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The prediction methods also return types of the requested data engine\n",
    "atom.lr.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "3    0\n",
       "4    0\n",
       "Name: target, dtype: int64[pyarrow]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "atom.lr.engine = \"pandas-pyarrow\"\n",
    "atom.lr.predict(X.head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dask Series Structure:\n",
       "npartitions=1\n",
       "0    int64\n",
       "4      ...\n",
       "Name: target, dtype: int64\n",
       "Dask Name: from_pandas, 1 graph layer"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "atom.lr.engine = \"dask\"\n",
    "atom.lr.predict(X.head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pyarrow.lib.Int64Array object at 0x0000016E06BCD1E0>\n",
       "[\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0\n",
       "]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "atom.lr.engine = \"pyarrow\"\n",
    "atom.lr.predict(X.head(5))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}