{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "58265941",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-26T19:40:48.404491Z",
     "iopub.status.busy": "2024-10-26T19:40:48.404387Z",
     "iopub.status.idle": "2024-10-26T19:40:48.925201Z",
     "shell.execute_reply": "2024-10-26T19:40:48.924906Z"
    }
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "from rpy2.ipython import html\n",
    "html.html_rdataframe=partial(html.html_rdataframe, table_class=\"docutils\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19cd09e8",
   "metadata": {},
   "source": [
    "# dplyr in Python\n",
    "\n",
    "We need 2 things for this:\n",
    "\n",
    "1- A data frame (using one of R's demo datasets)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "08ff77eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-26T19:40:48.926542Z",
     "iopub.status.busy": "2024-10-26T19:40:48.926407Z",
     "iopub.status.idle": "2024-10-26T19:40:48.946051Z",
     "shell.execute_reply": "2024-10-26T19:40:48.945805Z"
    }
   },
   "outputs": [],
   "source": [
    "from rpy2.robjects.packages import importr, data\n",
    "datasets = importr('datasets')\n",
    "mtcars_env = data(datasets).fetch('mtcars')\n",
    "mtcars = mtcars_env['mtcars']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6e05e9d",
   "metadata": {},
   "source": [
    "In addition to that, and because this tutorial is in a notebook,\n",
    "we initialize HTML rendering for R objects (pretty display of\n",
    "R data frames)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2a0bb180",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-26T19:40:48.947022Z",
     "iopub.status.busy": "2024-10-26T19:40:48.946924Z",
     "iopub.status.idle": "2024-10-26T19:40:48.949327Z",
     "shell.execute_reply": "2024-10-26T19:40:48.949092Z"
    }
   },
   "outputs": [],
   "source": [
    "import rpy2.ipython.html\n",
    "rpy2.ipython.html.init_printing()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80b70b58",
   "metadata": {},
   "source": [
    "2- dplyr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b25ba68e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-26T19:40:48.950204Z",
     "iopub.status.busy": "2024-10-26T19:40:48.950013Z",
     "iopub.status.idle": "2024-10-26T19:40:49.797424Z",
     "shell.execute_reply": "2024-10-26T19:40:49.797089Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/laurent/Desktop/software/python/py312_env/lib/python3.12/site-packages/rpy2/robjects/lib/dplyr.py:27: UserWarning: This was designed against dplyr versions starting with 1.0 but you have 1.1.4\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from rpy2.robjects.lib.dplyr import DataFrame\n",
    "from rpy2.robjects import rl"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca6ecce5",
   "metadata": {},
   "source": [
    "With this we have the choice of chaining (D3-style)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "30fe97fb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-26T19:40:49.799002Z",
     "iopub.status.busy": "2024-10-26T19:40:49.798864Z",
     "iopub.status.idle": "2024-10-26T19:40:49.809906Z",
     "shell.execute_reply": "2024-10-26T19:40:49.809675Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<emph>DataFrame</emph> with 2 rows and\n",
       "  2 columns:\n",
       "<table class=\"docutils\">\n",
       "  <thead>\n",
       "    <tr class=\"rpy2_names\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>gear</th>\n",
       "      <th>mean_ptw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td class=\"rpy2_rowname\">0</td>\n",
       "        <td class=\"rpy2_names\">1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1237.1266499803169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td class=\"rpy2_rowname\">1</td>\n",
       "        <td class=\"rpy2_names\">2</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2574.0331639315027</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<rpy2.robjects.lib.dplyr.DataFrame object at 0x72130ab34750> [19]\n",
       "R classes: ('tbl_df', 'tbl', 'data.frame')\n",
       "[FloatSexpVector, FloatSexpVector]\n",
       "  gear: <class 'rpy2.rinterface.FloatSexpVector'>\n",
       "  <rpy2.rinterface.FloatSexpVector object at 0x7212f0fae950> [14]\n",
       "  mean_ptw: <class 'rpy2.rinterface.FloatSexpVector'>\n",
       "  <rpy2.rinterface.FloatSexpVector object at 0x7212f0fae910> [14]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataf = (\n",
    "    DataFrame(mtcars)\n",
    "    .filter(rl('gear>3'))\n",
    "    .mutate(powertoweight=rl('hp*36/wt'))\n",
    "    .group_by(rl('gear'))\n",
    "    .summarize(mean_ptw=rl('mean(powertoweight)'))\n",
    ")\n",
    "\n",
    "dataf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bec626d",
   "metadata": {},
   "source": [
    "or with pipes (magrittr style).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1546a6af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-26T19:40:49.811345Z",
     "iopub.status.busy": "2024-10-26T19:40:49.811249Z",
     "iopub.status.idle": "2024-10-26T19:40:49.813391Z",
     "shell.execute_reply": "2024-10-26T19:40:49.813157Z"
    }
   },
   "outputs": [],
   "source": [
    "# currently no longer working\n",
    "from rpy2.robjects.lib.dplyr import (filter,\n",
    "                                     mutate,\n",
    "                                     group_by,\n",
    "                                     summarize)\n",
    "\n",
    "if False:\n",
    "    dataf = (DataFrame(mtcars) >>\n",
    "             filter(rl('gear>3')) >>\n",
    "             mutate(powertoweight=rl('hp*36/wt')) >>\n",
    "             group_by(rl('gear')) >>\n",
    "             summarize(mean_ptw=rl('mean(powertoweight)')))\n",
    "\n",
    "    dataf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e7aef92",
   "metadata": {},
   "source": [
    "The function `rl` creates unevaluated R language objects, which\n",
    "are then consummed by the `dplyr` function, just like it would be\n",
    "happening when using `dplyr` in R itself. This means that\n",
    "when writing `mean(powertoweight)` the R function `mean()` is used.\n",
    "\n",
    "Using a Python function is not too difficult though. We can just\n",
    "call Python back from R. To achieve this we simply\n",
    "use the decorator `rternalize`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f238a456",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-26T19:40:49.814909Z",
     "iopub.status.busy": "2024-10-26T19:40:49.814807Z",
     "iopub.status.idle": "2024-10-26T19:40:49.824340Z",
     "shell.execute_reply": "2024-10-26T19:40:49.824110Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<emph>DataFrame</emph> with 2 rows and\n",
       "  3 columns:\n",
       "<table class=\"docutils\">\n",
       "  <thead>\n",
       "    <tr class=\"rpy2_names\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>gear</th>\n",
       "      <th>mean_ptw</th>\n",
       "      <th>mean_np_ptw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td class=\"rpy2_rowname\">0</td>\n",
       "        <td class=\"rpy2_names\">1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1237.1266499803169</td>\n",
       "      <td>1237.1266499803169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td class=\"rpy2_rowname\">1</td>\n",
       "        <td class=\"rpy2_names\">2</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2574.0331639315027</td>\n",
       "      <td>2574.0331639315027</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<rpy2.robjects.lib.dplyr.DataFrame object at 0x721308d70550> [19]\n",
       "R classes: ('tbl_df', 'tbl', 'data.frame')\n",
       "[FloatSexpVector, FloatSexpVector, FloatSexpVector]\n",
       "  gear: <class 'rpy2.rinterface.FloatSexpVector'>\n",
       "  <rpy2.rinterface.FloatSexpVector object at 0x7212f0d48a10> [14]\n",
       "  mean_ptw: <class 'rpy2.rinterface.FloatSexpVector'>\n",
       "  <rpy2.rinterface.FloatSexpVector object at 0x7212f1573d90> [14]\n",
       "  mean_np_ptw: <class 'rpy2.rinterface.FloatSexpVector'>\n",
       "  <rpy2.rinterface.FloatSexpVector object at 0x7212f0d3be10> [14]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define a python function, and make\n",
    "# it a function R can use through `rternalize`\n",
    "from rpy2.rinterface import rternalize\n",
    "@rternalize(signature=False)\n",
    "def mean_np(x):\n",
    "    import statistics\n",
    "    return statistics.mean(x)\n",
    "\n",
    "# Bind that function to a symbol in R's\n",
    "# global environment\n",
    "from rpy2.robjects import globalenv\n",
    "globalenv['mean_np'] = mean_np\n",
    "\n",
    "# Write a dplyr chain of operations,\n",
    "# using our Python function `mean_np`\n",
    "dataf = (\n",
    "    DataFrame(mtcars)\n",
    "    .filter(rl('gear>3'))\n",
    "    .mutate(powertoweight=rl('hp*36/wt'))\n",
    "    .group_by(rl('gear'))\n",
    "    .summarize(mean_ptw=rl('mean(powertoweight)'),\n",
    "               mean_np_ptw=rl('mean_np(powertoweight)'))\n",
    ")\n",
    "\n",
    "dataf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7980a003",
   "metadata": {},
   "source": [
    "It is also possible to carry this out without having to\n",
    "place the custom function in R's global environment, although\n",
    "this is not straightforward."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7478e897",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-26T19:40:49.825861Z",
     "iopub.status.busy": "2024-10-26T19:40:49.825766Z",
     "iopub.status.idle": "2024-10-26T19:40:49.833790Z",
     "shell.execute_reply": "2024-10-26T19:40:49.833557Z"
    },
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<emph>DataFrame</emph> with 2 rows and\n",
       "  3 columns:\n",
       "<table class=\"docutils\">\n",
       "  <thead>\n",
       "    <tr class=\"rpy2_names\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>gear</th>\n",
       "      <th>mean_ptw</th>\n",
       "      <th>mean_np_ptw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td class=\"rpy2_rowname\">0</td>\n",
       "        <td class=\"rpy2_names\">1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1237.1266499803169</td>\n",
       "      <td>1237.1266499803169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td class=\"rpy2_rowname\">1</td>\n",
       "        <td class=\"rpy2_names\">2</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2574.0331639315027</td>\n",
       "      <td>2574.0331639315027</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<rpy2.robjects.lib.dplyr.DataFrame object at 0x7212f05bd010> [19]\n",
       "R classes: ('tbl_df', 'tbl', 'data.frame')\n",
       "[FloatSexpVector, FloatSexpVector, FloatSexpVector]\n",
       "  gear: <class 'rpy2.rinterface.FloatSexpVector'>\n",
       "  <rpy2.rinterface.FloatSexpVector object at 0x7212f05bccd0> [14]\n",
       "  mean_ptw: <class 'rpy2.rinterface.FloatSexpVector'>\n",
       "  <rpy2.rinterface.FloatSexpVector object at 0x7212f05bca50> [14]\n",
       "  mean_np_ptw: <class 'rpy2.rinterface.FloatSexpVector'>\n",
       "  <rpy2.rinterface.FloatSexpVector object at 0x7212f05bcb90> [14]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# First we delete our Python callback in globalenv to\n",
    "# ensure that we are picking up our callback in our\n",
    "# specific environment rather than this one.\n",
    "del(globalenv['mean_np'])\n",
    "\n",
    "from rpy2.robjects import Environment\n",
    "my_env = Environment()\n",
    "my_env['mean_np'] = mean_np\n",
    "\n",
    "\n",
    "# Create an rlang \"quosure\" object within\n",
    "# a given environment. We use the R package\n",
    "# rlang used by dplyr.\n",
    "from rpy2.robjects.lib.dplyr import rlang\n",
    "\n",
    "myquo = rlang.quo.rcall(\n",
    "    [(None, rl('mean_np(rlang::enexpr(powertoweight))'))],\n",
    "    environment=my_env\n",
    ")\n",
    "\n",
    "dataf = (\n",
    "    DataFrame(mtcars)\n",
    "    .filter(rl('gear>3'))\n",
    "    .mutate(powertoweight=rl('hp*36/wt'))\n",
    "    .group_by(rl('gear'))\n",
    "    .summarize(\n",
    "        mean_ptw=rl('mean(powertoweight)'),\n",
    "        mean_np_ptw=myquo)\n",
    ")\n",
    "\n",
    "dataf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b16c3e08",
   "metadata": {},
   "source": [
    "**note**: rpy2's interface to dplyr is implementing a fix to the (non-?)issue 1323\n",
    "(https://github.com/hadley/dplyr/issues/1323)\n",
    "\n",
    "The seamless translation of transformations to SQL whenever the\n",
    "data are in a table can be used directly. Since we are lifting\n",
    "the original implementation of `dplyr`, it *just works*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "67d5cffa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-26T19:40:49.835128Z",
     "iopub.status.busy": "2024-10-26T19:40:49.835033Z",
     "iopub.status.idle": "2024-10-26T19:40:50.221879Z",
     "shell.execute_reply": "2024-10-26T19:40:50.221542Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Source:   SQL [2 x 2]\n",
      "# Database: sqlite 3.46.0 [/tmp/tmp_igkv381]\n",
      "   gear mean_ptw\n",
      "  <dbl>    <dbl>\n",
      "1     4    1237.\n",
      "2     5    2574.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from rpy2.robjects.lib.dplyr import dplyr\n",
    "# in-memory SQLite database broken in dplyr's src_sqlite\n",
    "# db = dplyr.src_sqlite(\":memory:\")\n",
    "import tempfile\n",
    "with tempfile.NamedTemporaryFile() as db_fh:\n",
    "    db = dplyr.src_sqlite(db_fh.name)\n",
    "    # copy the table to that database\n",
    "    dataf_db = DataFrame(mtcars).copy_to(db, name=\"mtcars\")\n",
    "    res = (dataf_db\n",
    "           .filter(rl('gear>3'))\n",
    "           .mutate(powertoweight=rl('hp*36/wt'))\n",
    "           .group_by(rl('gear'))\n",
    "           .summarize(mean_ptw=rl('mean(powertoweight)')))\n",
    "    print(res)\n",
    "# "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f6eaa58",
   "metadata": {},
   "source": [
    "Since we are manipulating R objects, anything available to R is also available\n",
    "to us. If we want to see the SQL code generated that's:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ec9175e0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-26T19:40:50.223483Z",
     "iopub.status.busy": "2024-10-26T19:40:50.223375Z",
     "iopub.status.idle": "2024-10-26T19:40:50.265989Z",
     "shell.execute_reply": "2024-10-26T19:40:50.265673Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<SQL>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SELECT `gear`, AVG(`powertoweight`) AS `mean_ptw`\n",
      "FROM (\n",
      "  SELECT `mtcars`.*, (`hp` * 36.0) / `wt` AS `powertoweight`\n",
      "  FROM `mtcars`\n",
      "  WHERE (`gear` > 3.0)\n",
      ") AS `q01`\n",
      "GROUP BY `gear`\n"
     ]
    }
   ],
   "source": [
    "silent = dplyr.show_query(res)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfd7694e",
   "metadata": {},
   "source": [
    "The conversion rules in rpy2 make the above easily applicable to pandas data frames,\n",
    "completing the \"lexical loan\" of the dplyr vocabulary from R.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0c8c87ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-26T19:40:50.267560Z",
     "iopub.status.busy": "2024-10-26T19:40:50.267458Z",
     "iopub.status.idle": "2024-10-26T19:40:50.271375Z",
     "shell.execute_reply": "2024-10-26T19:40:50.271128Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n"
     ]
    }
   ],
   "source": [
    "from rpy2.robjects import pandas2ri\n",
    "from rpy2.robjects import default_converter\n",
    "\n",
    "# Using a conversion context in which the pandas conversion is\n",
    "# added to the default conversion rules, the rpy2 object\n",
    "# `mtcars` (an R data frame) is converted to a pandas data frame.\n",
    "with (default_converter + pandas2ri.converter).context() as cv:\n",
    "    pd_mtcars = mtcars_env['mtcars']\n",
    "print(type(pd_mtcars))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9317f99c",
   "metadata": {},
   "source": [
    "Using a local conversion context lets us also go from the pandas data frame\n",
    "to our dplyr-augmented R data frame and use the dplyr transformations\n",
    "on it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9bae8658",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-26T19:40:50.272952Z",
     "iopub.status.busy": "2024-10-26T19:40:50.272826Z",
     "iopub.status.idle": "2024-10-26T19:40:50.292528Z",
     "shell.execute_reply": "2024-10-26T19:40:50.292291Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<emph>DataFrame</emph> with 1 rows and\n",
       "  1 columns:\n",
       "<table class=\"docutils\">\n",
       "  <thead>\n",
       "    <tr class=\"rpy2_names\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean_ptw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td class=\"rpy2_rowname\">0</td>\n",
       "        <td class=\"rpy2_names\">1</td>\n",
       "      <td>1632.0477884748632</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<rpy2.robjects.lib.dplyr.DataFrame object at 0x7212f0016910> [19]\n",
       "R classes: ('data.frame',)\n",
       "[FloatSexpVector]\n",
       "  mean_ptw: <class 'rpy2.rinterface.FloatSexpVector'>\n",
       "  <rpy2.rinterface.FloatSexpVector object at 0x7212f05bf810> [14]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with (default_converter + pandas2ri.converter).context() as cv:\n",
    "    dataf = (DataFrame(pd_mtcars)\n",
    "             .filter(rl('gear>=3'))\n",
    "             .mutate(powertoweight=rl('hp*36/wt'))\n",
    "             .group_by(rl('gear'))\n",
    "             .summarize(mean_ptw=rl('mean(powertoweight)')))\n",
    "\n",
    "dataf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43a4a1de",
   "metadata": {},
   "source": [
    "**Reuse. Get things done. Don't reimplement.**"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "-all"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
