remove airway use case

meetings/2021/session_12_functional_programming_with_toolz/Toolz-use-case_1_airway.ipynb

-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "7ca3358f",
-   "metadata": {},
-   "source": [
-    "## Functional programming and `toolz` use cases\n",
-    "\n",
-    "### Streaming read count data and filtering\n",
-    "\n",
-    "This example is a adapted from Juan Nunez-Iglesias' [Linux conference Australia talk in 2016](https://www.youtube.com/watch?v=INAvD4Kmdbc). Refer to [his github repo](https://github.com/jni/functional-programming-in-python) for additional examples.\n",
-    "\n",
-    "#### Airway dataset\n",
-    "\n",
-    "Himes BE, Jiang X, Wagner P, Hu R, Wang Q, Klanderman B, Whitaker RM, Duan Q, Lasky-Su J, Nikolos C, Jester W, Johnson M, Panettieri R Jr, Tantisira KG, Weiss ST, Lu Q. “RNA-Seq Transcriptome Profiling Identifies CRISPLD2 as a Glucocorticoid Responsive Gene that Modulates Cytokine Function in Airway Smooth Muscle Cells.” [PLoS One. 2014 Jun 13;9(6):e99625](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0099625)\n",
-    "\n",
-    "Airway smooth muscle cells were treated with dexamethasone, a synthetic steroid with anti-inflammatory effects and expression levels were measured using RNA-seq."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "cb5b2da3",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "17d48aeb",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "airway = pd.read_csv('data/airway.csv',sep=\"\\t\", header=None, index_col = 0)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "aa45fc7b",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>1</th>\n",
-       "      <th>2</th>\n",
-       "      <th>3</th>\n",
-       "      <th>4</th>\n",
-       "      <th>5</th>\n",
-       "      <th>6</th>\n",
-       "      <th>7</th>\n",
-       "      <th>8</th>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>ENSG00000000003</th>\n",
-       "      <td>679</td>\n",
-       "      <td>448</td>\n",
-       "      <td>873</td>\n",
-       "      <td>408</td>\n",
-       "      <td>1138</td>\n",
-       "      <td>1047</td>\n",
-       "      <td>770</td>\n",
-       "      <td>572</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>ENSG00000000005</th>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>ENSG00000000419</th>\n",
-       "      <td>467</td>\n",
-       "      <td>515</td>\n",
-       "      <td>621</td>\n",
-       "      <td>365</td>\n",
-       "      <td>587</td>\n",
-       "      <td>799</td>\n",
-       "      <td>417</td>\n",
-       "      <td>508</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>ENSG00000000457</th>\n",
-       "      <td>260</td>\n",
-       "      <td>211</td>\n",
-       "      <td>263</td>\n",
-       "      <td>164</td>\n",
-       "      <td>245</td>\n",
-       "      <td>331</td>\n",
-       "      <td>233</td>\n",
-       "      <td>229</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>ENSG00000000460</th>\n",
-       "      <td>60</td>\n",
-       "      <td>55</td>\n",
-       "      <td>40</td>\n",
-       "      <td>35</td>\n",
-       "      <td>78</td>\n",
-       "      <td>63</td>\n",
-       "      <td>76</td>\n",
-       "      <td>60</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>LRG_94</th>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>LRG_96</th>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>LRG_97</th>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>LRG_98</th>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>LRG_99</th>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "      <td>0</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>64102 rows × 8 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                   1    2    3    4     5     6    7    8\n",
-       "0                                                        \n",
-       "ENSG00000000003  679  448  873  408  1138  1047  770  572\n",
-       "ENSG00000000005    0    0    0    0     0     0    0    0\n",
-       "ENSG00000000419  467  515  621  365   587   799  417  508\n",
-       "ENSG00000000457  260  211  263  164   245   331  233  229\n",
-       "ENSG00000000460   60   55   40   35    78    63   76   60\n",
-       "...              ...  ...  ...  ...   ...   ...  ...  ...\n",
-       "LRG_94             0    0    0    0     0     0    0    0\n",
-       "LRG_96             0    0    0    0     0     0    0    0\n",
-       "LRG_97             0    0    0    0     0     0    0    0\n",
-       "LRG_98             0    0    0    0     0     0    0    0\n",
-       "LRG_99             0    0    0    0     0     0    0    0\n",
-       "\n",
-       "[64102 rows x 8 columns]"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "airway"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "3e57eebe",
-   "metadata": {},
-   "source": [
-    "### Find all genes with mean read count $\\ge$  50\n",
-    "\n",
-    "#### Using `pandas`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "5d5ed72f",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "0\n",
-       "ENSG00000000003     741.875\n",
-       "ENSG00000000419     534.875\n",
-       "ENSG00000000457     242.000\n",
-       "ENSG00000000460      58.375\n",
-       "ENSG00000000971    6034.750\n",
-       "                     ...   \n",
-       "ENSG00000273179     174.875\n",
-       "ENSG00000273230     119.500\n",
-       "ENSG00000273270     137.375\n",
-       "ENSG00000273290     168.500\n",
-       "ENSG00000273344      65.500\n",
-       "Length: 12671, dtype: float64"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "airway_mean = airway.mean(axis=1)\n",
-    "airway_mean[airway_mean>=50]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "06b09f77",
-   "metadata": {},
-   "source": [
-    "#### Using `for` loop and `map`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "4e46b096",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from statistics import mean"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "6de40095",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "airway_genes = []\n",
-    "with open('data/airway.csv','r') as fh:\n",
-    "    for f in fh:\n",
-    "        seq_data = f.strip().split('\\t')\n",
-    "        count_data = map(int, seq_data[1:])\n",
-    "        count_mean = mean(count_data)\n",
-    "        if count_mean > 50.0:\n",
-    "            airway_genes.append((seq_data[0],count_mean))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "id": "6b4c2415",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Genes after filtering:  12666\n",
-      "First 3 from list:  [('ENSG00000000003', 741.875), ('ENSG00000000419', 534.875), ('ENSG00000000457', 242)]\n"
-     ]
-    }
-   ],
-   "source": [
-    "print('Genes after filtering: ',len(airway_genes))\n",
-    "print('First 3 from list: ', airway_genes[:3])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "8a30ccc7",
-   "metadata": {},
-   "source": [
-    "#### Rewriting using `pipe` and curried versions of `reduce` and `filter`\n",
-    "\n",
-    "Data flow: `open` $\\rightarrow$ `split` $\\rightarrow$ `map` $\\rightarrow$ `reduce` $\\rightarrow$ `filter`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "2403784d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from toolz.curried import map as mapc\n",
-    "from toolz.curried import filter as filterc\n",
-    "from toolz.functoolz import pipe, curry"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "id": "b4ba3e35",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def read_data(fin):\n",
-    "    with open(fin,'r') as fh:\n",
-    "        for l in fh:\n",
-    "            yield l.strip().split('\\t')\n",
-    "\n",
-    "def map_int_mean(raw_data):\n",
-    "    count_mean = pipe(raw_data[1:], mapc(int), mean)\n",
-    "    return (raw_data[0],count_mean)\n",
-    "\n",
-    "def filter_mean(gene_tuple):\n",
-    "    return gene_tuple[1] >= 50.00"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "id": "72de5559",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "airway_genes_50 = pipe('data/airway.csv',read_data, mapc(map_int_mean), filterc(filter_mean))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "372e9613",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "12671"
-      ]
-     },
-     "execution_count": 11,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len(list(airway_genes_50))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "702565cb",
-   "metadata": {},
-   "source": [
-    "#### Rewriting `map_int_mean` using `compose`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
-   "id": "1791bccc",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from toolz.functoolz import compose"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 26,
-   "id": "1561cba8",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def map_int_mean_compose(raw_data):\n",
-    "    mean_from_str = compose(mean, mapc(int))\n",
-    "    return (raw_data[0], mean_from_str(raw_data[1:]))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 29,
-   "id": "fb813580",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "airway_genes_50 = pipe('data/airway.csv',read_data, mapc(map_int_mean_compose), filterc(filter_mean))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 30,
-   "id": "e7aacccd",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "12671"
-      ]
-     },
-     "execution_count": 30,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len(list(airway_genes_50))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "7a8c580b",
-   "metadata": {},
-   "source": [
-    "#### Rewriting `filter_mean` as a curried version\n",
-    "\n",
-    "Adding an additional input `thresh` for  `filter_mean` "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "id": "55b4cb77",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "@curry # <--- using curry function as a decorator\n",
-    "def filter_mean_thresh(gene_tuple,thresh=0.0):\n",
-    "    return gene_tuple[1] >= thresh"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "id": "fc9f6e5e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "airway_genes_100 = pipe('data/airway.csv',read_data, mapc(map_int_mean_compose), filterc(filter_mean_thresh(thresh=100.0)),list)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "id": "d4c65bee",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "11244"
-      ]
-     },
-     "execution_count": 14,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "len(airway_genes_100)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 16,
-   "id": "b543592a",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[('ENSG00000000003', 741.875),\n",
-       " ('ENSG00000000419', 534.875),\n",
-       " ('ENSG00000000457', 242)]"
-      ]
-     },
-     "execution_count": 16,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "airway_genes_100[:3]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 25,
-   "id": "7bd836f3",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "2"
-      ]
-     },
-     "execution_count": 25,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "mean_from_str = compose(mean, mapc(int))\n",
-    "mean_from_str(['1','2','3'])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "618281ff",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.2"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}