diff --git a/.gitignore b/.gitignore
index 6ad5a192191288c6b22db2fdc4e27eb10b6c776b..4dc5d2b8f61ed0101fde761959f729aabf11c3ee 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+*.log
+
MANIFEST
build
dist
@@ -33,6 +35,7 @@ venv*/
# various auxiliary files
analysis/figures/*
+old_data/*
data/*
*.h5ad
*.tsv
diff --git a/analysis/Untitled.ipynb b/analysis/Untitled.ipynb
index 5ed3f7c0d3144a5d682191f586f694048ec68e03..46e76239e90df7e75f66aa33c204d20b8c3b4342 100644
--- a/analysis/Untitled.ipynb
+++ b/analysis/Untitled.ipynb
@@ -3,14 +3,14 @@
{
"cell_type": "code",
"execution_count": 1,
- "id": "814d8de9",
+ "id": "273c6a8d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "2022-07-27 13:53\n"
+ "2022-07-29 14:01\n"
]
}
],
@@ -28,12 +28,14 @@
{
"cell_type": "code",
"execution_count": 2,
- "id": "3808ea96",
+ "id": "09defaef-09b9-4d8e-84a1-dc3c7a56b80a",
"metadata": {},
"outputs": [],
"source": [
"import glob\n",
"from os.path import exists\n",
+ "from tqdm import tqdm\n",
+ "import requests\n",
"\n",
"import pandas as pd\n",
"import numpy as np\n",
@@ -45,7 +47,7 @@
{
"cell_type": "code",
"execution_count": 3,
- "id": "ad6f9837",
+ "id": "8aaa0695",
"metadata": {},
"outputs": [],
"source": [
@@ -56,23 +58,23 @@
{
"cell_type": "code",
"execution_count": 4,
- "id": "1a0df0c0",
+ "id": "95457088",
"metadata": {},
"outputs": [],
"source": [
- "just_mmseqs = pd.read_csv('../data/cfres_mmseqs_s75_e1.m8', sep=\"\\s+\", header=None)\n",
- "just_mmseqs.columns = [\"query\", \"target\", \"seq. id.\", \"alignment length\", \"no. mismatches\",\n",
+ "mmseqs = pd.read_csv('../data/cfres_mmseqs_s75_e1.m8', sep=\"\\s+\", header=None)\n",
+ "mmseqs.columns = [\"query\", \"target\", \"seq. id.\", \"alignment length\", \"no. mismatches\",\n",
" \"no. gap open\", \"query start\", \"query end\", \"target start\", \"target end\",\n",
" \"e value\", \"bit score\"]\n",
"\n",
- "just_mmseqs['gene_id'] = just_mmseqs['query'].str.split('_').str[:2].apply(lambda x: '_'.join(x))\n",
- "just_mmseqs['normalized bit score'] = just_mmseqs['bit score'] / just_mmseqs['alignment length']"
+ "mmseqs['gene_id'] = mmseqs['query'].str.split('_').str[:2].apply(lambda x: '_'.join(x))\n",
+ "mmseqs['normalized bit score'] = mmseqs['bit score'] / mmseqs['alignment length']"
]
},
{
"cell_type": "code",
"execution_count": 5,
- "id": "b09c96b5",
+ "id": "093bff38",
"metadata": {},
"outputs": [],
"source": [
@@ -91,17 +93,18 @@
{
"cell_type": "code",
"execution_count": 6,
- "id": "ff6f8bf7",
+ "id": "042ff08e",
"metadata": {},
"outputs": [],
"source": [
- "just_mmseqs_filtered = keep_best(just_mmseqs)"
+ "mmseqs_filtered = keep_best(mmseqs)\n",
+ "del mmseqs"
]
},
{
"cell_type": "code",
"execution_count": 7,
- "id": "af68258b",
+ "id": "9645fec8",
"metadata": {},
"outputs": [],
"source": [
@@ -117,17 +120,18 @@
{
"cell_type": "code",
"execution_count": 8,
- "id": "5380700c",
+ "id": "7e9d0601",
"metadata": {},
"outputs": [],
"source": [
- "hhblits_filtered = keep_best(hhblits)"
+ "hhblits_filtered = keep_best(hhblits)\n",
+ "del hhblits"
]
},
{
"cell_type": "code",
"execution_count": 9,
- "id": "2d829347",
+ "id": "260c26ec",
"metadata": {},
"outputs": [
{
@@ -151,7 +155,7 @@
{
"cell_type": "code",
"execution_count": 10,
- "id": "b72335fe",
+ "id": "c92b4a01",
"metadata": {},
"outputs": [],
"source": [
@@ -161,13 +165,13 @@
{
"cell_type": "code",
"execution_count": 11,
- "id": "e7595d52",
+ "id": "5ceaaff2",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "<matplotlib.legend.Legend at 0x7fdd93502580>"
+ "<matplotlib.legend.Legend at 0x7f58ab679250>"
]
},
"execution_count": 11,
@@ -176,7 +180,7 @@
},
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZdUlEQVR4nO3df3BV5b3v8feXgAUUxJLcthJs0jOIAgkBg3oLF+GgGNCRudoZRUFOW0qZi0due0XwtMcWOu30jJxbD1WhDKWWuVSoSCn3CtpSsIK1hx+6G34oGmlKQlqNoShyoPzwe//ITs4m7JCdZO2svdf+vGYyZK317LW/e0/45Mmzn/Usc3dERCT7dQu7ABERCYYCXUQkIhToIiIRoUAXEYkIBbqISER0D+uJ8/PzvaioKKynFxHJSnv27Hnf3QuSHQst0IuKiti9e3dYTy8ikpXM7E+tHdOQi4hIRCjQRUQiQoEuIhIRoY2hi0juOHPmDLW1tZw6dSrsUrJGz549KSwspEePHik/RoEuImlXW1tLnz59KCoqwszCLifjuTsNDQ3U1tZSXFyc8uM05CIiaXfq1Cn69++vME+RmdG/f/92/0WjQBeRLqEwb5+OvF8KdBGRiNAYuoh0uS0H3g30fDcP+VSg58tW6qFfzMHNYVcgIpIyBbqI5ITq6mquueYaZs6cybBhw7jvvvvYsmULo0ePZtCgQezcuZNvf/vbzJgxg4kTJ1JUVMT69et5+OGHKSkpoaKigjNnzgCwYMEChgwZQmlpKQ899BAA9fX13HXXXYwaNYpRo0bxyiuvANDQ0MDEiRMZMWIEX/3qV/nsZz/L+++/z4kTJ7jtttsYPnw4w4YNY+3atZ1+jQp0EckZVVVVzJ07l8rKSt58801+9rOfsWPHDhYvXsz3vvc9AN555x2ef/55fvnLXzJt2jTGjx/P3r176dWrF88//zxHjx7lF7/4Bfv376eyspJvfvObAMydO5evfe1r7Nq1i+eee46ZM2cCsHDhQsaMGcPrr7/OHXfcweHDhwF44YUXuPLKK/nDH/7Avn37qKio6PTrU6CLSM4oLi6mpKSEbt26MXToUCZMmICZUVJSQnV1NQCTJk2iR48elJSUcO7cueagbWrTt29fevbsycyZM1m/fj29e/cGYMuWLTzwwAOUlZVxxx138OGHH3L8+HFefvllpk2bBsBtt93GFVdc0Xy+LVu2MH/+fLZv387ll1/e6denQL+IWM2xsEsQkQB94hOfaP6+W7duzdvdunXj7Nmz57Xp1q0bPXr0aJ4+2NSme/fu7Ny5k7vuuosNGzY0B/7HH3/Mq6++SiwWIxaLceTIEfr06QMkn4J49dVXs2fPHkpKSnjkkUdYtGhRp1+fAr01+kBURJL46KOP+OCDD5g8eTKPP/44sVgMgIkTJ/LEE080t2vaP3bsWFavXg3A5s2b+etf/wpAXV0dvXv3Ztq0aTz00EO89tprna5N0xZFpMtl8zTD48ePM2XKFE6dOoW784Mf/ACAJUuWMGfOHEpLSzl79ixjx45l2bJlfOtb32Lq1KmMHDmSm266iauuugqAvXv3Mm/evOa/BJYuXdrp2szdO32SjigvL/eMvsHFwc3Eao5RdvPUsCsRyXpvvPEG1157bdhlZISmm/vk5+e32TbZ+2Zme9y9PFn7NodczGylmb1nZvtaOX6fmVXGv35nZsPbrFJERAKXyhj608DF5tP8EbjJ3UuB7wDLA6hLRCSSqqurU+qdd0SbY+ju/rKZFV3k+O8SNn8PFAZQl4iItFPQs1y+DLQ6PcTMZpnZbjPbXV9fH/BTB0tTFkUk2wQW6GY2nsZAn99aG3df7u7l7l5eUFAQ1FN3maAXFBIRCVIg0xbNrBRYAUxy94YgzikiIu3T6UA3s6uA9cB0d3+r8yVlrvy6rTBE0xhFOi3oC/cGT2qzSXV1Nbfffjv79p0/YW/cuHEsXryY8vLzZwI+/fTT7N69+7yLhZpcdtllfPTRR9TV1fHggw+ybt06YrEYdXV1TJ48uXOvpRNSmbb4DPAqMNjMas3sy2Y228xmx5s8CvQHnjKzmJll8OTyTtCVoyLSwpVXXsm6deuAxitDN23aFGo9bQa6u09198+4ew93L3T3H7v7MndfFj8+092vcPey+FfSCe/Z5LyxcgW5SGScO3eOr3zlKwwdOpSJEydy8uRJAJ599lmuv/56rr76arZv397cvqamhoqKCgYPHszChQsvOF91dTXDhg3j9OnTPProo6xdu5aysjLWrl3Lb3/7W8rKyigrK2PEiBEcP3487a9Pa7k0aQrug5sbh1biNNtFJDrefvtt5syZw/79++nXrx/PPfccAGfPnmXnzp08/vjj5wX3zp07Wb16NbFYjGeffZbWrm6/5JJLWLRoEXfffTexWIy7776bxYsX8+STTxKLxdi+fTu9evVK++tToItIziguLqasrAyA6667rnnJ3DvvvPOCfQC33HIL/fv3p1evXtx5553s2LEj5ecaPXo0X//611myZAnHjh2je/f0L52lQBeRnJG4fG5eXt4FS+Ym7oMLl71NtgxuaxYsWMCKFSs4efIkN954I2+++WZnSk+JAl1EpBW//vWvOXr0KCdPnmTDhg2MHj261bZ9+vQ5b5z8nXfeoaSkhPnz51NeXt4lga7lc0Wk66UwzTATjBkzhunTp1NVVcW99957wdTGROPHj+f73/8+ZWVlPPLII+zYsYNt27aRl5fHkCFDmDQp/a9Zy+c2Obi58YcsvmxuorKbp2o5XZFO0PK5HRP48rnSSLNdRCTTKdBFRCJCgZ5IFxGJSBZToIuIRIQCXUQkIhToIiIRoXno7bTlwLvcPORTYZchktVeqnkp0PONGzgusHPV19dz++23c/r0aZYsWcJf/vIXHn30UT796U+zbdu2wJ4nHRToCWI1xygb2C/sMkQkRL/5zW+45ppr+OlPfwpARUUFTz31FOPHjw+5srZpyEVEcsKqVasoLS1l+PDhTJ8+nT/96U9MmDCB0tJSJkyYwOHDh4nFYjz88MNs2rSJsrIyFi5cyI4dO5g9ezbz5s3j3LlzzJs3j1GjRlFaWsqPfvSjsF/WedRD74imq0pFJCvs37+f7373u7zyyivk5+dz9OhRZsyYwf3338+MGTNYuXIlDz74IBs2bGDRokXn3alo27ZtzXc0Wr58OZdffjm7du3ib3/7G6NHj2bixIkUFxeH/AobqYcepytBRaJr69atfOELXyA/Px+AT37yk7z66qvce++9AEyfPj2lpXF/9atfsWrVKsrKyrjhhhtoaGjg7bffTmvt7aEeejvl120FjbOLZBV3b3Pp21SWxnV3fvjDH3LrrbcGVVqg1EMXkcibMGECP//5z2loaADg6NGjfP7zn2fNmjUArF69mjFjxrR5nltvvZWlS5dy5swZAN566y1OnDiRvsLbST30FMS2PBN2CSKREuQ0w1QMHTqUb3zjG9x0003k5eUxYsQIlixZwpe+9CUee+wxCgoK+MlPftLmeWbOnEl1dTUjR47E3SkoKGDDhg3pfwEp0vK5cU2hXTawX5vj6WUD++lDUZF20PK5HaPlc0VEcpQCXUQkIhToItIlwhrezVYdeb/aDHQzW2lm75nZvlaOm5ktMbMqM6s0s5HtriKLbTnwbtgliGS8nj170tDQoFBPkbvT0NBAz5492/W4VGa5PA08Aaxq5fgkYFD86wZgafxfEREACgsLqa2tpb6+PuxSskbPnj0pLCxs12PaDHR3f9nMii7SZAqwyht/9f7ezPqZ2Wfc/c/tqkREIqtHjx4Zc3l8lAUxhj4AqEnYro3vu4CZzTKz3Wa2O1N/U2sJABHJVkEEerLrZZMOlLn7cncvd/fygoKCAJ5aRESaBBHotcDAhO1CoC6A83aZznywmV+3NcBKREQ6LohA3wjcH5/tciPwQc6Mnx/cHHYFIiLN2vxQ1MyeAcYB+WZWC3wL6AHg7suATcBkoAr4D+CL6So2LQ5uJr/uWNhViIh0WiqzXKa2cdyBOYFVJCIiHaIrRUVEIkKB3gma4igimUSB3gEKchHJRAp0EZGIUKCLiESEAl1EJCJyPtA1Hi4iUZHzgS4iEhUKdBGRiFCgB0FruohIBlCgd5BuPScimUaBLiISEQp0EZGIUKCLiESEAl1EJCIU6CIiEaFAD4CuNhWRTKBAFxGJCAW6iEhE5Hagd+IKz/y6rQEWIiLSebkd6AHSlaMiEjYFuohIROR0oGt2iohESUqBbmYVZnbQzKrMbEGS45eb2f81sz+Y2X4z+2LwpYqIyMW0Gehmlgc8CUwChgBTzWxIi2ZzgAPuPhwYB/yrmV0ScK0iInIRqfTQrweq3P2Qu58G1gBTWrRxoI+ZGXAZcBQ4G2ilIiJyUakE+gCgJmG7Nr4v0RPAtUAdsBeY6+4fB1JhmmhWiohETSqBbkn2eYvtW4EYcCVQBjxhZn0vOJHZLDPbbWa76+vr21lqsDSPXESiJpVArwUGJmwX0tgTT/RFYL03qgL+CFzT8kTuvtzdy929vKCgoKM1Zy7dik5EQpRKoO8CBplZcfyDznuAjS3aHAYmAJjZp4DBwKEgCxURkYvr3lYDdz9rZg8ALwJ5wEp3329ms+PHlwHfAZ42s700DtHMd/f301h3xsmv2woD+4VdhojksDYDHcDdNwGbWuxblvB9HTAx2NJERKQ9cvpK0aDpylMRCZMCXUQkIhToIiIRoUBPB01fFJEQ5GSg6ypREYminAx0EZEoUqAHTL1/EQmLAl1EJCIU6CIiEaFAFxGJCAW6iEhEKNBFRCJCgS4iEhEKdBGRiFCgi4hEhAJdRCQiFOgiIhGRc4G+5cC7jbeLSyPd6EJEwpBzgS4iElUKdBGRiFCgi4hEhAI9YOkenxcRaY0CXUQkIhToIiIRkVKgm1mFmR00syozW9BKm3FmFjOz/Wb222DLDI6GREQkqrq31cDM8oAngVuAWmCXmW109wMJbfoBTwEV7n7YzP5LmuoVEZFWpNJDvx6ocvdD7n4aWANMadHmXmC9ux8GcPf3gi0z++jeoiLS1VIJ9AFATcJ2bXxfoquBK8zsJTPbY2b3JzuRmc0ys91mtru+vr5jFYuISFKpBLol2ecttrsD1wG3AbcC/2xmV1/wIPfl7l7u7uUFBQXtLrbTDm7usqfSWL2IdLU2x9Bp7JEPTNguBOqStHnf3U8AJ8zsZWA48FYgVYqISJtS6aHvAgaZWbGZXQLcA2xs0eaXwH8zs+5m1hu4AXgj2FJFRORi2uyhu/tZM3sAeBHIA1a6+34zmx0/vszd3zCzF4BK4GNghbvvS2fhIiJyvlSGXHD3TcCmFvuWtdh+DHgsuNKCp2VtRSTKdKWoiEhEKNBFRCJCgZ5uXThVUkRymwI9nQ5u1ri9iHQZBbqISEQo0NNIvXMR6UoKdBGRiEhpHnqUvXby7aT7R/Ya1MWViIh0jnroIiIRoUAXEYmInBxyaW2YRUQkm6mHLiISEQr0rqCrRUWkCyjQRUQiQoEuIhIRCvQuoCtGRaQrKNBFRCIiZ6YtbjnwLvntaJ84tVFXjYpINlAPXUQkIhToXUnTF0UkjXIm0PPrtoZdgohIWuVMoIuIRJ0CvatouEVE0ixnZrlkwoJcsZpjlA0OuwoRiaqUeuhmVmFmB82syswWXKTdKDM7Z2ZfCK7EAKh3LCI5oM0eupnlAU8CtwC1wC4z2+juB5K0+xfgxXQUGqYg5qTralERSbdUeujXA1XufsjdTwNrgClJ2v0j8BzwXoD1BUJhKiK5IJVAHwDUJGzXxvc1M7MBwH8Hll3sRGY2y8x2m9nu+vr69tYqIiIXkUqgW5J93mL7cWC+u5+72Incfbm7l7t7eUFBQYoliohIKlKZ5VILDEzYLgTqWrQpB9aYGUA+MNnMzrr7hiCKFBGRtqXSQ98FDDKzYjO7BLgH2JjYwN2L3b3I3YuAdcD/UJgnt+XAu2GXICIR1WYP3d3PmtkDNM5eyQNWuvt+M5sdP37RcXMREekaKV1Y5O6bgE0t9iUNcnf/h86XFXEHN8PgSWFXISIRo0v/Q6BplCKSDpEP9NiWZ8IuQUSkS0Q+0EVEcoUCXUQkIhToIiIRkTPL5walswt16c5JIpIu6qGHRUv6ikjAFOgiIhER6SGXl2pe4lAG3KlIRKQrqIcuIhIRCnQRkYiI9JBLrOYYfdN4/iBuTSciEhT10EVEIiLSgd63oTLsElqlBbpEJGiRDnQRkVyiQBcRiYjIBno23OotG2oUkewR2UDPFgp1EQmKAl1EJCIU6AF57eTbzV/tkbj6onrrItIZkb2wKL9uK4fDLiJVzSsvjgy1DBHJbuqhZwDNSReRICjQM4RCXUQ6K6VAN7MKMztoZlVmtiDJ8fvMrDL+9TszGx58qdkj1fF03b1IRILUZqCbWR7wJDAJGAJMNbMhLZr9EbjJ3UuB7wDLgy40FyjgRaQzUumhXw9Uufshdz8NrAGmJDZw99+5+1/jm78HCoMtU0RE2pLKLJcBQE3Cdi1ww0XafxlIesNMM5sFzAK46qqrUiyxfV6qeQkgY+5U1KEldg9uhsGT0lSRiERVKj10S7LPkzY0G09joM9Pdtzdl7t7ubuXFxQUpF5lDtFcdBHpqFQCvRYYmLBdCNS1bGRmpcAKYIq7NwRTXm7SjBcR6YhUhlx2AYPMrBg4AtwD3JvYwMyuAtYD0939rcCrjIjWZr3obkciEoQ2A93dz5rZA8CLQB6w0t33m9ns+PFlwKNAf+ApMwM46+7l6StbRERaSunSf3ffBGxqsW9ZwvczgZnBlpY7zvvgNGEwq+kDXoBxA8d1WT0ikp0iu5ZLtmoK92M1L523P5VwT2zTUuJjWmunXxoi2S2SgR6rOUbfsItIo4703C8W9kG0b+0XRnt/+eiXikjHRTLQc0l7gzdd50rlsUHWKiIXilyga8pfZlGIi3SdyK222LehMuwSRERCEbkeOkQn1A/FXuZzZWPDLqNLpTK2rtk/IslFMtAj4ciesCvIKK0FvcJd5D8p0DPUofoTYZeQ1RT0kosU6BJ5qc7PF8l2WR/omkUhTfSzILku6wM98o7sgQHXhV1FTtAwjWS7aAV6BD9IPFR/gs+hUO9qCnfJRtEK9Ig6VH+CDz8+RtnAfmGXEjmducJVQS+ZRoGeJfo2VEK3S9VTz1Dq0UsmiFSgR32q36H6E3xuQNhVSBPNjZdME6lAzwWxGg29ZBOFu3SlyAT6odjLYZfQJfo2VMLA3FoOIIo0Li/pEJlAzyXqpWcnfQAr6aZAF8kyGsaR1ijQs1CyGS/qtUebPoCVVEQj0CN4QVG7HNkD/F3YVUjIFO6S9YEeqzlG34ZoT1dMpuliI4CyyN2mRDorlfF6hX70ZH2gR+VmFh3Rt6GSD/uXNn/PwLEaepGUXaxHrw9ns1PWB7okiA+95OKdjqRzUl2pUkGf2VIKdDOrAP4NyANWuPv3Wxy3+PHJwH8A/+DurwVcazMtk/qf+jZUQsGlQPxK2f6N+5tulq3eunSF1nr77d0vndNmoJtZHvAkcAtQC+wys43ufiCh2SRgUPzrBmBp/N+0idUco6zbO+l8iqyRbMmD5qGobpcS+/jvGoP9yJ7Gsff4MI3CXtIhlRk5qexX0LdfKj3064Eqdz8EYGZrgClAYqBPAVa5uwO/N7N+ZvYZd/9z4BXH9W2o5FC6Tp7FWn6mcKj+BH2p5FDDhW0ONcCH/UsV7JKRgvpLvLW/DFprk81SCfQBQE3Cdi0X9r6TtRkAnBfoZjYLmBXf/MjMDrarWsgH3m/nY3KB3pfk9L5cSO9Jctn0vny2tQOpBLol2ecdaIO7LweWp/CcyQsx2+3u5R19fFTpfUlO78uF9J4kF5X3JZUZzLXAwITtQqCuA21ERCSNUgn0XcAgMys2s0uAe4CNLdpsBO63RjcCH6Rz/FxERC7U5pCLu581sweAF2mctrjS3feb2ez48WXAJhqnLFbROG3xi2mqt8PDNRGn9yU5vS8X0nuSXCTeF2ucmCIiItlOq4CIiESEAl1EJCKyJtDNrMLMDppZlZktCLuesJnZQDPbZmZvmNl+M5sbdk2ZxMzyzOx1M/t/YdeSKeIX/K0zszfjPzf/NeyawmZmX4v//9lnZs+YWc+wa+qMrAj0hOUHJgFDgKlmNiTcqkJ3Fvhf7n4tcCMwR+/JeeYCb4RdRIb5N+AFd78GGE6Ovz9mNgB4ECh392E0Tvq4J9yqOicrAp2E5Qfc/TTQtPxAznL3PzctgObux2n8zzkg3Koyg5kVArcBK8KuJVOYWV9gLPBjAHc/7e7HQi0qM3QHeplZd6A3WX79TLYEemtLCwhgZkXACODfQy4lUzwOPAx8HHIdmeRzQD3wk/hQ1AozuzTsosLk7keAxcBhGpcp+cDdfxVuVZ2TLYGe0tICucjMLgOeA/6nu38Ydj1hM7PbgffcPcfvS3iB7sBIYKm7jwBOADn9WZSZXUHjX/rFwJXApWY2LdyqOidbAl1LCyRhZj1oDPPV7r4+7HoyxGjgDjOrpnFo7u/N7P+EW1JGqAVq3b3pr7h1NAZ8LrsZ+KO717v7GWA98PmQa+qUbAn0VJYfyCnxm4r8GHjD3f932PVkCnd/xN0L3b2Ixp+Tre6e1b2uILj7X4AaMxsc3zWB85fAzkWHgRvNrHf8/9MEsvyD4qy4BV1ryw+EXFbYRgPTgb1mFovv+yd33xReSZLh/hFYHe8UHSJ9S3RkBXf/dzNbB7xG46yx18nyJQB06b+ISERky5CLiIi0QYEuIhIRCnQRkYhQoIuIRIQCXUQkIhToIiIRoUAXEYmI/w9iflt0knsXdQAAAABJRU5ErkJggg==\n",
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZdUlEQVR4nO3df3BV5b3v8feXgAUUxJLcthJs0jOIAgkBg3oLF+GgGNCRudoZRUFOW0qZi0due0XwtMcWOu30jJxbD1WhDKWWuVSoSCn3CtpSsIK1hx+6G34oGmlKQlqNoShyoPzwe//ITs4m7JCdZO2svdf+vGYyZK317LW/e0/45Mmzn/Usc3dERCT7dQu7ABERCYYCXUQkIhToIiIRoUAXEYkIBbqISER0D+uJ8/PzvaioKKynFxHJSnv27Hnf3QuSHQst0IuKiti9e3dYTy8ikpXM7E+tHdOQi4hIRCjQRUQiQoEuIhIRoY2hi0juOHPmDLW1tZw6dSrsUrJGz549KSwspEePHik/RoEuImlXW1tLnz59KCoqwszCLifjuTsNDQ3U1tZSXFyc8uM05CIiaXfq1Cn69++vME+RmdG/f/92/0WjQBeRLqEwb5+OvF8KdBGRiNAYuoh0uS0H3g30fDcP+VSg58tW6qFfzMHNYVcgIpIyBbqI5ITq6mquueYaZs6cybBhw7jvvvvYsmULo0ePZtCgQezcuZNvf/vbzJgxg4kTJ1JUVMT69et5+OGHKSkpoaKigjNnzgCwYMEChgwZQmlpKQ899BAA9fX13HXXXYwaNYpRo0bxyiuvANDQ0MDEiRMZMWIEX/3qV/nsZz/L+++/z4kTJ7jtttsYPnw4w4YNY+3atZ1+jQp0EckZVVVVzJ07l8rKSt58801+9rOfsWPHDhYvXsz3vvc9AN555x2ef/55fvnLXzJt2jTGjx/P3r176dWrF88//zxHjx7lF7/4Bfv376eyspJvfvObAMydO5evfe1r7Nq1i+eee46ZM2cCsHDhQsaMGcPrr7/OHXfcweHDhwF44YUXuPLKK/nDH/7Avn37qKio6PTrU6CLSM4oLi6mpKSEbt26MXToUCZMmICZUVJSQnV1NQCTJk2iR48elJSUcO7cueagbWrTt29fevbsycyZM1m/fj29e/cGYMuWLTzwwAOUlZVxxx138OGHH3L8+HFefvllpk2bBsBtt93GFVdc0Xy+LVu2MH/+fLZv387ll1/e6denQL+IWM2xsEsQkQB94hOfaP6+W7duzdvdunXj7Nmz57Xp1q0bPXr0aJ4+2NSme/fu7Ny5k7vuuosNGzY0B/7HH3/Mq6++SiwWIxaLceTIEfr06QMkn4J49dVXs2fPHkpKSnjkkUdYtGhRp1+fAr01+kBURJL46KOP+OCDD5g8eTKPP/44sVgMgIkTJ/LEE080t2vaP3bsWFavXg3A5s2b+etf/wpAXV0dvXv3Ztq0aTz00EO89tprna5N0xZFpMtl8zTD48ePM2XKFE6dOoW784Mf/ACAJUuWMGfOHEpLSzl79ixjx45l2bJlfOtb32Lq1KmMHDmSm266iauuugqAvXv3Mm/evOa/BJYuXdrp2szdO32SjigvL/eMvsHFwc3Eao5RdvPUsCsRyXpvvPEG1157bdhlZISmm/vk5+e32TbZ+2Zme9y9PFn7NodczGylmb1nZvtaOX6fmVXGv35nZsPbrFJERAKXyhj608DF5tP8EbjJ3UuB7wDLA6hLRCSSqqurU+qdd0SbY+ju/rKZFV3k+O8SNn8PFAZQl4iItFPQs1y+DLQ6PcTMZpnZbjPbXV9fH/BTB0tTFkUk2wQW6GY2nsZAn99aG3df7u7l7l5eUFAQ1FN3maAXFBIRCVIg0xbNrBRYAUxy94YgzikiIu3T6UA3s6uA9cB0d3+r8yVlrvy6rTBE0xhFOi3oC/cGT2qzSXV1Nbfffjv79p0/YW/cuHEsXryY8vLzZwI+/fTT7N69+7yLhZpcdtllfPTRR9TV1fHggw+ybt06YrEYdXV1TJ48uXOvpRNSmbb4DPAqMNjMas3sy2Y228xmx5s8CvQHnjKzmJll8OTyTtCVoyLSwpVXXsm6deuAxitDN23aFGo9bQa6u09198+4ew93L3T3H7v7MndfFj8+092vcPey+FfSCe/Z5LyxcgW5SGScO3eOr3zlKwwdOpSJEydy8uRJAJ599lmuv/56rr76arZv397cvqamhoqKCgYPHszChQsvOF91dTXDhg3j9OnTPProo6xdu5aysjLWrl3Lb3/7W8rKyigrK2PEiBEcP3487a9Pa7k0aQrug5sbh1biNNtFJDrefvtt5syZw/79++nXrx/PPfccAGfPnmXnzp08/vjj5wX3zp07Wb16NbFYjGeffZbWrm6/5JJLWLRoEXfffTexWIy7776bxYsX8+STTxKLxdi+fTu9evVK++tToItIziguLqasrAyA6667rnnJ3DvvvPOCfQC33HIL/fv3p1evXtx5553s2LEj5ecaPXo0X//611myZAnHjh2je/f0L52lQBeRnJG4fG5eXt4FS+Ym7oMLl71NtgxuaxYsWMCKFSs4efIkN954I2+++WZnSk+JAl1EpBW//vWvOXr0KCdPnmTDhg2MHj261bZ9+vQ5b5z8nXfeoaSkhPnz51NeXt4lga7lc0Wk66UwzTATjBkzhunTp1NVVcW99957wdTGROPHj+f73/8+ZWVlPPLII+zYsYNt27aRl5fHkCFDmDQp/a9Zy+c2Obi58YcsvmxuorKbp2o5XZFO0PK5HRP48rnSSLNdRCTTKdBFRCJCgZ5IFxGJSBZToIuIRIQCXUQkIhToIiIRoXno7bTlwLvcPORTYZchktVeqnkp0PONGzgusHPV19dz++23c/r0aZYsWcJf/vIXHn30UT796U+zbdu2wJ4nHRToCWI1xygb2C/sMkQkRL/5zW+45ppr+OlPfwpARUUFTz31FOPHjw+5srZpyEVEcsKqVasoLS1l+PDhTJ8+nT/96U9MmDCB0tJSJkyYwOHDh4nFYjz88MNs2rSJsrIyFi5cyI4dO5g9ezbz5s3j3LlzzJs3j1GjRlFaWsqPfvSjsF/WedRD74imq0pFJCvs37+f7373u7zyyivk5+dz9OhRZsyYwf3338+MGTNYuXIlDz74IBs2bGDRokXn3alo27ZtzXc0Wr58OZdffjm7du3ib3/7G6NHj2bixIkUFxeH/AobqYcepytBRaJr69atfOELXyA/Px+AT37yk7z66qvce++9AEyfPj2lpXF/9atfsWrVKsrKyrjhhhtoaGjg7bffTmvt7aEeejvl120FjbOLZBV3b3Pp21SWxnV3fvjDH3LrrbcGVVqg1EMXkcibMGECP//5z2loaADg6NGjfP7zn2fNmjUArF69mjFjxrR5nltvvZWlS5dy5swZAN566y1OnDiRvsLbST30FMS2PBN2CSKREuQ0w1QMHTqUb3zjG9x0003k5eUxYsQIlixZwpe+9CUee+wxCgoK+MlPftLmeWbOnEl1dTUjR47E3SkoKGDDhg3pfwEp0vK5cU2hXTawX5vj6WUD++lDUZF20PK5HaPlc0VEcpQCXUQkIhToItIlwhrezVYdeb/aDHQzW2lm75nZvlaOm5ktMbMqM6s0s5HtriKLbTnwbtgliGS8nj170tDQoFBPkbvT0NBAz5492/W4VGa5PA08Aaxq5fgkYFD86wZgafxfEREACgsLqa2tpb6+PuxSskbPnj0pLCxs12PaDHR3f9nMii7SZAqwyht/9f7ezPqZ2Wfc/c/tqkREIqtHjx4Zc3l8lAUxhj4AqEnYro3vu4CZzTKz3Wa2O1N/U2sJABHJVkEEerLrZZMOlLn7cncvd/fygoKCAJ5aRESaBBHotcDAhO1CoC6A83aZznywmV+3NcBKREQ6LohA3wjcH5/tciPwQc6Mnx/cHHYFIiLN2vxQ1MyeAcYB+WZWC3wL6AHg7suATcBkoAr4D+CL6So2LQ5uJr/uWNhViIh0WiqzXKa2cdyBOYFVJCIiHaIrRUVEIkKB3gma4igimUSB3gEKchHJRAp0EZGIUKCLiESEAl1EJCJyPtA1Hi4iUZHzgS4iEhUKdBGRiFCgB0FruohIBlCgd5BuPScimUaBLiISEQp0EZGIUKCLiESEAl1EJCIU6CIiEaFAD4CuNhWRTKBAFxGJCAW6iEhE5Hagd+IKz/y6rQEWIiLSebkd6AHSlaMiEjYFuohIROR0oGt2iohESUqBbmYVZnbQzKrMbEGS45eb2f81sz+Y2X4z+2LwpYqIyMW0Gehmlgc8CUwChgBTzWxIi2ZzgAPuPhwYB/yrmV0ScK0iInIRqfTQrweq3P2Qu58G1gBTWrRxoI+ZGXAZcBQ4G2ilIiJyUakE+gCgJmG7Nr4v0RPAtUAdsBeY6+4fB1JhmmhWiohETSqBbkn2eYvtW4EYcCVQBjxhZn0vOJHZLDPbbWa76+vr21lqsDSPXESiJpVArwUGJmwX0tgTT/RFYL03qgL+CFzT8kTuvtzdy929vKCgoKM1Zy7dik5EQpRKoO8CBplZcfyDznuAjS3aHAYmAJjZp4DBwKEgCxURkYvr3lYDdz9rZg8ALwJ5wEp3329ms+PHlwHfAZ42s700DtHMd/f301h3xsmv2woD+4VdhojksDYDHcDdNwGbWuxblvB9HTAx2NJERKQ9cvpK0aDpylMRCZMCXUQkIhToIiIRoUBPB01fFJEQ5GSg6ypREYminAx0EZEoUqAHTL1/EQmLAl1EJCIU6CIiEaFAFxGJCAW6iEhEKNBFRCJCgS4iEhEKdBGRiFCgi4hEhAJdRCQiFOgiIhGRc4G+5cC7jbeLSyPd6EJEwpBzgS4iElUKdBGRiFCgi4hEhAI9YOkenxcRaY0CXUQkIhToIiIRkVKgm1mFmR00syozW9BKm3FmFjOz/Wb222DLDI6GREQkqrq31cDM8oAngVuAWmCXmW109wMJbfoBTwEV7n7YzP5LmuoVEZFWpNJDvx6ocvdD7n4aWANMadHmXmC9ux8GcPf3gi0z++jeoiLS1VIJ9AFATcJ2bXxfoquBK8zsJTPbY2b3JzuRmc0ys91mtru+vr5jFYuISFKpBLol2ecttrsD1wG3AbcC/2xmV1/wIPfl7l7u7uUFBQXtLrbTDm7usqfSWL2IdLU2x9Bp7JEPTNguBOqStHnf3U8AJ8zsZWA48FYgVYqISJtS6aHvAgaZWbGZXQLcA2xs0eaXwH8zs+5m1hu4AXgj2FJFRORi2uyhu/tZM3sAeBHIA1a6+34zmx0/vszd3zCzF4BK4GNghbvvS2fhIiJyvlSGXHD3TcCmFvuWtdh+DHgsuNKCp2VtRSTKdKWoiEhEKNBFRCJCgZ5uXThVUkRymwI9nQ5u1ri9iHQZBbqISEQo0NNIvXMR6UoKdBGRiEhpHnqUvXby7aT7R/Ya1MWViIh0jnroIiIRoUAXEYmInBxyaW2YRUQkm6mHLiISEQr0rqCrRUWkCyjQRUQiQoEuIhIRCvQuoCtGRaQrKNBFRCIiZ6YtbjnwLvntaJ84tVFXjYpINlAPXUQkIhToXUnTF0UkjXIm0PPrtoZdgohIWuVMoIuIRJ0CvatouEVE0ixnZrlkwoJcsZpjlA0OuwoRiaqUeuhmVmFmB82syswWXKTdKDM7Z2ZfCK7EAKh3LCI5oM0eupnlAU8CtwC1wC4z2+juB5K0+xfgxXQUGqYg5qTralERSbdUeujXA1XufsjdTwNrgClJ2v0j8BzwXoD1BUJhKiK5IJVAHwDUJGzXxvc1M7MBwH8Hll3sRGY2y8x2m9nu+vr69tYqIiIXkUqgW5J93mL7cWC+u5+72Incfbm7l7t7eUFBQYoliohIKlKZ5VILDEzYLgTqWrQpB9aYGUA+MNnMzrr7hiCKFBGRtqXSQ98FDDKzYjO7BLgH2JjYwN2L3b3I3YuAdcD/UJgnt+XAu2GXICIR1WYP3d3PmtkDNM5eyQNWuvt+M5sdP37RcXMREekaKV1Y5O6bgE0t9iUNcnf/h86XFXEHN8PgSWFXISIRo0v/Q6BplCKSDpEP9NiWZ8IuQUSkS0Q+0EVEcoUCXUQkIhToIiIRkTPL5walswt16c5JIpIu6qGHRUv6ikjAFOgiIhER6SGXl2pe4lAG3KlIRKQrqIcuIhIRCnQRkYiI9JBLrOYYfdN4/iBuTSciEhT10EVEIiLSgd63oTLsElqlBbpEJGiRDnQRkVyiQBcRiYjIBno23OotG2oUkewR2UDPFgp1EQmKAl1EJCIU6AF57eTbzV/tkbj6onrrItIZkb2wKL9uK4fDLiJVzSsvjgy1DBHJbuqhZwDNSReRICjQM4RCXUQ6K6VAN7MKMztoZlVmtiDJ8fvMrDL+9TszGx58qdkj1fF03b1IRILUZqCbWR7wJDAJGAJMNbMhLZr9EbjJ3UuB7wDLgy40FyjgRaQzUumhXw9Uufshdz8NrAGmJDZw99+5+1/jm78HCoMtU0RE2pLKLJcBQE3Cdi1ww0XafxlIesNMM5sFzAK46qqrUiyxfV6qeQkgY+5U1KEldg9uhsGT0lSRiERVKj10S7LPkzY0G09joM9Pdtzdl7t7ubuXFxQUpF5lDtFcdBHpqFQCvRYYmLBdCNS1bGRmpcAKYIq7NwRTXm7SjBcR6YhUhlx2AYPMrBg4AtwD3JvYwMyuAtYD0939rcCrjIjWZr3obkciEoQ2A93dz5rZA8CLQB6w0t33m9ns+PFlwKNAf+ApMwM46+7l6StbRERaSunSf3ffBGxqsW9ZwvczgZnBlpY7zvvgNGEwq+kDXoBxA8d1WT0ikp0iu5ZLtmoK92M1L523P5VwT2zTUuJjWmunXxoi2S2SgR6rOUbfsItIo4703C8W9kG0b+0XRnt/+eiXikjHRTLQc0l7gzdd50rlsUHWKiIXilyga8pfZlGIi3SdyK222LehMuwSRERCEbkeOkQn1A/FXuZzZWPDLqNLpTK2rtk/IslFMtAj4ciesCvIKK0FvcJd5D8p0DPUofoTYZeQ1RT0kosU6BJ5qc7PF8l2WR/omkUhTfSzILku6wM98o7sgQHXhV1FTtAwjWS7aAV6BD9IPFR/gs+hUO9qCnfJRtEK9Ig6VH+CDz8+RtnAfmGXEjmducJVQS+ZRoGeJfo2VEK3S9VTz1Dq0UsmiFSgR32q36H6E3xuQNhVSBPNjZdME6lAzwWxGg29ZBOFu3SlyAT6odjLYZfQJfo2VMLA3FoOIIo0Li/pEJlAzyXqpWcnfQAr6aZAF8kyGsaR1ijQs1CyGS/qtUebPoCVVEQj0CN4QVG7HNkD/F3YVUjIFO6S9YEeqzlG34ZoT1dMpuliI4CyyN2mRDorlfF6hX70ZH2gR+VmFh3Rt6GSD/uXNn/PwLEaepGUXaxHrw9ns1PWB7okiA+95OKdjqRzUl2pUkGf2VIKdDOrAP4NyANWuPv3Wxy3+PHJwH8A/+DurwVcazMtk/qf+jZUQsGlQPxK2f6N+5tulq3eunSF1nr77d0vndNmoJtZHvAkcAtQC+wys43ufiCh2SRgUPzrBmBp/N+0idUco6zbO+l8iqyRbMmD5qGobpcS+/jvGoP9yJ7Gsff4MI3CXtIhlRk5qexX0LdfKj3064Eqdz8EYGZrgClAYqBPAVa5uwO/N7N+ZvYZd/9z4BXH9W2o5FC6Tp7FWn6mcKj+BH2p5FDDhW0ONcCH/UsV7JKRgvpLvLW/DFprk81SCfQBQE3Cdi0X9r6TtRkAnBfoZjYLmBXf/MjMDrarWsgH3m/nY3KB3pfk9L5cSO9Jctn0vny2tQOpBLol2ecdaIO7LweWp/CcyQsx2+3u5R19fFTpfUlO78uF9J4kF5X3JZUZzLXAwITtQqCuA21ERCSNUgn0XcAgMys2s0uAe4CNLdpsBO63RjcCH6Rz/FxERC7U5pCLu581sweAF2mctrjS3feb2ez48WXAJhqnLFbROG3xi2mqt8PDNRGn9yU5vS8X0nuSXCTeF2ucmCIiItlOq4CIiESEAl1EJCKyJtDNrMLMDppZlZktCLuesJnZQDPbZmZvmNl+M5sbdk2ZxMzyzOx1M/t/YdeSKeIX/K0zszfjPzf/NeyawmZmX4v//9lnZs+YWc+wa+qMrAj0hOUHJgFDgKlmNiTcqkJ3Fvhf7n4tcCMwR+/JeeYCb4RdRIb5N+AFd78GGE6Ovz9mNgB4ECh392E0Tvq4J9yqOicrAp2E5Qfc/TTQtPxAznL3PzctgObux2n8zzkg3Koyg5kVArcBK8KuJVOYWV9gLPBjAHc/7e7HQi0qM3QHeplZd6A3WX79TLYEemtLCwhgZkXACODfQy4lUzwOPAx8HHIdmeRzQD3wk/hQ1AozuzTsosLk7keAxcBhGpcp+cDdfxVuVZ2TLYGe0tICucjMLgOeA/6nu38Ydj1hM7PbgffcPcfvS3iB7sBIYKm7jwBOADn9WZSZXUHjX/rFwJXApWY2LdyqOidbAl1LCyRhZj1oDPPV7r4+7HoyxGjgDjOrpnFo7u/N7P+EW1JGqAVq3b3pr7h1NAZ8LrsZ+KO717v7GWA98PmQa+qUbAn0VJYfyCnxm4r8GHjD3f932PVkCnd/xN0L3b2Ixp+Tre6e1b2uILj7X4AaMxsc3zWB85fAzkWHgRvNrHf8/9MEsvyD4qy4BV1ryw+EXFbYRgPTgb1mFovv+yd33xReSZLh/hFYHe8UHSJ9S3RkBXf/dzNbB7xG46yx18nyJQB06b+ISERky5CLiIi0QYEuIhIRCnQRkYhQoIuIRIQCXUQkIhToIiIRoUAXEYmI/w9iflt0knsXdQAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
@@ -189,7 +193,7 @@
],
"source": [
"fig, ax = plt.subplots()\n",
- "ax.hist(just_mmseqs_filtered['normalized bit score'], bins=100, label='mmseqs', alpha=0.3, density=True);\n",
+ "ax.hist(mmseqs_filtered['normalized bit score'], bins=100, label='mmseqs', alpha=0.3, density=True);\n",
"ax.hist(hhblits_filtered['normalized bit score'], bins=100, label='hhblits', alpha=0.3, density=True);\n",
"ax.hist(structural_annotation['normalized bit score'], bins=100, label='coffe', alpha=0.3, density=True);\n",
"ax.legend()"
@@ -198,75 +202,184 @@
{
"cell_type": "code",
"execution_count": 12,
- "id": "efdcc8b4",
+ "id": "7649fb1a",
"metadata": {},
"outputs": [],
"source": [
"unique_up_id = pd.concat([hhblits_filtered['target'].drop_duplicates(),\n",
- " just_mmseqs_filtered['target'].drop_duplicates()])\n",
+ " mmseqs_filtered['target'].drop_duplicates()])\n",
"unique_up_id.drop_duplicates(inplace=True)"
]
},
{
"cell_type": "code",
- "execution_count": 13,
- "id": "e1fcdc85",
- "metadata": {},
- "outputs": [],
- "source": [
- "with open(f'../data/milot_unique_ids.txt', 'w') as tmp:\n",
- " ids_as_str = unique_up_id.astype(str).values\n",
- " tmp.write(','.join(ids_as_str))"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "ec480a86",
+ "execution_count": 22,
+ "id": "e8d0cfb7-c86d-41a6-99e7-d514bc1f3873",
"metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "148"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "<div class=\"alert alert-block alert-warning\"><b>DISCLAIMER:</b> To make this cell execute, you may need to copy the contents of the UPIMAPI/resource folder into the main repository directory.</div>"
+ "request_size = 500\n",
+ "no_chunks = np.ceil((len(unique_up_id) / request_size) * 3).astype(int)\n",
+ "no_chunks"
]
},
{
"cell_type": "code",
- "execution_count": 42,
- "id": "87d494b2",
- "metadata": {},
+ "execution_count": 25,
+ "id": "85231f72-12ad-4cc7-897a-544e0be27228",
+ "metadata": {
+ "tags": []
+ },
"outputs": [
{
- "name": "stdout",
+ "name": "stderr",
"output_type": "stream",
"text": [
- "2022-07-27 11:48:46: ID mapping has begun.\n",
- "Auto determined \"full id\" as: False\n",
- "../data/uniprotinfo.fasta not found. Will perform mapping for all IDs.\n",
- "Checking which IDs are missing information.\n",
- "Checking which IDs are missing information.: 100%|â–ˆ| 24540/24540 [00:00<00:00, 3\n",
- "Information already gathered for 0 ids. Still missing for 24540.\n",
- "Building FASTA from 24540 IDs.\n",
- "UniProt ID mapping: 0%| | 0/3 [00:00<?, ?it/s]\n",
- "Traceback (most recent call last):\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 1130, in <module>\n",
- " upimapi()\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 1125, in upimapi\n",
- " uniprot_fasta_workflow(ids, f'{args.output}/uniprotinfo.fasta', step=args.step, sleep_time=args.sleep)\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 309, in uniprot_fasta_workflow\n",
- " uniprotinfo = get_uniprot_fasta(ids_missing, step=step, sleep_time=sleep_time)\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 285, in get_uniprot_fasta\n",
- " data = uniprot_request(ids[i:j], original_database='ACC+ID', database_destination='', output_format='fasta')\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 219, in uniprot_request\n",
- " 'columns': string4mapping(columns=columns, databases=databases)\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 176, in string4mapping\n",
- " result = ','.join([columns_dict[column] for column in columns])\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 176, in <listcomp>\n",
- " result = ','.join([columns_dict[column] for column in columns])\n",
- "KeyError: 'Entry'\n"
+ "100%|██████████| 148/148 [01:19<00:00, 1.86it/s]\n"
]
}
],
"source": [
- "!python /g/arendt/npapadop/repos/UPIMAPI/upimapi.py -i ../data/milot_unique_ids.txt -o ../data/ --fasta --no-annotation"
+ "request_size = 500\n",
+ "no_chunks = np.ceil((len(unique_up_id) / request_size) * 3).astype(int)\n",
+ "\n",
+ "with open('../data/profile_sequences.fasta', 'w') as result:\n",
+ " for i in tqdm(range(no_chunks)):\n",
+ " a = (i * request_size) // 3\n",
+ " b = ((i+1) * request_size) // 3\n",
+ " chunk = unique_up_id[a:b]\n",
+ " bait50 = ['UniRef50_' + c for c in chunk]\n",
+ " bait90 = ['UniRef90_' + c for c in chunk]\n",
+ " bait100 = ['UniRef100_' + c for c in chunk]\n",
+ " expanded_chunk = bait50 + bait90 + bait100\n",
+ " url = f\"https://rest.uniprot.org/uniref/ids?ids={','.join(expanded_chunk)}&format=fasta\"\n",
+ " response = requests.get(url)\n",
+ " result.write(response.text)"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "da58c74f-5cd2-435a-8281-6195ed7f15af",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mmseqs_filtered.drop(columns=['index'], inplace=True)\n",
+ "hhblits_filtered.drop(columns=['index'], inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "eb0c48d8-ef71-440f-8ea0-6e7f3118f957",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "file = '/g/arendt/npapadop/data/spongfold_publish/'\n",
+ "sensitive = pd.read_csv(file, sep='\\t', skiprows=4, skipfooter=3, engine='python')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1c156c27-d2ec-47d6-b764-2b54816922a0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sensitive['uniprot'] = sensitive['#query'].str.split('|').str[1]\n",
+ "sensitive.reset_index(drop=True, inplace=True)\n",
+ "# remove unnecessary columns\n",
+ "dead_weight = ['#query', 'seed_ortholog', 'EC', 'KEGG_ko', 'KEGG_Pathway',\n",
+ " 'KEGG_Module', 'KEGG_Reaction', 'KEGG_rclass', 'BRITE',\n",
+ " 'KEGG_TC', 'CAZy', 'BiGG_Reaction']\n",
+ "sensitive.drop(dead_weight, axis=1, inplace=True)\n",
+ "# convert rest to categorical to save space\n",
+ "to_categorical = ['uniprot', 'eggNOG_OGs', 'max_annot_lvl', 'COG_category',\n",
+ " 'Description', 'Preferred_name', 'GOs', 'PFAMs']\n",
+ "sensitive[to_categorical] = sensitive[to_categorical].astype(\"category\")\n",
+ "# finally save in parquet format\n",
+ "sensitive.to_parquet('../data/uniprot_profiles.parquet')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "821c79ed-2090-45c8-bc5c-736ba1096989",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "response"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fd08dceb-c307-4ad6-bd23-91b70aaf7ef4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mmseqs_filtered[~mmseqs_filtered['target'].isin(sensitive['uniprot'])]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c0f35e75-1a3f-4bab-8afe-160d3347738f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mmseqs_filtered"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f99bd75f-ae5a-4d00-8a01-5588c711f45c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "foldseek_full = pd.read_parquet('../old_data/fs_targets.parquet')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a1359934-a10c-4301-9ffa-000acb4159e4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mmseqs_filtered = mmseqs_filtered.merge(sensitive, on='uniprot', how='left')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "72eed180-8cb0-4f76-b316-f5717f56b026",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n",
+ "hhblits = hhblits.merge(sensitive, on='uniprot', how='left')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e5b356e4-10f9-4782-a39c-4f19a221ebdb",
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
@@ -285,7 +398,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.6"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/analysis/read-write.ipynb b/analysis/read-write.ipynb
index 669250e37c33cc854de224080bc2122821a887b5..1a74c5b7d6d914b9fb487052445f6bf0618f3d27 100644
--- a/analysis/read-write.ipynb
+++ b/analysis/read-write.ipynb
@@ -526,118 +526,43 @@
]
},
{
- "cell_type": "code",
- "execution_count": 18,
- "id": "563af56b-7a1b-4c3a-be9c-8c79fe684c01",
+ "cell_type": "markdown",
+ "id": "c2343e57-f527-4634-87a9-cbd2bc8aee5f",
"metadata": {},
- "outputs": [],
"source": [
- "plddt = np.load('../data/spongilla_plddt.npz')\n",
- "\n",
- "sequence_info = pd.read_csv(\"../data/sequence_info.csv\")\n",
- "query_to_isoform = sequence_info[['query', 'isoform']].set_index('query').to_dict()['isoform']"
+ "For PDB the situation is more complicated. We will take the PDB IDs and translate them to UniProt accession numbers using the UniProt API. This will return an inflated list of IDs since some PDB entries contain complexes."
]
},
{
"cell_type": "code",
- "execution_count": 19,
- "id": "9496151e-6ac4-40ed-a5c1-e3fd1e84fe05",
+ "execution_count": 1,
+ "id": "5b403921-8514-431a-a34d-f92fbf034ab8",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "1470483it [27:00, 907.18it/s] \n"
+ "perl: warning: Setting locale failed.\n",
+ "perl: warning: Please check that your locale settings:\n",
+ "\tLANGUAGE = (unset),\n",
+ "\tLC_ALL = (unset),\n",
+ "\tLC_CTYPE = \"UTF-8\",\n",
+ "\tLC_TERMINAL = \"iTerm2\",\n",
+ "\tLANG = \"en_US.UTF-8\"\n",
+ " are supported and installed on your system.\n",
+ "perl: warning: Falling back to a fallback locale (\"en_US.UTF-8\").\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "../data/uniprotinfo.fasta: 7206 sequences, 4638410 bp => dividing into 1 part of <= 100000 sequences\n",
+ "All done, 0 seconds elapsed\n"
]
}
],
- "source": [
- "pdb['aligned_plddt'] = get_aligned_plddt(pdb, plddt, query_to_isoform)\n",
- "pdb.to_parquet('../data/pdb_tmp.parquet')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "85f5b721-880c-400a-a330-37a35503a740",
- "metadata": {},
- "outputs": [],
- "source": [
- "print(\"done\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "c2343e57-f527-4634-87a9-cbd2bc8aee5f",
- "metadata": {},
- "source": [
- "For PDB the situation is more complicated. We will take the PDB IDs and translate them to UniProt accession numbers using the UniProt API. This will return an inflated list of IDs since some PDB entries contain complexes."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "76ff0420-e724-4152-abbc-29d25bfd3f59",
- "metadata": {
- "tags": []
- },
- "outputs": [],
- "source": [
- "pdb = enrich_from_uniprot(pdb, \"target\", \"uniprot\", uniprot_from=\"PDB_ID\", uniprot_to=\"ACC\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "7262d562-3c2d-4a7a-9361-1bd197d30a7c",
- "metadata": {},
- "outputs": [],
- "source": [
- "unique_up_id = pd.concat([pdb['uniprot'].drop_duplicates(),\n",
- " afdb['uniprot'].drop_duplicates(),\n",
- " swp['uniprot'].drop_duplicates()])\n",
- "unique_up_id.drop_duplicates(inplace=True)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "df0fdae3-52e6-4e7a-87e8-c81401278b09",
- "metadata": {},
- "outputs": [],
- "source": [
- "unique_up_id.to_csv('../data/unique_uniprot_ids.csv', header=False, index=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "ec3015c3-316e-48fe-a08a-4fd4323cb9ea",
- "metadata": {},
- "outputs": [],
- "source": [
- "with open(f'../data/foldseek_unique_ids.txt', 'w') as tmp:\n",
- " ids_as_str = unique_up_id.astype(str)[0]\n",
- " tmp.write(','.join(ids_as_str))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "4650ac1f-4d11-4590-b39f-2a16468d808e",
- "metadata": {},
- "outputs": [],
- "source": [
- "!python /g/arendt/npapadop/repos/UPIMAPI/upimapi.py -i ../data/foldseek_unique_ids.txt -o ../data/ --fasta --no-annotation"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "5b403921-8514-431a-a34d-f92fbf034ab8",
- "metadata": {},
- "outputs": [],
"source": [
"%%bash\n",
"mkdir ../data/uniprot_fastas/ -p\n",
diff --git a/analysis/suppl-ModelArchive_metadata.ipynb b/analysis/suppl-ModelArchive_metadata.ipynb
index 5f7375199ec2e200f9d5f4ec160e0c8e14ab248f..f30f805665bd86d7c70271dc760178d496809305 100644
--- a/analysis/suppl-ModelArchive_metadata.ipynb
+++ b/analysis/suppl-ModelArchive_metadata.ipynb
@@ -3,14 +3,14 @@
{
"cell_type": "code",
"execution_count": 1,
- "id": "e951dd81-8a76-45fb-a00d-8de54697bf0f",
+ "id": "a121491b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "2022-06-08 15:01\n"
+ "2022-07-28 08:02\n"
]
}
],
@@ -28,7 +28,7 @@
},
{
"cell_type": "markdown",
- "id": "84d83807-5bdb-4fb7-8bcc-069c17f984a4",
+ "id": "308f25e4",
"metadata": {},
"source": [
"In this script I will prepare the metadata for deposition in [ModelArchive](https://www.modelarchive.org). I will use the lookup table produced by the CoFFE pipeline and concatenate all the possibly relevant information into a single column (description)."
@@ -37,7 +37,7 @@
{
"cell_type": "code",
"execution_count": 2,
- "id": "3bebc329-4921-4890-baac-5f3718a687d9",
+ "id": "9eb7d317",
"metadata": {},
"outputs": [],
"source": [
@@ -46,19 +46,39 @@
},
{
"cell_type": "code",
- "execution_count": 3,
- "id": "38b0ec79-e498-4ac0-ae63-9b1ce7ff44a1",
+ "execution_count": 4,
+ "id": "d7d72594",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "ename": "FileNotFoundError",
+ "evalue": "[Errno 2] No such file or directory: '../ol_data/Slacustris_eggnog.tsv'",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m/tmp/ipykernel_136/1368120052.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msequence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../ol_data/Slacustris_eggnog.tsv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'\\t'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mstructure\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_parquet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../old_data/structure_annotation.parquet'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0mstacklevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstacklevel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 310\u001b[0m )\n\u001b[0;32m--> 311\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[1;32m 584\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 585\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 587\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 588\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 481\u001b[0m \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 482\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 484\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 809\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 811\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 812\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 813\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m 1038\u001b[0m )\n\u001b[1;32m 1039\u001b[0m \u001b[0;31m# error: Too many arguments for \"ParserBase\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1040\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mmapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1041\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1042\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_failover_to_python\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/parsers/c_parser_wrapper.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[0;31m# open handles\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open_handles\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 52\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandles\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/parsers/base_parser.py\u001b[0m in \u001b[0;36m_open_handles\u001b[0;34m(self, src, kwds)\u001b[0m\n\u001b[1;32m 220\u001b[0m \u001b[0mLet\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mreaders\u001b[0m \u001b[0mopen\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[0mafter\u001b[0m \u001b[0mthey\u001b[0m \u001b[0mare\u001b[0m \u001b[0mdone\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtheir\u001b[0m \u001b[0mpotential\u001b[0m \u001b[0mraises\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 221\u001b[0m \"\"\"\n\u001b[0;32m--> 222\u001b[0;31m self.handles = get_handle(\n\u001b[0m\u001b[1;32m 223\u001b[0m \u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 224\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m 699\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 700\u001b[0m \u001b[0;31m# Encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 701\u001b[0;31m handle = open(\n\u001b[0m\u001b[1;32m 702\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 703\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../ol_data/Slacustris_eggnog.tsv'"
+ ]
+ }
+ ],
"source": [
- "sequence = pd.read_csv('../data/Slacustris_eggnog.tsv', sep='\\t')\n",
- "structure = pd.read_parquet('../data/structure_annotation.parquet')"
+ "sequence = pd.read_csv('../old_data/Slacustris_eggnog.tsv', sep='\\t')\n",
+ "structure = pd.read_parquet('../old_data/structure_annotation.parquet')"
]
},
{
"cell_type": "code",
- "execution_count": 4,
- "id": "a51fa323-ff59-4acf-abaa-6ddddfc8de5c",
+ "execution_count": null,
+ "id": "6532f2fc",
"metadata": {},
"outputs": [],
"source": [
@@ -67,14 +87,14 @@
},
{
"cell_type": "code",
- "execution_count": 5,
- "id": "fa874500-61c0-4410-ae55-b081e0912670",
+ "execution_count": null,
+ "id": "4a137f30",
"metadata": {},
"outputs": [],
"source": [
"def generate_description(row):\n",
- " desc = f\"Emapper preferred name [EggNOG]: {row['Preferred_name_seq']}\\n\" + \\\n",
- " f\"Emapper description [EggNOG]: {row['Description_seq']}\\n\" + \\\n",
+ " desc = f\"Preferred name [EggNOG]: {row['Preferred_name_seq']}\\n\" + \\\n",
+ " f\"Description [EggNOG]: {row['Description_seq']}\\n\" + \\\n",
" f\"CoFFE preferred name [EggNOG]: {row['Preferred_name_str']}\\n\" + \\\n",
" f\"CoFFE description [EggNOG]: {row['Description_str']}\\n\" + \\\n",
" f\"CoFFE UniProt function: {row['Function [CC]']}\\n\\n\" + \\\n",
@@ -93,19 +113,19 @@
" f\"CoFFE Emapper bit score [EggNOG]: {row['score_str']}\\n\" + \\\n",
" f\"CoFFE orthogroups [EggNOG]: {row['eggNOG_OGs_str']}\\n\" + \\\n",
" f\"CoFFE maximum annotation level [EggNOG]: {row['max_annot_lvl_str']}\\n\" + \\\n",
- " f\"CoFFE GO terms [EggNOG]: {row['GOs_str']}\\n\" + \\\n",
+ "# f\"CoFFE GO terms [EggNOG]: {row['GOs_str']}\\n\" + \\\n",
" f\"CoFFE PFAM domains [EggNOG]: {row['PFAMs_str']}\\n\\n\" + \\\n",
" f\"best sequence hit bit score [Emapper]: {row['score_seq']}\\n\" + \\\n",
- " f\"Emapper orthogroups [EggNOG]: {row['eggNOG_OGs_seq']}\\n\" + \\\n",
- " f\"Emapper maximum annotation level [EggNOG]: {row['max_annot_lvl_seq']}\\n\" + \\\n",
- " f\"Emapper GO terms [EggNOG]: {row['GOs_seq']}\\n\" + \\\n",
- " f\"Emapper PFAM domains [EggNOG]: {row['PFAMs_seq']}\\n\"\n",
+ " f\"Orthogroups [EggNOG]: {row['eggNOG_OGs_seq']}\\n\" + \\\n",
+ " f\"Maximum annotation level [EggNOG]: {row['max_annot_lvl_seq']}\\n\" + \\\n",
+ "# f\"Emapper GO terms [EggNOG]: {row['GOs_seq']}\\n\" + \\\n",
+ " f\"PFAM domains [EggNOG]: {row['PFAMs_seq']}\\n\"\n",
" return desc"
]
},
{
"cell_type": "markdown",
- "id": "2afdd7a7-1131-415d-aaa6-337220672d4d",
+ "id": "6712bc16",
"metadata": {},
"source": [
"Let's test it first:"
@@ -114,7 +134,7 @@
{
"cell_type": "code",
"execution_count": 6,
- "id": "63193db9-9f82-4126-80e3-ec76033ad1c6",
+ "id": "935a770c",
"metadata": {},
"outputs": [
{
@@ -163,7 +183,7 @@
},
{
"cell_type": "markdown",
- "id": "8097273d-2713-4597-9c94-7feb6f6280f9",
+ "id": "6c9e5b53",
"metadata": {},
"source": [
"Looks good! Now let's apply it to our table:"
@@ -172,7 +192,7 @@
{
"cell_type": "code",
"execution_count": 7,
- "id": "43e594b0-0a9b-4e37-b21a-a8a6cd8eda3c",
+ "id": "1c5133d7",
"metadata": {},
"outputs": [],
"source": [
@@ -181,7 +201,7 @@
},
{
"cell_type": "markdown",
- "id": "9c16526c-9a46-4abd-ac8f-25da62d23563",
+ "id": "34028222",
"metadata": {},
"source": [
"Subset the table, rename columns, and save:"
@@ -190,7 +210,7 @@
{
"cell_type": "code",
"execution_count": 8,
- "id": "97598d9d-34f8-4644-b1cd-a5be8d2c45dd",
+ "id": "fa278548",
"metadata": {},
"outputs": [],
"source": [
@@ -202,7 +222,7 @@
{
"cell_type": "code",
"execution_count": 9,
- "id": "3463393a-74ff-4239-bbee-8f240dc1902f",
+ "id": "1f718df1",
"metadata": {},
"outputs": [],
"source": [
@@ -226,7 +246,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.6"
+ "version": "3.8.13"
}
},
"nbformat": 4,
diff --git a/scripts/results_processing/README.md b/scripts/results_processing/README.md
new file mode 100755
index 0000000000000000000000000000000000000000..6d01cd19ad779b7a69658b342b24378a6e5c7a44
--- /dev/null
+++ b/scripts/results_processing/README.md
@@ -0,0 +1,29 @@
+# Read/write operations for result analysis
+
+The directory contains all scripts to rerun the processing of ColabFold and Foldseek results. The
+`python` files contain the code for processing; the corresponding `.sh` scripts are used to submit
+to the SLURM cluster.
+
+### io_parse_structures.py
+
+Will go through the predicted best models and extract the pLDDT score for each residue. It saves the
+per-residue-pLDDT for each protein in a dictionary (`npz` file) and the average pLDDT in a CSV
+table.
+
+### io_read_afdb.py
+
+Processes the Foldseek predictions against AlphaFold DB. Will return a table that lists all
+isoforms, their Foldseek scores, the average pLDDT of the aligned residues, and the UniProt ID of
+the target protein.
+
+### io_read_swp.py
+
+Processes the Foldseek predictions against SwissProt. Will return a table that lists all isoforms,
+their Foldseek scores, the average pLDDT of the aligned residues, and the UniProt ID of the target
+protein.
+
+### io_read_pdb.py
+
+Processes the Foldseek predictions against PDB. Will return a table that lists all isoforms, their
+Foldseek scores, the average pLDDT of the aligned residues, and the UniProt ID of the target
+protein. The UniProt ID is fetched by querying UniProt (UniProt API).
\ No newline at end of file
diff --git a/scripts/io_parse_structures.py b/scripts/results_processing/io_parse_structures.py
similarity index 100%
rename from scripts/io_parse_structures.py
rename to scripts/results_processing/io_parse_structures.py
diff --git a/scripts/io_parse_structures.sh b/scripts/results_processing/io_parse_structures.sh
similarity index 100%
rename from scripts/io_parse_structures.sh
rename to scripts/results_processing/io_parse_structures.sh
diff --git a/scripts/io_read_afdb.py b/scripts/results_processing/io_read_afdb.py
similarity index 84%
rename from scripts/io_read_afdb.py
rename to scripts/results_processing/io_read_afdb.py
index c3e197e44a5d207914ab810e1b066788ee2a6bce..57347ddb638afdf7b492d4dce3d4de31d14b45e6 100644
--- a/scripts/io_read_afdb.py
+++ b/scripts/results_processing/io_read_afdb.py
@@ -66,17 +66,9 @@ def enrich_from_uniprot(df, column_from, column_to, uniprot_from="PDB_ID", unipr
df_map = create_id_map(response, column_from, column_to)
return df.join(df_map, on=column_from)
-# MSA location
-msas = '/g/arendt/npapadop/repos/coffe/data/msa/'
+
# FOLDSEEK scores
-fs_pdb = '/g/arendt/npapadop/repos/coffe/data/pdb_score.tsv'
fs_afdb = '/g/arendt/npapadop/repos/coffe/data/afdb_score.tsv'
-fs_swp = '/g/arendt/npapadop/repos/coffe/data/swissprot_score.tsv'
-# AlphaFold predictions
-structure_list = '/g/arendt/npapadop/repos/coffe/data/named_models/'
-metadata = '/g/arendt/npapadop/repos/coffe/data/named_metadata/'
-# input fasta
-all_peptides = '/g/arendt/npapadop/repos/coffe/data/Spongilla_lacustris_translated_proteome.fasta'
plddt = np.load('/g/arendt/npapadop/repos/coffe/data/spongilla_plddt.npz')
@@ -88,4 +80,4 @@ afdb["query"] = afdb["query"].values.astype(int)
afdb['aligned_plddt'] = get_aligned_plddt(afdb, plddt, query_to_isoform)
afdb["uniprot"] = afdb["target"].str.split("-").str[1]
-afdb_best.to_parquet('/g/arendt/npapadop/repos/coffe/data/afdb_tmp.parquet')
\ No newline at end of file
+afdb.to_parquet('/g/arendt/npapadop/repos/coffe/data/afdb_tmp.parquet')
\ No newline at end of file
diff --git a/scripts/io_read_afdb.sh b/scripts/results_processing/io_read_afdb.sh
similarity index 60%
rename from scripts/io_read_afdb.sh
rename to scripts/results_processing/io_read_afdb.sh
index 23dfba3f9149b93d4c8dbbe8ee8a830f59e46e63..b9eed768fc13d3ca0c5986a53a2a135062f0ba40 100644
--- a/scripts/io_read_afdb.sh
+++ b/scripts/results_processing/io_read_afdb.sh
@@ -2,12 +2,12 @@
#SBATCH -J coffe_read_afdb
#SBATCH -t 5:00:00
#SBATCH -c 1
-#SBATCH --mem=16G
+#SBATCH --mem=32G
#SBATCH -o /g/arendt/npapadop/cluster/io_read_afdb.out
#SBATCH -e /g/arendt/npapadop/cluster/io_read_afdb.err
module load Anaconda3
source ~/.bash_profile
-conda activate /g/arendt/npapadop/repos/condas/milot
+conda activate /g/arendt/npapadop/repos/condas/CoFFE
-python /g/arendt/npapadop/repos/coffe/scripts/io_read_afdb.py
\ No newline at end of file
+python /g/arendt/npapadop/repos/coffe/scripts/results_processing/io_read_afdb.py
\ No newline at end of file
diff --git a/scripts/io_read_pdb.py b/scripts/results_processing/io_read_pdb.py
similarity index 81%
rename from scripts/io_read_pdb.py
rename to scripts/results_processing/io_read_pdb.py
index 556b717230a4de3e8f25baa1a27f816dd52a53f0..f1c8e3b0694daa40d4727078501b85dda9bb21ad 100644
--- a/scripts/io_read_pdb.py
+++ b/scripts/results_processing/io_read_pdb.py
@@ -1,6 +1,7 @@
import os
import json
from tqdm import tqdm
+from urllib import parse, request
import numpy as np
import pandas as pd
@@ -32,7 +33,7 @@ def get_aligned_plddt(df, plddt, name_dict):
def get_from_uniprot(df, column, uniprot_from="ACC", uniprot_to="EGGNOG_ID", request_size=40000):
ids = df[column][~df[column].isnull()]
- url = 'https://www.uniprot.org/uploadlists/'
+ url = 'https://rest.uniprot.org/'
params = {
'from': uniprot_from,
@@ -41,10 +42,10 @@ def get_from_uniprot(df, column, uniprot_from="ACC", uniprot_to="EGGNOG_ID", req
'query': " ".join(ids.unique())
}
- data = urllib.parse.urlencode(params)
+ data = parse.urlencode(params)
data = data.encode('utf-8')
- req = urllib.request.Request(url, data)
- with urllib.request.urlopen(req) as f:
+ req = request.Request(url, data)
+ with request.urlopen(req) as f:
response = f.read()
return response
@@ -66,18 +67,9 @@ def enrich_from_uniprot(df, column_from, column_to, uniprot_from="PDB_ID", unipr
df_map = create_id_map(response, column_from, column_to)
return df.join(df_map, on=column_from)
-# MSA location
-msas = '/g/arendt/npapadop/repos/coffe/data/msa/'
# FOLDSEEK scores
fs_pdb = '/g/arendt/npapadop/repos/coffe/data/pdb_score.tsv'
-fs_afdb = '/g/arendt/npapadop/repos/coffe/data/afdb_score.tsv'
-fs_swp = '/g/arendt/npapadop/repos/coffe/data/swissprot_score.tsv'
-# AlphaFold predictions
-structure_list = '/g/arendt/npapadop/repos/coffe/data/named_models/'
-metadata = '/g/arendt/npapadop/repos/coffe/data/named_metadata/'
-# input fasta
-all_peptides = '/g/arendt/npapadop/repos/coffe/data/Spongilla_lacustris_translated_proteome.fasta'
-
+# per-residue pLDDT score
plddt = np.load('/g/arendt/npapadop/repos/coffe/data/spongilla_plddt.npz')
sequence_info = pd.read_csv("/g/arendt/npapadop/repos/coffe/data/sequence_info.csv")
diff --git a/scripts/io_read_pdb.sh b/scripts/results_processing/io_read_pdb.sh
similarity index 65%
rename from scripts/io_read_pdb.sh
rename to scripts/results_processing/io_read_pdb.sh
index 63dbd2e8a30a2142f1911f7a3fc5745548a29ce2..800685e0773ba58c532a4029c6b4f7dc1f84b04a 100644
--- a/scripts/io_read_pdb.sh
+++ b/scripts/results_processing/io_read_pdb.sh
@@ -8,6 +8,6 @@
module load Anaconda3
source ~/.bash_profile
-conda activate /g/arendt/npapadop/repos/condas/milot
+conda activate /g/arendt/npapadop/repos/condas/CoFFE
-python /g/arendt/npapadop/repos/coffe/scripts/io_read_pdb.py
\ No newline at end of file
+python /g/arendt/npapadop/repos/coffe/scripts/results_processing/io_read_pdb.py
\ No newline at end of file
diff --git a/scripts/io_read_swp.py b/scripts/results_processing/io_read_swp.py
similarity index 97%
rename from scripts/io_read_swp.py
rename to scripts/results_processing/io_read_swp.py
index 7a7213d79153823e155ac6ae132813c2e55cb043..ec5dc0529f45f7823e9c98eca14722c7f85ede33 100644
--- a/scripts/io_read_swp.py
+++ b/scripts/results_processing/io_read_swp.py
@@ -88,4 +88,4 @@ swp["query"] = swp["query"].values.astype(int)
swp['aligned_plddt'] = get_aligned_plddt(swp, plddt, query_to_isoform)
swp["uniprot"] = swp["target"].str.split("-").str[1]
-swp_best.to_parquet('/g/arendt/npapadop/repos/coffe/data/swp_tmp.parquet')
\ No newline at end of file
+swp.to_parquet('/g/arendt/npapadop/repos/coffe/data/swp_tmp.parquet')
\ No newline at end of file
diff --git a/scripts/io_read_swp.sh b/scripts/results_processing/io_read_swp.sh
similarity index 65%
rename from scripts/io_read_swp.sh
rename to scripts/results_processing/io_read_swp.sh
index 947d670c8c273f2acbb4e568a9572c0a8cce5c2b..f665f7905baee7df8d782e2c8e874469a6120b8a 100644
--- a/scripts/io_read_swp.sh
+++ b/scripts/results_processing/io_read_swp.sh
@@ -8,6 +8,6 @@
module load Anaconda3
source ~/.bash_profile
-conda activate /g/arendt/npapadop/repos/condas/milot
+conda activate /g/arendt/npapadop/repos/condas/CoFFE
-python /g/arendt/npapadop/repos/coffe/scripts/io_read_swp.py
\ No newline at end of file
+python /g/arendt/npapadop/repos/coffe/scripts/results_processing/io_read_swp.py
\ No newline at end of file