From 8dd17e2e2a218c12da5802e1f6c1a45d1cf272fd Mon Sep 17 00:00:00 2001
From: Nikolaos Papadopoulos <npapadop@embl.de>
Date: Fri, 29 Jul 2022 14:41:13 +0200
Subject: [PATCH] temp saving of AFDB, SWP parquets now possible; minor file
structure changes
---
.gitignore | 3 +
analysis/Untitled.ipynb | 247 +++++++++++++-----
analysis/read-write.ipynb | 119 ++-------
analysis/suppl-ModelArchive_metadata.ipynb | 76 ++++--
scripts/results_processing/README.md | 29 ++
.../io_parse_structures.py | 0
.../io_parse_structures.sh | 0
.../{ => results_processing}/io_read_afdb.py | 12 +-
.../{ => results_processing}/io_read_afdb.sh | 6 +-
.../{ => results_processing}/io_read_pdb.py | 20 +-
.../{ => results_processing}/io_read_pdb.sh | 4 +-
.../{ => results_processing}/io_read_swp.py | 2 +-
.../{ => results_processing}/io_read_swp.sh | 4 +-
13 files changed, 298 insertions(+), 224 deletions(-)
create mode 100755 scripts/results_processing/README.md
rename scripts/{ => results_processing}/io_parse_structures.py (100%)
rename scripts/{ => results_processing}/io_parse_structures.sh (100%)
rename scripts/{ => results_processing}/io_read_afdb.py (84%)
rename scripts/{ => results_processing}/io_read_afdb.sh (60%)
rename scripts/{ => results_processing}/io_read_pdb.py (81%)
rename scripts/{ => results_processing}/io_read_pdb.sh (65%)
rename scripts/{ => results_processing}/io_read_swp.py (97%)
rename scripts/{ => results_processing}/io_read_swp.sh (65%)
diff --git a/.gitignore b/.gitignore
index 6ad5a19..4dc5d2b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+*.log
+
MANIFEST
build
dist
@@ -33,6 +35,7 @@ venv*/
# various auxiliary files
analysis/figures/*
+old_data/*
data/*
*.h5ad
*.tsv
diff --git a/analysis/Untitled.ipynb b/analysis/Untitled.ipynb
index 5ed3f7c..46e7623 100644
--- a/analysis/Untitled.ipynb
+++ b/analysis/Untitled.ipynb
@@ -3,14 +3,14 @@
{
"cell_type": "code",
"execution_count": 1,
- "id": "814d8de9",
+ "id": "273c6a8d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "2022-07-27 13:53\n"
+ "2022-07-29 14:01\n"
]
}
],
@@ -28,12 +28,14 @@
{
"cell_type": "code",
"execution_count": 2,
- "id": "3808ea96",
+ "id": "09defaef-09b9-4d8e-84a1-dc3c7a56b80a",
"metadata": {},
"outputs": [],
"source": [
"import glob\n",
"from os.path import exists\n",
+ "from tqdm import tqdm\n",
+ "import requests\n",
"\n",
"import pandas as pd\n",
"import numpy as np\n",
@@ -45,7 +47,7 @@
{
"cell_type": "code",
"execution_count": 3,
- "id": "ad6f9837",
+ "id": "8aaa0695",
"metadata": {},
"outputs": [],
"source": [
@@ -56,23 +58,23 @@
{
"cell_type": "code",
"execution_count": 4,
- "id": "1a0df0c0",
+ "id": "95457088",
"metadata": {},
"outputs": [],
"source": [
- "just_mmseqs = pd.read_csv('../data/cfres_mmseqs_s75_e1.m8', sep=\"\\s+\", header=None)\n",
- "just_mmseqs.columns = [\"query\", \"target\", \"seq. id.\", \"alignment length\", \"no. mismatches\",\n",
+ "mmseqs = pd.read_csv('../data/cfres_mmseqs_s75_e1.m8', sep=\"\\s+\", header=None)\n",
+ "mmseqs.columns = [\"query\", \"target\", \"seq. id.\", \"alignment length\", \"no. mismatches\",\n",
" \"no. gap open\", \"query start\", \"query end\", \"target start\", \"target end\",\n",
" \"e value\", \"bit score\"]\n",
"\n",
- "just_mmseqs['gene_id'] = just_mmseqs['query'].str.split('_').str[:2].apply(lambda x: '_'.join(x))\n",
- "just_mmseqs['normalized bit score'] = just_mmseqs['bit score'] / just_mmseqs['alignment length']"
+ "mmseqs['gene_id'] = mmseqs['query'].str.split('_').str[:2].apply(lambda x: '_'.join(x))\n",
+ "mmseqs['normalized bit score'] = mmseqs['bit score'] / mmseqs['alignment length']"
]
},
{
"cell_type": "code",
"execution_count": 5,
- "id": "b09c96b5",
+ "id": "093bff38",
"metadata": {},
"outputs": [],
"source": [
@@ -91,17 +93,18 @@
{
"cell_type": "code",
"execution_count": 6,
- "id": "ff6f8bf7",
+ "id": "042ff08e",
"metadata": {},
"outputs": [],
"source": [
- "just_mmseqs_filtered = keep_best(just_mmseqs)"
+ "mmseqs_filtered = keep_best(mmseqs)\n",
+ "del mmseqs"
]
},
{
"cell_type": "code",
"execution_count": 7,
- "id": "af68258b",
+ "id": "9645fec8",
"metadata": {},
"outputs": [],
"source": [
@@ -117,17 +120,18 @@
{
"cell_type": "code",
"execution_count": 8,
- "id": "5380700c",
+ "id": "7e9d0601",
"metadata": {},
"outputs": [],
"source": [
- "hhblits_filtered = keep_best(hhblits)"
+ "hhblits_filtered = keep_best(hhblits)\n",
+ "del hhblits"
]
},
{
"cell_type": "code",
"execution_count": 9,
- "id": "2d829347",
+ "id": "260c26ec",
"metadata": {},
"outputs": [
{
@@ -151,7 +155,7 @@
{
"cell_type": "code",
"execution_count": 10,
- "id": "b72335fe",
+ "id": "c92b4a01",
"metadata": {},
"outputs": [],
"source": [
@@ -161,13 +165,13 @@
{
"cell_type": "code",
"execution_count": 11,
- "id": "e7595d52",
+ "id": "5ceaaff2",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "<matplotlib.legend.Legend at 0x7fdd93502580>"
+ "<matplotlib.legend.Legend at 0x7f58ab679250>"
]
},
"execution_count": 11,
@@ -176,7 +180,7 @@
},
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZdUlEQVR4nO3df3BV5b3v8feXgAUUxJLcthJs0jOIAgkBg3oLF+GgGNCRudoZRUFOW0qZi0due0XwtMcWOu30jJxbD1WhDKWWuVSoSCn3CtpSsIK1hx+6G34oGmlKQlqNoShyoPzwe//ITs4m7JCdZO2svdf+vGYyZK317LW/e0/45Mmzn/Usc3dERCT7dQu7ABERCYYCXUQkIhToIiIRoUAXEYkIBbqISER0D+uJ8/PzvaioKKynFxHJSnv27Hnf3QuSHQst0IuKiti9e3dYTy8ikpXM7E+tHdOQi4hIRCjQRUQiQoEuIhIRoY2hi0juOHPmDLW1tZw6dSrsUrJGz549KSwspEePHik/RoEuImlXW1tLnz59KCoqwszCLifjuTsNDQ3U1tZSXFyc8uM05CIiaXfq1Cn69++vME+RmdG/f/92/0WjQBeRLqEwb5+OvF8KdBGRiNAYuoh0uS0H3g30fDcP+VSg58tW6qFfzMHNYVcgIpIyBbqI5ITq6mquueYaZs6cybBhw7jvvvvYsmULo0ePZtCgQezcuZNvf/vbzJgxg4kTJ1JUVMT69et5+OGHKSkpoaKigjNnzgCwYMEChgwZQmlpKQ899BAA9fX13HXXXYwaNYpRo0bxyiuvANDQ0MDEiRMZMWIEX/3qV/nsZz/L+++/z4kTJ7jtttsYPnw4w4YNY+3atZ1+jQp0EckZVVVVzJ07l8rKSt58801+9rOfsWPHDhYvXsz3vvc9AN555x2ef/55fvnLXzJt2jTGjx/P3r176dWrF88//zxHjx7lF7/4Bfv376eyspJvfvObAMydO5evfe1r7Nq1i+eee46ZM2cCsHDhQsaMGcPrr7/OHXfcweHDhwF44YUXuPLKK/nDH/7Avn37qKio6PTrU6CLSM4oLi6mpKSEbt26MXToUCZMmICZUVJSQnV1NQCTJk2iR48elJSUcO7cueagbWrTt29fevbsycyZM1m/fj29e/cGYMuWLTzwwAOUlZVxxx138OGHH3L8+HFefvllpk2bBsBtt93GFVdc0Xy+LVu2MH/+fLZv387ll1/e6denQL+IWM2xsEsQkQB94hOfaP6+W7duzdvdunXj7Nmz57Xp1q0bPXr0aJ4+2NSme/fu7Ny5k7vuuosNGzY0B/7HH3/Mq6++SiwWIxaLceTIEfr06QMkn4J49dVXs2fPHkpKSnjkkUdYtGhRp1+fAr01+kBURJL46KOP+OCDD5g8eTKPP/44sVgMgIkTJ/LEE080t2vaP3bsWFavXg3A5s2b+etf/wpAXV0dvXv3Ztq0aTz00EO89tprna5N0xZFpMtl8zTD48ePM2XKFE6dOoW784Mf/ACAJUuWMGfOHEpLSzl79ixjx45l2bJlfOtb32Lq1KmMHDmSm266iauuugqAvXv3Mm/evOa/BJYuXdrp2szdO32SjigvL/eMvsHFwc3Eao5RdvPUsCsRyXpvvPEG1157bdhlZISmm/vk5+e32TbZ+2Zme9y9PFn7NodczGylmb1nZvtaOX6fmVXGv35nZsPbrFJERAKXyhj608DF5tP8EbjJ3UuB7wDLA6hLRCSSqqurU+qdd0SbY+ju/rKZFV3k+O8SNn8PFAZQl4iItFPQs1y+DLQ6PcTMZpnZbjPbXV9fH/BTB0tTFkUk2wQW6GY2nsZAn99aG3df7u7l7l5eUFAQ1FN3maAXFBIRCVIg0xbNrBRYAUxy94YgzikiIu3T6UA3s6uA9cB0d3+r8yVlrvy6rTBE0xhFOi3oC/cGT2qzSXV1Nbfffjv79p0/YW/cuHEsXryY8vLzZwI+/fTT7N69+7yLhZpcdtllfPTRR9TV1fHggw+ybt06YrEYdXV1TJ48uXOvpRNSmbb4DPAqMNjMas3sy2Y228xmx5s8CvQHnjKzmJll8OTyTtCVoyLSwpVXXsm6deuAxitDN23aFGo9bQa6u09198+4ew93L3T3H7v7MndfFj8+092vcPey+FfSCe/Z5LyxcgW5SGScO3eOr3zlKwwdOpSJEydy8uRJAJ599lmuv/56rr76arZv397cvqamhoqKCgYPHszChQsvOF91dTXDhg3j9OnTPProo6xdu5aysjLWrl3Lb3/7W8rKyigrK2PEiBEcP3487a9Pa7k0aQrug5sbh1biNNtFJDrefvtt5syZw/79++nXrx/PPfccAGfPnmXnzp08/vjj5wX3zp07Wb16NbFYjGeffZbWrm6/5JJLWLRoEXfffTexWIy7776bxYsX8+STTxKLxdi+fTu9evVK++tToItIziguLqasrAyA6667rnnJ3DvvvPOCfQC33HIL/fv3p1evXtx5553s2LEj5ecaPXo0X//611myZAnHjh2je/f0L52lQBeRnJG4fG5eXt4FS+Ym7oMLl71NtgxuaxYsWMCKFSs4efIkN954I2+++WZnSk+JAl1EpBW//vWvOXr0KCdPnmTDhg2MHj261bZ9+vQ5b5z8nXfeoaSkhPnz51NeXt4lga7lc0Wk66UwzTATjBkzhunTp1NVVcW99957wdTGROPHj+f73/8+ZWVlPPLII+zYsYNt27aRl5fHkCFDmDQp/a9Zy+c2Obi58YcsvmxuorKbp2o5XZFO0PK5HRP48rnSSLNdRCTTKdBFRCJCgZ5IFxGJSBZToIuIRIQCXUQkIhToIiIRoXno7bTlwLvcPORTYZchktVeqnkp0PONGzgusHPV19dz++23c/r0aZYsWcJf/vIXHn30UT796U+zbdu2wJ4nHRToCWI1xygb2C/sMkQkRL/5zW+45ppr+OlPfwpARUUFTz31FOPHjw+5srZpyEVEcsKqVasoLS1l+PDhTJ8+nT/96U9MmDCB0tJSJkyYwOHDh4nFYjz88MNs2rSJsrIyFi5cyI4dO5g9ezbz5s3j3LlzzJs3j1GjRlFaWsqPfvSjsF/WedRD74imq0pFJCvs37+f7373u7zyyivk5+dz9OhRZsyYwf3338+MGTNYuXIlDz74IBs2bGDRokXn3alo27ZtzXc0Wr58OZdffjm7du3ib3/7G6NHj2bixIkUFxeH/AobqYcepytBRaJr69atfOELXyA/Px+AT37yk7z66qvce++9AEyfPj2lpXF/9atfsWrVKsrKyrjhhhtoaGjg7bffTmvt7aEeejvl120FjbOLZBV3b3Pp21SWxnV3fvjDH3LrrbcGVVqg1EMXkcibMGECP//5z2loaADg6NGjfP7zn2fNmjUArF69mjFjxrR5nltvvZWlS5dy5swZAN566y1OnDiRvsLbST30FMS2PBN2CSKREuQ0w1QMHTqUb3zjG9x0003k5eUxYsQIlixZwpe+9CUee+wxCgoK+MlPftLmeWbOnEl1dTUjR47E3SkoKGDDhg3pfwEp0vK5cU2hXTawX5vj6WUD++lDUZF20PK5HaPlc0VEcpQCXUQkIhToItIlwhrezVYdeb/aDHQzW2lm75nZvlaOm5ktMbMqM6s0s5HtriKLbTnwbtgliGS8nj170tDQoFBPkbvT0NBAz5492/W4VGa5PA08Aaxq5fgkYFD86wZgafxfEREACgsLqa2tpb6+PuxSskbPnj0pLCxs12PaDHR3f9nMii7SZAqwyht/9f7ezPqZ2Wfc/c/tqkREIqtHjx4Zc3l8lAUxhj4AqEnYro3vu4CZzTKz3Wa2O1N/U2sJABHJVkEEerLrZZMOlLn7cncvd/fygoKCAJ5aRESaBBHotcDAhO1CoC6A83aZznywmV+3NcBKREQ6LohA3wjcH5/tciPwQc6Mnx/cHHYFIiLN2vxQ1MyeAcYB+WZWC3wL6AHg7suATcBkoAr4D+CL6So2LQ5uJr/uWNhViIh0WiqzXKa2cdyBOYFVJCIiHaIrRUVEIkKB3gma4igimUSB3gEKchHJRAp0EZGIUKCLiESEAl1EJCJyPtA1Hi4iUZHzgS4iEhUKdBGRiFCgB0FruohIBlCgd5BuPScimUaBLiISEQp0EZGIUKCLiESEAl1EJCIU6CIiEaFAD4CuNhWRTKBAFxGJCAW6iEhE5Hagd+IKz/y6rQEWIiLSebkd6AHSlaMiEjYFuohIROR0oGt2iohESUqBbmYVZnbQzKrMbEGS45eb2f81sz+Y2X4z+2LwpYqIyMW0Gehmlgc8CUwChgBTzWxIi2ZzgAPuPhwYB/yrmV0ScK0iInIRqfTQrweq3P2Qu58G1gBTWrRxoI+ZGXAZcBQ4G2ilIiJyUakE+gCgJmG7Nr4v0RPAtUAdsBeY6+4fB1JhmmhWiohETSqBbkn2eYvtW4EYcCVQBjxhZn0vOJHZLDPbbWa76+vr21lqsDSPXESiJpVArwUGJmwX0tgTT/RFYL03qgL+CFzT8kTuvtzdy929vKCgoKM1Zy7dik5EQpRKoO8CBplZcfyDznuAjS3aHAYmAJjZp4DBwKEgCxURkYvr3lYDdz9rZg8ALwJ5wEp3329ms+PHlwHfAZ42s700DtHMd/f301h3xsmv2woD+4VdhojksDYDHcDdNwGbWuxblvB9HTAx2NJERKQ9cvpK0aDpylMRCZMCXUQkIhToIiIRoUBPB01fFJEQ5GSg6ypREYminAx0EZEoUqAHTL1/EQmLAl1EJCIU6CIiEaFAFxGJCAW6iEhEKNBFRCJCgS4iEhEKdBGRiFCgi4hEhAJdRCQiFOgiIhGRc4G+5cC7jbeLSyPd6EJEwpBzgS4iElUKdBGRiFCgi4hEhAI9YOkenxcRaY0CXUQkIhToIiIRkVKgm1mFmR00syozW9BKm3FmFjOz/Wb222DLDI6GREQkqrq31cDM8oAngVuAWmCXmW109wMJbfoBTwEV7n7YzP5LmuoVEZFWpNJDvx6ocvdD7n4aWANMadHmXmC9ux8GcPf3gi0z++jeoiLS1VIJ9AFATcJ2bXxfoquBK8zsJTPbY2b3JzuRmc0ys91mtru+vr5jFYuISFKpBLol2ecttrsD1wG3AbcC/2xmV1/wIPfl7l7u7uUFBQXtLrbTDm7usqfSWL2IdLU2x9Bp7JEPTNguBOqStHnf3U8AJ8zsZWA48FYgVYqISJtS6aHvAgaZWbGZXQLcA2xs0eaXwH8zs+5m1hu4AXgj2FJFRORi2uyhu/tZM3sAeBHIA1a6+34zmx0/vszd3zCzF4BK4GNghbvvS2fhIiJyvlSGXHD3TcCmFvuWtdh+DHgsuNKCp2VtRSTKdKWoiEhEKNBFRCJCgZ5uXThVUkRymwI9nQ5u1ri9iHQZBbqISEQo0NNIvXMR6UoKdBGRiEhpHnqUvXby7aT7R/Ya1MWViIh0jnroIiIRoUAXEYmInBxyaW2YRUQkm6mHLiISEQr0rqCrRUWkCyjQRUQiQoEuIhIRCvQuoCtGRaQrKNBFRCIiZ6YtbjnwLvntaJ84tVFXjYpINlAPXUQkIhToXUnTF0UkjXIm0PPrtoZdgohIWuVMoIuIRJ0CvatouEVE0ixnZrlkwoJcsZpjlA0OuwoRiaqUeuhmVmFmB82syswWXKTdKDM7Z2ZfCK7EAKh3LCI5oM0eupnlAU8CtwC1wC4z2+juB5K0+xfgxXQUGqYg5qTralERSbdUeujXA1XufsjdTwNrgClJ2v0j8BzwXoD1BUJhKiK5IJVAHwDUJGzXxvc1M7MBwH8Hll3sRGY2y8x2m9nu+vr69tYqIiIXkUqgW5J93mL7cWC+u5+72Incfbm7l7t7eUFBQYoliohIKlKZ5VILDEzYLgTqWrQpB9aYGUA+MNnMzrr7hiCKFBGRtqXSQ98FDDKzYjO7BLgH2JjYwN2L3b3I3YuAdcD/UJgnt+XAu2GXICIR1WYP3d3PmtkDNM5eyQNWuvt+M5sdP37RcXMREekaKV1Y5O6bgE0t9iUNcnf/h86XFXEHN8PgSWFXISIRo0v/Q6BplCKSDpEP9NiWZ8IuQUSkS0Q+0EVEcoUCXUQkIhToIiIRkTPL5walswt16c5JIpIu6qGHRUv6ikjAFOgiIhER6SGXl2pe4lAG3KlIRKQrqIcuIhIRCnQRkYiI9JBLrOYYfdN4/iBuTSciEhT10EVEIiLSgd63oTLsElqlBbpEJGiRDnQRkVyiQBcRiYjIBno23OotG2oUkewR2UDPFgp1EQmKAl1EJCIU6AF57eTbzV/tkbj6onrrItIZkb2wKL9uK4fDLiJVzSsvjgy1DBHJbuqhZwDNSReRICjQM4RCXUQ6K6VAN7MKMztoZlVmtiDJ8fvMrDL+9TszGx58qdkj1fF03b1IRILUZqCbWR7wJDAJGAJMNbMhLZr9EbjJ3UuB7wDLgy40FyjgRaQzUumhXw9Uufshdz8NrAGmJDZw99+5+1/jm78HCoMtU0RE2pLKLJcBQE3Cdi1ww0XafxlIesNMM5sFzAK46qqrUiyxfV6qeQkgY+5U1KEldg9uhsGT0lSRiERVKj10S7LPkzY0G09joM9Pdtzdl7t7ubuXFxQUpF5lDtFcdBHpqFQCvRYYmLBdCNS1bGRmpcAKYIq7NwRTXm7SjBcR6YhUhlx2AYPMrBg4AtwD3JvYwMyuAtYD0939rcCrjIjWZr3obkciEoQ2A93dz5rZA8CLQB6w0t33m9ns+PFlwKNAf+ApMwM46+7l6StbRERaSunSf3ffBGxqsW9ZwvczgZnBlpY7zvvgNGEwq+kDXoBxA8d1WT0ikp0iu5ZLtmoK92M1L523P5VwT2zTUuJjWmunXxoi2S2SgR6rOUbfsItIo4703C8W9kG0b+0XRnt/+eiXikjHRTLQc0l7gzdd50rlsUHWKiIXilyga8pfZlGIi3SdyK222LehMuwSRERCEbkeOkQn1A/FXuZzZWPDLqNLpTK2rtk/IslFMtAj4ciesCvIKK0FvcJd5D8p0DPUofoTYZeQ1RT0kosU6BJ5qc7PF8l2WR/omkUhTfSzILku6wM98o7sgQHXhV1FTtAwjWS7aAV6BD9IPFR/gs+hUO9qCnfJRtEK9Ig6VH+CDz8+RtnAfmGXEjmducJVQS+ZRoGeJfo2VEK3S9VTz1Dq0UsmiFSgR32q36H6E3xuQNhVSBPNjZdME6lAzwWxGg29ZBOFu3SlyAT6odjLYZfQJfo2VMLA3FoOIIo0Li/pEJlAzyXqpWcnfQAr6aZAF8kyGsaR1ijQs1CyGS/qtUebPoCVVEQj0CN4QVG7HNkD/F3YVUjIFO6S9YEeqzlG34ZoT1dMpuliI4CyyN2mRDorlfF6hX70ZH2gR+VmFh3Rt6GSD/uXNn/PwLEaepGUXaxHrw9ns1PWB7okiA+95OKdjqRzUl2pUkGf2VIKdDOrAP4NyANWuPv3Wxy3+PHJwH8A/+DurwVcazMtk/qf+jZUQsGlQPxK2f6N+5tulq3eunSF1nr77d0vndNmoJtZHvAkcAtQC+wys43ufiCh2SRgUPzrBmBp/N+0idUco6zbO+l8iqyRbMmD5qGobpcS+/jvGoP9yJ7Gsff4MI3CXtIhlRk5qexX0LdfKj3064Eqdz8EYGZrgClAYqBPAVa5uwO/N7N+ZvYZd/9z4BXH9W2o5FC6Tp7FWn6mcKj+BH2p5FDDhW0ONcCH/UsV7JKRgvpLvLW/DFprk81SCfQBQE3Cdi0X9r6TtRkAnBfoZjYLmBXf/MjMDrarWsgH3m/nY3KB3pfk9L5cSO9Jctn0vny2tQOpBLol2ecdaIO7LweWp/CcyQsx2+3u5R19fFTpfUlO78uF9J4kF5X3JZUZzLXAwITtQqCuA21ERCSNUgn0XcAgMys2s0uAe4CNLdpsBO63RjcCH6Rz/FxERC7U5pCLu581sweAF2mctrjS3feb2ez48WXAJhqnLFbROG3xi2mqt8PDNRGn9yU5vS8X0nuSXCTeF2ucmCIiItlOq4CIiESEAl1EJCKyJtDNrMLMDppZlZktCLuesJnZQDPbZmZvmNl+M5sbdk2ZxMzyzOx1M/t/YdeSKeIX/K0zszfjPzf/NeyawmZmX4v//9lnZs+YWc+wa+qMrAj0hOUHJgFDgKlmNiTcqkJ3Fvhf7n4tcCMwR+/JeeYCb4RdRIb5N+AFd78GGE6Ovz9mNgB4ECh392E0Tvq4J9yqOicrAp2E5Qfc/TTQtPxAznL3PzctgObux2n8zzkg3Koyg5kVArcBK8KuJVOYWV9gLPBjAHc/7e7HQi0qM3QHeplZd6A3WX79TLYEemtLCwhgZkXACODfQy4lUzwOPAx8HHIdmeRzQD3wk/hQ1AozuzTsosLk7keAxcBhGpcp+cDdfxVuVZ2TLYGe0tICucjMLgOeA/6nu38Ydj1hM7PbgffcPcfvS3iB7sBIYKm7jwBOADn9WZSZXUHjX/rFwJXApWY2LdyqOidbAl1LCyRhZj1oDPPV7r4+7HoyxGjgDjOrpnFo7u/N7P+EW1JGqAVq3b3pr7h1NAZ8LrsZ+KO717v7GWA98PmQa+qUbAn0VJYfyCnxm4r8GHjD3f932PVkCnd/xN0L3b2Ixp+Tre6e1b2uILj7X4AaMxsc3zWB85fAzkWHgRvNrHf8/9MEsvyD4qy4BV1ryw+EXFbYRgPTgb1mFovv+yd33xReSZLh/hFYHe8UHSJ9S3RkBXf/dzNbB7xG46yx18nyJQB06b+ISERky5CLiIi0QYEuIhIRCnQRkYhQoIuIRIQCXUQkIhToIiIRoUAXEYmI/w9iflt0knsXdQAAAABJRU5ErkJggg==\n",
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZdUlEQVR4nO3df3BV5b3v8feXgAUUxJLcthJs0jOIAgkBg3oLF+GgGNCRudoZRUFOW0qZi0due0XwtMcWOu30jJxbD1WhDKWWuVSoSCn3CtpSsIK1hx+6G34oGmlKQlqNoShyoPzwe//ITs4m7JCdZO2svdf+vGYyZK317LW/e0/45Mmzn/Usc3dERCT7dQu7ABERCYYCXUQkIhToIiIRoUAXEYkIBbqISER0D+uJ8/PzvaioKKynFxHJSnv27Hnf3QuSHQst0IuKiti9e3dYTy8ikpXM7E+tHdOQi4hIRCjQRUQiQoEuIhIRoY2hi0juOHPmDLW1tZw6dSrsUrJGz549KSwspEePHik/RoEuImlXW1tLnz59KCoqwszCLifjuTsNDQ3U1tZSXFyc8uM05CIiaXfq1Cn69++vME+RmdG/f/92/0WjQBeRLqEwb5+OvF8KdBGRiNAYuoh0uS0H3g30fDcP+VSg58tW6qFfzMHNYVcgIpIyBbqI5ITq6mquueYaZs6cybBhw7jvvvvYsmULo0ePZtCgQezcuZNvf/vbzJgxg4kTJ1JUVMT69et5+OGHKSkpoaKigjNnzgCwYMEChgwZQmlpKQ899BAA9fX13HXXXYwaNYpRo0bxyiuvANDQ0MDEiRMZMWIEX/3qV/nsZz/L+++/z4kTJ7jtttsYPnw4w4YNY+3atZ1+jQp0EckZVVVVzJ07l8rKSt58801+9rOfsWPHDhYvXsz3vvc9AN555x2ef/55fvnLXzJt2jTGjx/P3r176dWrF88//zxHjx7lF7/4Bfv376eyspJvfvObAMydO5evfe1r7Nq1i+eee46ZM2cCsHDhQsaMGcPrr7/OHXfcweHDhwF44YUXuPLKK/nDH/7Avn37qKio6PTrU6CLSM4oLi6mpKSEbt26MXToUCZMmICZUVJSQnV1NQCTJk2iR48elJSUcO7cueagbWrTt29fevbsycyZM1m/fj29e/cGYMuWLTzwwAOUlZVxxx138OGHH3L8+HFefvllpk2bBsBtt93GFVdc0Xy+LVu2MH/+fLZv387ll1/e6denQL+IWM2xsEsQkQB94hOfaP6+W7duzdvdunXj7Nmz57Xp1q0bPXr0aJ4+2NSme/fu7Ny5k7vuuosNGzY0B/7HH3/Mq6++SiwWIxaLceTIEfr06QMkn4J49dVXs2fPHkpKSnjkkUdYtGhRp1+fAr01+kBURJL46KOP+OCDD5g8eTKPP/44sVgMgIkTJ/LEE080t2vaP3bsWFavXg3A5s2b+etf/wpAXV0dvXv3Ztq0aTz00EO89tprna5N0xZFpMtl8zTD48ePM2XKFE6dOoW784Mf/ACAJUuWMGfOHEpLSzl79ixjx45l2bJlfOtb32Lq1KmMHDmSm266iauuugqAvXv3Mm/evOa/BJYuXdrp2szdO32SjigvL/eMvsHFwc3Eao5RdvPUsCsRyXpvvPEG1157bdhlZISmm/vk5+e32TbZ+2Zme9y9PFn7NodczGylmb1nZvtaOX6fmVXGv35nZsPbrFJERAKXyhj608DF5tP8EbjJ3UuB7wDLA6hLRCSSqqurU+qdd0SbY+ju/rKZFV3k+O8SNn8PFAZQl4iItFPQs1y+DLQ6PcTMZpnZbjPbXV9fH/BTB0tTFkUk2wQW6GY2nsZAn99aG3df7u7l7l5eUFAQ1FN3maAXFBIRCVIg0xbNrBRYAUxy94YgzikiIu3T6UA3s6uA9cB0d3+r8yVlrvy6rTBE0xhFOi3oC/cGT2qzSXV1Nbfffjv79p0/YW/cuHEsXryY8vLzZwI+/fTT7N69+7yLhZpcdtllfPTRR9TV1fHggw+ybt06YrEYdXV1TJ48uXOvpRNSmbb4DPAqMNjMas3sy2Y228xmx5s8CvQHnjKzmJll8OTyTtCVoyLSwpVXXsm6deuAxitDN23aFGo9bQa6u09198+4ew93L3T3H7v7MndfFj8+092vcPey+FfSCe/Z5LyxcgW5SGScO3eOr3zlKwwdOpSJEydy8uRJAJ599lmuv/56rr76arZv397cvqamhoqKCgYPHszChQsvOF91dTXDhg3j9OnTPProo6xdu5aysjLWrl3Lb3/7W8rKyigrK2PEiBEcP3487a9Pa7k0aQrug5sbh1biNNtFJDrefvtt5syZw/79++nXrx/PPfccAGfPnmXnzp08/vjj5wX3zp07Wb16NbFYjGeffZbWrm6/5JJLWLRoEXfffTexWIy7776bxYsX8+STTxKLxdi+fTu9evVK++tToItIziguLqasrAyA6667rnnJ3DvvvPOCfQC33HIL/fv3p1evXtx5553s2LEj5ecaPXo0X//611myZAnHjh2je/f0L52lQBeRnJG4fG5eXt4FS+Ym7oMLl71NtgxuaxYsWMCKFSs4efIkN954I2+++WZnSk+JAl1EpBW//vWvOXr0KCdPnmTDhg2MHj261bZ9+vQ5b5z8nXfeoaSkhPnz51NeXt4lga7lc0Wk66UwzTATjBkzhunTp1NVVcW99957wdTGROPHj+f73/8+ZWVlPPLII+zYsYNt27aRl5fHkCFDmDQp/a9Zy+c2Obi58YcsvmxuorKbp2o5XZFO0PK5HRP48rnSSLNdRCTTKdBFRCJCgZ5IFxGJSBZToIuIRIQCXUQkIhToIiIRoXno7bTlwLvcPORTYZchktVeqnkp0PONGzgusHPV19dz++23c/r0aZYsWcJf/vIXHn30UT796U+zbdu2wJ4nHRToCWI1xygb2C/sMkQkRL/5zW+45ppr+OlPfwpARUUFTz31FOPHjw+5srZpyEVEcsKqVasoLS1l+PDhTJ8+nT/96U9MmDCB0tJSJkyYwOHDh4nFYjz88MNs2rSJsrIyFi5cyI4dO5g9ezbz5s3j3LlzzJs3j1GjRlFaWsqPfvSjsF/WedRD74imq0pFJCvs37+f7373u7zyyivk5+dz9OhRZsyYwf3338+MGTNYuXIlDz74IBs2bGDRokXn3alo27ZtzXc0Wr58OZdffjm7du3ib3/7G6NHj2bixIkUFxeH/AobqYcepytBRaJr69atfOELXyA/Px+AT37yk7z66qvce++9AEyfPj2lpXF/9atfsWrVKsrKyrjhhhtoaGjg7bffTmvt7aEeejvl120FjbOLZBV3b3Pp21SWxnV3fvjDH3LrrbcGVVqg1EMXkcibMGECP//5z2loaADg6NGjfP7zn2fNmjUArF69mjFjxrR5nltvvZWlS5dy5swZAN566y1OnDiRvsLbST30FMS2PBN2CSKREuQ0w1QMHTqUb3zjG9x0003k5eUxYsQIlixZwpe+9CUee+wxCgoK+MlPftLmeWbOnEl1dTUjR47E3SkoKGDDhg3pfwEp0vK5cU2hXTawX5vj6WUD++lDUZF20PK5HaPlc0VEcpQCXUQkIhToItIlwhrezVYdeb/aDHQzW2lm75nZvlaOm5ktMbMqM6s0s5HtriKLbTnwbtgliGS8nj170tDQoFBPkbvT0NBAz5492/W4VGa5PA08Aaxq5fgkYFD86wZgafxfEREACgsLqa2tpb6+PuxSskbPnj0pLCxs12PaDHR3f9nMii7SZAqwyht/9f7ezPqZ2Wfc/c/tqkREIqtHjx4Zc3l8lAUxhj4AqEnYro3vu4CZzTKz3Wa2O1N/U2sJABHJVkEEerLrZZMOlLn7cncvd/fygoKCAJ5aRESaBBHotcDAhO1CoC6A83aZznywmV+3NcBKREQ6LohA3wjcH5/tciPwQc6Mnx/cHHYFIiLN2vxQ1MyeAcYB+WZWC3wL6AHg7suATcBkoAr4D+CL6So2LQ5uJr/uWNhViIh0WiqzXKa2cdyBOYFVJCIiHaIrRUVEIkKB3gma4igimUSB3gEKchHJRAp0EZGIUKCLiESEAl1EJCJyPtA1Hi4iUZHzgS4iEhUKdBGRiFCgB0FruohIBlCgd5BuPScimUaBLiISEQp0EZGIUKCLiESEAl1EJCIU6CIiEaFAD4CuNhWRTKBAFxGJCAW6iEhE5Hagd+IKz/y6rQEWIiLSebkd6AHSlaMiEjYFuohIROR0oGt2iohESUqBbmYVZnbQzKrMbEGS45eb2f81sz+Y2X4z+2LwpYqIyMW0Gehmlgc8CUwChgBTzWxIi2ZzgAPuPhwYB/yrmV0ScK0iInIRqfTQrweq3P2Qu58G1gBTWrRxoI+ZGXAZcBQ4G2ilIiJyUakE+gCgJmG7Nr4v0RPAtUAdsBeY6+4fB1JhmmhWiohETSqBbkn2eYvtW4EYcCVQBjxhZn0vOJHZLDPbbWa76+vr21lqsDSPXESiJpVArwUGJmwX0tgTT/RFYL03qgL+CFzT8kTuvtzdy929vKCgoKM1Zy7dik5EQpRKoO8CBplZcfyDznuAjS3aHAYmAJjZp4DBwKEgCxURkYvr3lYDdz9rZg8ALwJ5wEp3329ms+PHlwHfAZ42s700DtHMd/f301h3xsmv2woD+4VdhojksDYDHcDdNwGbWuxblvB9HTAx2NJERKQ9cvpK0aDpylMRCZMCXUQkIhToIiIRoUBPB01fFJEQ5GSg6ypREYminAx0EZEoUqAHTL1/EQmLAl1EJCIU6CIiEaFAFxGJCAW6iEhEKNBFRCJCgS4iEhEKdBGRiFCgi4hEhAJdRCQiFOgiIhGRc4G+5cC7jbeLSyPd6EJEwpBzgS4iElUKdBGRiFCgi4hEhAI9YOkenxcRaY0CXUQkIhToIiIRkVKgm1mFmR00syozW9BKm3FmFjOz/Wb222DLDI6GREQkqrq31cDM8oAngVuAWmCXmW109wMJbfoBTwEV7n7YzP5LmuoVEZFWpNJDvx6ocvdD7n4aWANMadHmXmC9ux8GcPf3gi0z++jeoiLS1VIJ9AFATcJ2bXxfoquBK8zsJTPbY2b3JzuRmc0ys91mtru+vr5jFYuISFKpBLol2ecttrsD1wG3AbcC/2xmV1/wIPfl7l7u7uUFBQXtLrbTDm7usqfSWL2IdLU2x9Bp7JEPTNguBOqStHnf3U8AJ8zsZWA48FYgVYqISJtS6aHvAgaZWbGZXQLcA2xs0eaXwH8zs+5m1hu4AXgj2FJFRORi2uyhu/tZM3sAeBHIA1a6+34zmx0/vszd3zCzF4BK4GNghbvvS2fhIiJyvlSGXHD3TcCmFvuWtdh+DHgsuNKCp2VtRSTKdKWoiEhEKNBFRCJCgZ5uXThVUkRymwI9nQ5u1ri9iHQZBbqISEQo0NNIvXMR6UoKdBGRiEhpHnqUvXby7aT7R/Ya1MWViIh0jnroIiIRoUAXEYmInBxyaW2YRUQkm6mHLiISEQr0rqCrRUWkCyjQRUQiQoEuIhIRCvQuoCtGRaQrKNBFRCIiZ6YtbjnwLvntaJ84tVFXjYpINlAPXUQkIhToXUnTF0UkjXIm0PPrtoZdgohIWuVMoIuIRJ0CvatouEVE0ixnZrlkwoJcsZpjlA0OuwoRiaqUeuhmVmFmB82syswWXKTdKDM7Z2ZfCK7EAKh3LCI5oM0eupnlAU8CtwC1wC4z2+juB5K0+xfgxXQUGqYg5qTralERSbdUeujXA1XufsjdTwNrgClJ2v0j8BzwXoD1BUJhKiK5IJVAHwDUJGzXxvc1M7MBwH8Hll3sRGY2y8x2m9nu+vr69tYqIiIXkUqgW5J93mL7cWC+u5+72Incfbm7l7t7eUFBQYoliohIKlKZ5VILDEzYLgTqWrQpB9aYGUA+MNnMzrr7hiCKFBGRtqXSQ98FDDKzYjO7BLgH2JjYwN2L3b3I3YuAdcD/UJgnt+XAu2GXICIR1WYP3d3PmtkDNM5eyQNWuvt+M5sdP37RcXMREekaKV1Y5O6bgE0t9iUNcnf/h86XFXEHN8PgSWFXISIRo0v/Q6BplCKSDpEP9NiWZ8IuQUSkS0Q+0EVEcoUCXUQkIhToIiIRkTPL5walswt16c5JIpIu6qGHRUv6ikjAFOgiIhER6SGXl2pe4lAG3KlIRKQrqIcuIhIRCnQRkYiI9JBLrOYYfdN4/iBuTSciEhT10EVEIiLSgd63oTLsElqlBbpEJGiRDnQRkVyiQBcRiYjIBno23OotG2oUkewR2UDPFgp1EQmKAl1EJCIU6AF57eTbzV/tkbj6onrrItIZkb2wKL9uK4fDLiJVzSsvjgy1DBHJbuqhZwDNSReRICjQM4RCXUQ6K6VAN7MKMztoZlVmtiDJ8fvMrDL+9TszGx58qdkj1fF03b1IRILUZqCbWR7wJDAJGAJMNbMhLZr9EbjJ3UuB7wDLgy40FyjgRaQzUumhXw9Uufshdz8NrAGmJDZw99+5+1/jm78HCoMtU0RE2pLKLJcBQE3Cdi1ww0XafxlIesNMM5sFzAK46qqrUiyxfV6qeQkgY+5U1KEldg9uhsGT0lSRiERVKj10S7LPkzY0G09joM9Pdtzdl7t7ubuXFxQUpF5lDtFcdBHpqFQCvRYYmLBdCNS1bGRmpcAKYIq7NwRTXm7SjBcR6YhUhlx2AYPMrBg4AtwD3JvYwMyuAtYD0939rcCrjIjWZr3obkciEoQ2A93dz5rZA8CLQB6w0t33m9ns+PFlwKNAf+ApMwM46+7l6StbRERaSunSf3ffBGxqsW9ZwvczgZnBlpY7zvvgNGEwq+kDXoBxA8d1WT0ikp0iu5ZLtmoK92M1L523P5VwT2zTUuJjWmunXxoi2S2SgR6rOUbfsItIo4703C8W9kG0b+0XRnt/+eiXikjHRTLQc0l7gzdd50rlsUHWKiIXilyga8pfZlGIi3SdyK222LehMuwSRERCEbkeOkQn1A/FXuZzZWPDLqNLpTK2rtk/IslFMtAj4ciesCvIKK0FvcJd5D8p0DPUofoTYZeQ1RT0kosU6BJ5qc7PF8l2WR/omkUhTfSzILku6wM98o7sgQHXhV1FTtAwjWS7aAV6BD9IPFR/gs+hUO9qCnfJRtEK9Ig6VH+CDz8+RtnAfmGXEjmducJVQS+ZRoGeJfo2VEK3S9VTz1Dq0UsmiFSgR32q36H6E3xuQNhVSBPNjZdME6lAzwWxGg29ZBOFu3SlyAT6odjLYZfQJfo2VMLA3FoOIIo0Li/pEJlAzyXqpWcnfQAr6aZAF8kyGsaR1ijQs1CyGS/qtUebPoCVVEQj0CN4QVG7HNkD/F3YVUjIFO6S9YEeqzlG34ZoT1dMpuliI4CyyN2mRDorlfF6hX70ZH2gR+VmFh3Rt6GSD/uXNn/PwLEaepGUXaxHrw9ns1PWB7okiA+95OKdjqRzUl2pUkGf2VIKdDOrAP4NyANWuPv3Wxy3+PHJwH8A/+DurwVcazMtk/qf+jZUQsGlQPxK2f6N+5tulq3eunSF1nr77d0vndNmoJtZHvAkcAtQC+wys43ufiCh2SRgUPzrBmBp/N+0idUco6zbO+l8iqyRbMmD5qGobpcS+/jvGoP9yJ7Gsff4MI3CXtIhlRk5qexX0LdfKj3064Eqdz8EYGZrgClAYqBPAVa5uwO/N7N+ZvYZd/9z4BXH9W2o5FC6Tp7FWn6mcKj+BH2p5FDDhW0ONcCH/UsV7JKRgvpLvLW/DFprk81SCfQBQE3Cdi0X9r6TtRkAnBfoZjYLmBXf/MjMDrarWsgH3m/nY3KB3pfk9L5cSO9Jctn0vny2tQOpBLol2ecdaIO7LweWp/CcyQsx2+3u5R19fFTpfUlO78uF9J4kF5X3JZUZzLXAwITtQqCuA21ERCSNUgn0XcAgMys2s0uAe4CNLdpsBO63RjcCH6Rz/FxERC7U5pCLu581sweAF2mctrjS3feb2ez48WXAJhqnLFbROG3xi2mqt8PDNRGn9yU5vS8X0nuSXCTeF2ucmCIiItlOq4CIiESEAl1EJCKyJtDNrMLMDppZlZktCLuesJnZQDPbZmZvmNl+M5sbdk2ZxMzyzOx1M/t/YdeSKeIX/K0zszfjPzf/NeyawmZmX4v//9lnZs+YWc+wa+qMrAj0hOUHJgFDgKlmNiTcqkJ3Fvhf7n4tcCMwR+/JeeYCb4RdRIb5N+AFd78GGE6Ovz9mNgB4ECh392E0Tvq4J9yqOicrAp2E5Qfc/TTQtPxAznL3PzctgObux2n8zzkg3Koyg5kVArcBK8KuJVOYWV9gLPBjAHc/7e7HQi0qM3QHeplZd6A3WX79TLYEemtLCwhgZkXACODfQy4lUzwOPAx8HHIdmeRzQD3wk/hQ1AozuzTsosLk7keAxcBhGpcp+cDdfxVuVZ2TLYGe0tICucjMLgOeA/6nu38Ydj1hM7PbgffcPcfvS3iB7sBIYKm7jwBOADn9WZSZXUHjX/rFwJXApWY2LdyqOidbAl1LCyRhZj1oDPPV7r4+7HoyxGjgDjOrpnFo7u/N7P+EW1JGqAVq3b3pr7h1NAZ8LrsZ+KO717v7GWA98PmQa+qUbAn0VJYfyCnxm4r8GHjD3f932PVkCnd/xN0L3b2Ixp+Tre6e1b2uILj7X4AaMxsc3zWB85fAzkWHgRvNrHf8/9MEsvyD4qy4BV1ryw+EXFbYRgPTgb1mFovv+yd33xReSZLh/hFYHe8UHSJ9S3RkBXf/dzNbB7xG46yx18nyJQB06b+ISERky5CLiIi0QYEuIhIRCnQRkYhQoIuIRIQCXUQkIhToIiIRoUAXEYmI/w9iflt0knsXdQAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
@@ -189,7 +193,7 @@
],
"source": [
"fig, ax = plt.subplots()\n",
- "ax.hist(just_mmseqs_filtered['normalized bit score'], bins=100, label='mmseqs', alpha=0.3, density=True);\n",
+ "ax.hist(mmseqs_filtered['normalized bit score'], bins=100, label='mmseqs', alpha=0.3, density=True);\n",
"ax.hist(hhblits_filtered['normalized bit score'], bins=100, label='hhblits', alpha=0.3, density=True);\n",
"ax.hist(structural_annotation['normalized bit score'], bins=100, label='coffe', alpha=0.3, density=True);\n",
"ax.legend()"
@@ -198,75 +202,184 @@
{
"cell_type": "code",
"execution_count": 12,
- "id": "efdcc8b4",
+ "id": "7649fb1a",
"metadata": {},
"outputs": [],
"source": [
"unique_up_id = pd.concat([hhblits_filtered['target'].drop_duplicates(),\n",
- " just_mmseqs_filtered['target'].drop_duplicates()])\n",
+ " mmseqs_filtered['target'].drop_duplicates()])\n",
"unique_up_id.drop_duplicates(inplace=True)"
]
},
{
"cell_type": "code",
- "execution_count": 13,
- "id": "e1fcdc85",
- "metadata": {},
- "outputs": [],
- "source": [
- "with open(f'../data/milot_unique_ids.txt', 'w') as tmp:\n",
- " ids_as_str = unique_up_id.astype(str).values\n",
- " tmp.write(','.join(ids_as_str))"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "ec480a86",
+ "execution_count": 22,
+ "id": "e8d0cfb7-c86d-41a6-99e7-d514bc1f3873",
"metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "148"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "<div class=\"alert alert-block alert-warning\"><b>DISCLAIMER:</b> To make this cell execute, you may need to copy the contents of the UPIMAPI/resource folder into the main repository directory.</div>"
+ "request_size = 500\n",
+ "no_chunks = np.ceil((len(unique_up_id) / request_size) * 3).astype(int)\n",
+ "no_chunks"
]
},
{
"cell_type": "code",
- "execution_count": 42,
- "id": "87d494b2",
- "metadata": {},
+ "execution_count": 25,
+ "id": "85231f72-12ad-4cc7-897a-544e0be27228",
+ "metadata": {
+ "tags": []
+ },
"outputs": [
{
- "name": "stdout",
+ "name": "stderr",
"output_type": "stream",
"text": [
- "2022-07-27 11:48:46: ID mapping has begun.\n",
- "Auto determined \"full id\" as: False\n",
- "../data/uniprotinfo.fasta not found. Will perform mapping for all IDs.\n",
- "Checking which IDs are missing information.\n",
- "Checking which IDs are missing information.: 100%|â–ˆ| 24540/24540 [00:00<00:00, 3\n",
- "Information already gathered for 0 ids. Still missing for 24540.\n",
- "Building FASTA from 24540 IDs.\n",
- "UniProt ID mapping: 0%| | 0/3 [00:00<?, ?it/s]\n",
- "Traceback (most recent call last):\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 1130, in <module>\n",
- " upimapi()\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 1125, in upimapi\n",
- " uniprot_fasta_workflow(ids, f'{args.output}/uniprotinfo.fasta', step=args.step, sleep_time=args.sleep)\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 309, in uniprot_fasta_workflow\n",
- " uniprotinfo = get_uniprot_fasta(ids_missing, step=step, sleep_time=sleep_time)\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 285, in get_uniprot_fasta\n",
- " data = uniprot_request(ids[i:j], original_database='ACC+ID', database_destination='', output_format='fasta')\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 219, in uniprot_request\n",
- " 'columns': string4mapping(columns=columns, databases=databases)\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 176, in string4mapping\n",
- " result = ','.join([columns_dict[column] for column in columns])\n",
- " File \"/g/arendt/npapadop/repos/UPIMAPI/upimapi.py\", line 176, in <listcomp>\n",
- " result = ','.join([columns_dict[column] for column in columns])\n",
- "KeyError: 'Entry'\n"
+ "100%|██████████| 148/148 [01:19<00:00, 1.86it/s]\n"
]
}
],
"source": [
- "!python /g/arendt/npapadop/repos/UPIMAPI/upimapi.py -i ../data/milot_unique_ids.txt -o ../data/ --fasta --no-annotation"
+ "request_size = 500\n",
+ "no_chunks = np.ceil((len(unique_up_id) / request_size) * 3).astype(int)\n",
+ "\n",
+ "with open('../data/profile_sequences.fasta', 'w') as result:\n",
+ " for i in tqdm(range(no_chunks)):\n",
+ " a = (i * request_size) // 3\n",
+ " b = ((i+1) * request_size) // 3\n",
+ " chunk = unique_up_id[a:b]\n",
+ " bait50 = ['UniRef50_' + c for c in chunk]\n",
+ " bait90 = ['UniRef90_' + c for c in chunk]\n",
+ " bait100 = ['UniRef100_' + c for c in chunk]\n",
+ " expanded_chunk = bait50 + bait90 + bait100\n",
+ " url = f\"https://rest.uniprot.org/uniref/ids?ids={','.join(expanded_chunk)}&format=fasta\"\n",
+ " response = requests.get(url)\n",
+ " result.write(response.text)"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "da58c74f-5cd2-435a-8281-6195ed7f15af",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mmseqs_filtered.drop(columns=['index'], inplace=True)\n",
+ "hhblits_filtered.drop(columns=['index'], inplace=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "eb0c48d8-ef71-440f-8ea0-6e7f3118f957",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "file = '/g/arendt/npapadop/data/spongfold_publish/'\n",
+ "sensitive = pd.read_csv(file, sep='\\t', skiprows=4, skipfooter=3, engine='python')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1c156c27-d2ec-47d6-b764-2b54816922a0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sensitive['uniprot'] = sensitive['#query'].str.split('|').str[1]\n",
+ "sensitive.reset_index(drop=True, inplace=True)\n",
+ "# remove unnecessary columns\n",
+ "dead_weight = ['#query', 'seed_ortholog', 'EC', 'KEGG_ko', 'KEGG_Pathway',\n",
+ " 'KEGG_Module', 'KEGG_Reaction', 'KEGG_rclass', 'BRITE',\n",
+ " 'KEGG_TC', 'CAZy', 'BiGG_Reaction']\n",
+ "sensitive.drop(dead_weight, axis=1, inplace=True)\n",
+ "# convert rest to categorical to save space\n",
+ "to_categorical = ['uniprot', 'eggNOG_OGs', 'max_annot_lvl', 'COG_category',\n",
+ " 'Description', 'Preferred_name', 'GOs', 'PFAMs']\n",
+ "sensitive[to_categorical] = sensitive[to_categorical].astype(\"category\")\n",
+ "# finally save in parquet format\n",
+ "sensitive.to_parquet('../data/uniprot_profiles.parquet')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "821c79ed-2090-45c8-bc5c-736ba1096989",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "response"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fd08dceb-c307-4ad6-bd23-91b70aaf7ef4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mmseqs_filtered[~mmseqs_filtered['target'].isin(sensitive['uniprot'])]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c0f35e75-1a3f-4bab-8afe-160d3347738f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mmseqs_filtered"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f99bd75f-ae5a-4d00-8a01-5588c711f45c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "foldseek_full = pd.read_parquet('../old_data/fs_targets.parquet')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a1359934-a10c-4301-9ffa-000acb4159e4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mmseqs_filtered = mmseqs_filtered.merge(sensitive, on='uniprot', how='left')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "72eed180-8cb0-4f76-b316-f5717f56b026",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n",
+ "hhblits = hhblits.merge(sensitive, on='uniprot', how='left')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e5b356e4-10f9-4782-a39c-4f19a221ebdb",
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
@@ -285,7 +398,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.6"
+ "version": "3.9.6"
}
},
"nbformat": 4,
diff --git a/analysis/read-write.ipynb b/analysis/read-write.ipynb
index 669250e..1a74c5b 100644
--- a/analysis/read-write.ipynb
+++ b/analysis/read-write.ipynb
@@ -526,118 +526,43 @@
]
},
{
- "cell_type": "code",
- "execution_count": 18,
- "id": "563af56b-7a1b-4c3a-be9c-8c79fe684c01",
+ "cell_type": "markdown",
+ "id": "c2343e57-f527-4634-87a9-cbd2bc8aee5f",
"metadata": {},
- "outputs": [],
"source": [
- "plddt = np.load('../data/spongilla_plddt.npz')\n",
- "\n",
- "sequence_info = pd.read_csv(\"../data/sequence_info.csv\")\n",
- "query_to_isoform = sequence_info[['query', 'isoform']].set_index('query').to_dict()['isoform']"
+ "For PDB the situation is more complicated. We will take the PDB IDs and translate them to UniProt accession numbers using the UniProt API. This will return an inflated list of IDs since some PDB entries contain complexes."
]
},
{
"cell_type": "code",
- "execution_count": 19,
- "id": "9496151e-6ac4-40ed-a5c1-e3fd1e84fe05",
+ "execution_count": 1,
+ "id": "5b403921-8514-431a-a34d-f92fbf034ab8",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "1470483it [27:00, 907.18it/s] \n"
+ "perl: warning: Setting locale failed.\n",
+ "perl: warning: Please check that your locale settings:\n",
+ "\tLANGUAGE = (unset),\n",
+ "\tLC_ALL = (unset),\n",
+ "\tLC_CTYPE = \"UTF-8\",\n",
+ "\tLC_TERMINAL = \"iTerm2\",\n",
+ "\tLANG = \"en_US.UTF-8\"\n",
+ " are supported and installed on your system.\n",
+ "perl: warning: Falling back to a fallback locale (\"en_US.UTF-8\").\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "../data/uniprotinfo.fasta: 7206 sequences, 4638410 bp => dividing into 1 part of <= 100000 sequences\n",
+ "All done, 0 seconds elapsed\n"
]
}
],
- "source": [
- "pdb['aligned_plddt'] = get_aligned_plddt(pdb, plddt, query_to_isoform)\n",
- "pdb.to_parquet('../data/pdb_tmp.parquet')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "85f5b721-880c-400a-a330-37a35503a740",
- "metadata": {},
- "outputs": [],
- "source": [
- "print(\"done\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "c2343e57-f527-4634-87a9-cbd2bc8aee5f",
- "metadata": {},
- "source": [
- "For PDB the situation is more complicated. We will take the PDB IDs and translate them to UniProt accession numbers using the UniProt API. This will return an inflated list of IDs since some PDB entries contain complexes."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "76ff0420-e724-4152-abbc-29d25bfd3f59",
- "metadata": {
- "tags": []
- },
- "outputs": [],
- "source": [
- "pdb = enrich_from_uniprot(pdb, \"target\", \"uniprot\", uniprot_from=\"PDB_ID\", uniprot_to=\"ACC\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "7262d562-3c2d-4a7a-9361-1bd197d30a7c",
- "metadata": {},
- "outputs": [],
- "source": [
- "unique_up_id = pd.concat([pdb['uniprot'].drop_duplicates(),\n",
- " afdb['uniprot'].drop_duplicates(),\n",
- " swp['uniprot'].drop_duplicates()])\n",
- "unique_up_id.drop_duplicates(inplace=True)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "df0fdae3-52e6-4e7a-87e8-c81401278b09",
- "metadata": {},
- "outputs": [],
- "source": [
- "unique_up_id.to_csv('../data/unique_uniprot_ids.csv', header=False, index=False)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "ec3015c3-316e-48fe-a08a-4fd4323cb9ea",
- "metadata": {},
- "outputs": [],
- "source": [
- "with open(f'../data/foldseek_unique_ids.txt', 'w') as tmp:\n",
- " ids_as_str = unique_up_id.astype(str)[0]\n",
- " tmp.write(','.join(ids_as_str))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "4650ac1f-4d11-4590-b39f-2a16468d808e",
- "metadata": {},
- "outputs": [],
- "source": [
- "!python /g/arendt/npapadop/repos/UPIMAPI/upimapi.py -i ../data/foldseek_unique_ids.txt -o ../data/ --fasta --no-annotation"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "5b403921-8514-431a-a34d-f92fbf034ab8",
- "metadata": {},
- "outputs": [],
"source": [
"%%bash\n",
"mkdir ../data/uniprot_fastas/ -p\n",
diff --git a/analysis/suppl-ModelArchive_metadata.ipynb b/analysis/suppl-ModelArchive_metadata.ipynb
index 5f73751..f30f805 100644
--- a/analysis/suppl-ModelArchive_metadata.ipynb
+++ b/analysis/suppl-ModelArchive_metadata.ipynb
@@ -3,14 +3,14 @@
{
"cell_type": "code",
"execution_count": 1,
- "id": "e951dd81-8a76-45fb-a00d-8de54697bf0f",
+ "id": "a121491b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "2022-06-08 15:01\n"
+ "2022-07-28 08:02\n"
]
}
],
@@ -28,7 +28,7 @@
},
{
"cell_type": "markdown",
- "id": "84d83807-5bdb-4fb7-8bcc-069c17f984a4",
+ "id": "308f25e4",
"metadata": {},
"source": [
"In this script I will prepare the metadata for deposition in [ModelArchive](https://www.modelarchive.org). I will use the lookup table produced by the CoFFE pipeline and concatenate all the possibly relevant information into a single column (description)."
@@ -37,7 +37,7 @@
{
"cell_type": "code",
"execution_count": 2,
- "id": "3bebc329-4921-4890-baac-5f3718a687d9",
+ "id": "9eb7d317",
"metadata": {},
"outputs": [],
"source": [
@@ -46,19 +46,39 @@
},
{
"cell_type": "code",
- "execution_count": 3,
- "id": "38b0ec79-e498-4ac0-ae63-9b1ce7ff44a1",
+ "execution_count": 4,
+ "id": "d7d72594",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "ename": "FileNotFoundError",
+ "evalue": "[Errno 2] No such file or directory: '../ol_data/Slacustris_eggnog.tsv'",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m/tmp/ipykernel_136/1368120052.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msequence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../ol_data/Slacustris_eggnog.tsv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'\\t'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mstructure\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_parquet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../old_data/structure_annotation.parquet'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0mstacklevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstacklevel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 310\u001b[0m )\n\u001b[0;32m--> 311\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[1;32m 584\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 585\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 587\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 588\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 481\u001b[0m \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 482\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 484\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 809\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 811\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 812\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 813\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/parsers/readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m 1038\u001b[0m )\n\u001b[1;32m 1039\u001b[0m \u001b[0;31m# error: Too many arguments for \"ParserBase\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1040\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mmapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1041\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1042\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_failover_to_python\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/parsers/c_parser_wrapper.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[0;31m# open handles\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open_handles\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 52\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandles\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/parsers/base_parser.py\u001b[0m in \u001b[0;36m_open_handles\u001b[0;34m(self, src, kwds)\u001b[0m\n\u001b[1;32m 220\u001b[0m \u001b[0mLet\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mreaders\u001b[0m \u001b[0mopen\u001b[0m \u001b[0mIOHandles\u001b[0m \u001b[0mafter\u001b[0m \u001b[0mthey\u001b[0m \u001b[0mare\u001b[0m \u001b[0mdone\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtheir\u001b[0m \u001b[0mpotential\u001b[0m \u001b[0mraises\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 221\u001b[0m \"\"\"\n\u001b[0;32m--> 222\u001b[0;31m self.handles = get_handle(\n\u001b[0m\u001b[1;32m 223\u001b[0m \u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 224\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m 699\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoding\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"b\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 700\u001b[0m \u001b[0;31m# Encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 701\u001b[0;31m handle = open(\n\u001b[0m\u001b[1;32m 702\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 703\u001b[0m \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../ol_data/Slacustris_eggnog.tsv'"
+ ]
+ }
+ ],
"source": [
- "sequence = pd.read_csv('../data/Slacustris_eggnog.tsv', sep='\\t')\n",
- "structure = pd.read_parquet('../data/structure_annotation.parquet')"
+ "sequence = pd.read_csv('../old_data/Slacustris_eggnog.tsv', sep='\\t')\n",
+ "structure = pd.read_parquet('../old_data/structure_annotation.parquet')"
]
},
{
"cell_type": "code",
- "execution_count": 4,
- "id": "a51fa323-ff59-4acf-abaa-6ddddfc8de5c",
+ "execution_count": null,
+ "id": "6532f2fc",
"metadata": {},
"outputs": [],
"source": [
@@ -67,14 +87,14 @@
},
{
"cell_type": "code",
- "execution_count": 5,
- "id": "fa874500-61c0-4410-ae55-b081e0912670",
+ "execution_count": null,
+ "id": "4a137f30",
"metadata": {},
"outputs": [],
"source": [
"def generate_description(row):\n",
- " desc = f\"Emapper preferred name [EggNOG]: {row['Preferred_name_seq']}\\n\" + \\\n",
- " f\"Emapper description [EggNOG]: {row['Description_seq']}\\n\" + \\\n",
+ " desc = f\"Preferred name [EggNOG]: {row['Preferred_name_seq']}\\n\" + \\\n",
+ " f\"Description [EggNOG]: {row['Description_seq']}\\n\" + \\\n",
" f\"CoFFE preferred name [EggNOG]: {row['Preferred_name_str']}\\n\" + \\\n",
" f\"CoFFE description [EggNOG]: {row['Description_str']}\\n\" + \\\n",
" f\"CoFFE UniProt function: {row['Function [CC]']}\\n\\n\" + \\\n",
@@ -93,19 +113,19 @@
" f\"CoFFE Emapper bit score [EggNOG]: {row['score_str']}\\n\" + \\\n",
" f\"CoFFE orthogroups [EggNOG]: {row['eggNOG_OGs_str']}\\n\" + \\\n",
" f\"CoFFE maximum annotation level [EggNOG]: {row['max_annot_lvl_str']}\\n\" + \\\n",
- " f\"CoFFE GO terms [EggNOG]: {row['GOs_str']}\\n\" + \\\n",
+ "# f\"CoFFE GO terms [EggNOG]: {row['GOs_str']}\\n\" + \\\n",
" f\"CoFFE PFAM domains [EggNOG]: {row['PFAMs_str']}\\n\\n\" + \\\n",
" f\"best sequence hit bit score [Emapper]: {row['score_seq']}\\n\" + \\\n",
- " f\"Emapper orthogroups [EggNOG]: {row['eggNOG_OGs_seq']}\\n\" + \\\n",
- " f\"Emapper maximum annotation level [EggNOG]: {row['max_annot_lvl_seq']}\\n\" + \\\n",
- " f\"Emapper GO terms [EggNOG]: {row['GOs_seq']}\\n\" + \\\n",
- " f\"Emapper PFAM domains [EggNOG]: {row['PFAMs_seq']}\\n\"\n",
+ " f\"Orthogroups [EggNOG]: {row['eggNOG_OGs_seq']}\\n\" + \\\n",
+ " f\"Maximum annotation level [EggNOG]: {row['max_annot_lvl_seq']}\\n\" + \\\n",
+ "# f\"Emapper GO terms [EggNOG]: {row['GOs_seq']}\\n\" + \\\n",
+ " f\"PFAM domains [EggNOG]: {row['PFAMs_seq']}\\n\"\n",
" return desc"
]
},
{
"cell_type": "markdown",
- "id": "2afdd7a7-1131-415d-aaa6-337220672d4d",
+ "id": "6712bc16",
"metadata": {},
"source": [
"Let's test it first:"
@@ -114,7 +134,7 @@
{
"cell_type": "code",
"execution_count": 6,
- "id": "63193db9-9f82-4126-80e3-ec76033ad1c6",
+ "id": "935a770c",
"metadata": {},
"outputs": [
{
@@ -163,7 +183,7 @@
},
{
"cell_type": "markdown",
- "id": "8097273d-2713-4597-9c94-7feb6f6280f9",
+ "id": "6c9e5b53",
"metadata": {},
"source": [
"Looks good! Now let's apply it to our table:"
@@ -172,7 +192,7 @@
{
"cell_type": "code",
"execution_count": 7,
- "id": "43e594b0-0a9b-4e37-b21a-a8a6cd8eda3c",
+ "id": "1c5133d7",
"metadata": {},
"outputs": [],
"source": [
@@ -181,7 +201,7 @@
},
{
"cell_type": "markdown",
- "id": "9c16526c-9a46-4abd-ac8f-25da62d23563",
+ "id": "34028222",
"metadata": {},
"source": [
"Subset the table, rename columns, and save:"
@@ -190,7 +210,7 @@
{
"cell_type": "code",
"execution_count": 8,
- "id": "97598d9d-34f8-4644-b1cd-a5be8d2c45dd",
+ "id": "fa278548",
"metadata": {},
"outputs": [],
"source": [
@@ -202,7 +222,7 @@
{
"cell_type": "code",
"execution_count": 9,
- "id": "3463393a-74ff-4239-bbee-8f240dc1902f",
+ "id": "1f718df1",
"metadata": {},
"outputs": [],
"source": [
@@ -226,7 +246,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.6"
+ "version": "3.8.13"
}
},
"nbformat": 4,
diff --git a/scripts/results_processing/README.md b/scripts/results_processing/README.md
new file mode 100755
index 0000000..6d01cd1
--- /dev/null
+++ b/scripts/results_processing/README.md
@@ -0,0 +1,29 @@
+# Read/write operations for result analysis
+
+The directory contains all scripts to rerun the processing of ColabFold and Foldseek results. The
+`python` files contain the code for processing; the corresponding `.sh` scripts are used to submit
+to the SLURM cluster.
+
+### io_parse_structures.py
+
+Will go through the predicted best models and extract the pLDDT score for each residue. It saves the
+per-residue-pLDDT for each protein in a dictionary (`npz` file) and the average pLDDT in a CSV
+table.
+
+### io_read_afdb.py
+
+Processes the Foldseek predictions against AlphaFold DB. Will return a table that lists all
+isoforms, their Foldseek scores, the average pLDDT of the aligned residues, and the UniProt ID of
+the target protein.
+
+### io_read_swp.py
+
+Processes the Foldseek predictions against SwissProt. Will return a table that lists all isoforms,
+their Foldseek scores, the average pLDDT of the aligned residues, and the UniProt ID of the target
+protein.
+
+### io_read_pdb.py
+
+Processes the Foldseek predictions against PDB. Will return a table that lists all isoforms, their
+Foldseek scores, the average pLDDT of the aligned residues, and the UniProt ID of the target
+protein. The UniProt ID is fetched by querying UniProt (UniProt API).
\ No newline at end of file
diff --git a/scripts/io_parse_structures.py b/scripts/results_processing/io_parse_structures.py
similarity index 100%
rename from scripts/io_parse_structures.py
rename to scripts/results_processing/io_parse_structures.py
diff --git a/scripts/io_parse_structures.sh b/scripts/results_processing/io_parse_structures.sh
similarity index 100%
rename from scripts/io_parse_structures.sh
rename to scripts/results_processing/io_parse_structures.sh
diff --git a/scripts/io_read_afdb.py b/scripts/results_processing/io_read_afdb.py
similarity index 84%
rename from scripts/io_read_afdb.py
rename to scripts/results_processing/io_read_afdb.py
index c3e197e..57347dd 100644
--- a/scripts/io_read_afdb.py
+++ b/scripts/results_processing/io_read_afdb.py
@@ -66,17 +66,9 @@ def enrich_from_uniprot(df, column_from, column_to, uniprot_from="PDB_ID", unipr
df_map = create_id_map(response, column_from, column_to)
return df.join(df_map, on=column_from)
-# MSA location
-msas = '/g/arendt/npapadop/repos/coffe/data/msa/'
+
# FOLDSEEK scores
-fs_pdb = '/g/arendt/npapadop/repos/coffe/data/pdb_score.tsv'
fs_afdb = '/g/arendt/npapadop/repos/coffe/data/afdb_score.tsv'
-fs_swp = '/g/arendt/npapadop/repos/coffe/data/swissprot_score.tsv'
-# AlphaFold predictions
-structure_list = '/g/arendt/npapadop/repos/coffe/data/named_models/'
-metadata = '/g/arendt/npapadop/repos/coffe/data/named_metadata/'
-# input fasta
-all_peptides = '/g/arendt/npapadop/repos/coffe/data/Spongilla_lacustris_translated_proteome.fasta'
plddt = np.load('/g/arendt/npapadop/repos/coffe/data/spongilla_plddt.npz')
@@ -88,4 +80,4 @@ afdb["query"] = afdb["query"].values.astype(int)
afdb['aligned_plddt'] = get_aligned_plddt(afdb, plddt, query_to_isoform)
afdb["uniprot"] = afdb["target"].str.split("-").str[1]
-afdb_best.to_parquet('/g/arendt/npapadop/repos/coffe/data/afdb_tmp.parquet')
\ No newline at end of file
+afdb.to_parquet('/g/arendt/npapadop/repos/coffe/data/afdb_tmp.parquet')
\ No newline at end of file
diff --git a/scripts/io_read_afdb.sh b/scripts/results_processing/io_read_afdb.sh
similarity index 60%
rename from scripts/io_read_afdb.sh
rename to scripts/results_processing/io_read_afdb.sh
index 23dfba3..b9eed76 100644
--- a/scripts/io_read_afdb.sh
+++ b/scripts/results_processing/io_read_afdb.sh
@@ -2,12 +2,12 @@
#SBATCH -J coffe_read_afdb
#SBATCH -t 5:00:00
#SBATCH -c 1
-#SBATCH --mem=16G
+#SBATCH --mem=32G
#SBATCH -o /g/arendt/npapadop/cluster/io_read_afdb.out
#SBATCH -e /g/arendt/npapadop/cluster/io_read_afdb.err
module load Anaconda3
source ~/.bash_profile
-conda activate /g/arendt/npapadop/repos/condas/milot
+conda activate /g/arendt/npapadop/repos/condas/CoFFE
-python /g/arendt/npapadop/repos/coffe/scripts/io_read_afdb.py
\ No newline at end of file
+python /g/arendt/npapadop/repos/coffe/scripts/results_processing/io_read_afdb.py
\ No newline at end of file
diff --git a/scripts/io_read_pdb.py b/scripts/results_processing/io_read_pdb.py
similarity index 81%
rename from scripts/io_read_pdb.py
rename to scripts/results_processing/io_read_pdb.py
index 556b717..f1c8e3b 100644
--- a/scripts/io_read_pdb.py
+++ b/scripts/results_processing/io_read_pdb.py
@@ -1,6 +1,7 @@
import os
import json
from tqdm import tqdm
+from urllib import parse, request
import numpy as np
import pandas as pd
@@ -32,7 +33,7 @@ def get_aligned_plddt(df, plddt, name_dict):
def get_from_uniprot(df, column, uniprot_from="ACC", uniprot_to="EGGNOG_ID", request_size=40000):
ids = df[column][~df[column].isnull()]
- url = 'https://www.uniprot.org/uploadlists/'
+ url = 'https://rest.uniprot.org/'
params = {
'from': uniprot_from,
@@ -41,10 +42,10 @@ def get_from_uniprot(df, column, uniprot_from="ACC", uniprot_to="EGGNOG_ID", req
'query': " ".join(ids.unique())
}
- data = urllib.parse.urlencode(params)
+ data = parse.urlencode(params)
data = data.encode('utf-8')
- req = urllib.request.Request(url, data)
- with urllib.request.urlopen(req) as f:
+ req = request.Request(url, data)
+ with request.urlopen(req) as f:
response = f.read()
return response
@@ -66,18 +67,9 @@ def enrich_from_uniprot(df, column_from, column_to, uniprot_from="PDB_ID", unipr
df_map = create_id_map(response, column_from, column_to)
return df.join(df_map, on=column_from)
-# MSA location
-msas = '/g/arendt/npapadop/repos/coffe/data/msa/'
# FOLDSEEK scores
fs_pdb = '/g/arendt/npapadop/repos/coffe/data/pdb_score.tsv'
-fs_afdb = '/g/arendt/npapadop/repos/coffe/data/afdb_score.tsv'
-fs_swp = '/g/arendt/npapadop/repos/coffe/data/swissprot_score.tsv'
-# AlphaFold predictions
-structure_list = '/g/arendt/npapadop/repos/coffe/data/named_models/'
-metadata = '/g/arendt/npapadop/repos/coffe/data/named_metadata/'
-# input fasta
-all_peptides = '/g/arendt/npapadop/repos/coffe/data/Spongilla_lacustris_translated_proteome.fasta'
-
+# per-residue pLDDT score
plddt = np.load('/g/arendt/npapadop/repos/coffe/data/spongilla_plddt.npz')
sequence_info = pd.read_csv("/g/arendt/npapadop/repos/coffe/data/sequence_info.csv")
diff --git a/scripts/io_read_pdb.sh b/scripts/results_processing/io_read_pdb.sh
similarity index 65%
rename from scripts/io_read_pdb.sh
rename to scripts/results_processing/io_read_pdb.sh
index 63dbd2e..800685e 100644
--- a/scripts/io_read_pdb.sh
+++ b/scripts/results_processing/io_read_pdb.sh
@@ -8,6 +8,6 @@
module load Anaconda3
source ~/.bash_profile
-conda activate /g/arendt/npapadop/repos/condas/milot
+conda activate /g/arendt/npapadop/repos/condas/CoFFE
-python /g/arendt/npapadop/repos/coffe/scripts/io_read_pdb.py
\ No newline at end of file
+python /g/arendt/npapadop/repos/coffe/scripts/results_processing/io_read_pdb.py
\ No newline at end of file
diff --git a/scripts/io_read_swp.py b/scripts/results_processing/io_read_swp.py
similarity index 97%
rename from scripts/io_read_swp.py
rename to scripts/results_processing/io_read_swp.py
index 7a7213d..ec5dc05 100644
--- a/scripts/io_read_swp.py
+++ b/scripts/results_processing/io_read_swp.py
@@ -88,4 +88,4 @@ swp["query"] = swp["query"].values.astype(int)
swp['aligned_plddt'] = get_aligned_plddt(swp, plddt, query_to_isoform)
swp["uniprot"] = swp["target"].str.split("-").str[1]
-swp_best.to_parquet('/g/arendt/npapadop/repos/coffe/data/swp_tmp.parquet')
\ No newline at end of file
+swp.to_parquet('/g/arendt/npapadop/repos/coffe/data/swp_tmp.parquet')
\ No newline at end of file
diff --git a/scripts/io_read_swp.sh b/scripts/results_processing/io_read_swp.sh
similarity index 65%
rename from scripts/io_read_swp.sh
rename to scripts/results_processing/io_read_swp.sh
index 947d670..f665f79 100644
--- a/scripts/io_read_swp.sh
+++ b/scripts/results_processing/io_read_swp.sh
@@ -8,6 +8,6 @@
module load Anaconda3
source ~/.bash_profile
-conda activate /g/arendt/npapadop/repos/condas/milot
+conda activate /g/arendt/npapadop/repos/condas/CoFFE
-python /g/arendt/npapadop/repos/coffe/scripts/io_read_swp.py
\ No newline at end of file
+python /g/arendt/npapadop/repos/coffe/scripts/results_processing/io_read_swp.py
\ No newline at end of file
--
GitLab