Curation Script.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "b0c69e97",
   "metadata": {},
   "source": [
    "CURATION SCRIPT\n",
    "by: Igor Sanches\n",
    "Edited by: Francisco Feitosa"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4e154193",
   "metadata": {},
   "source": [
    "**IMPORT DEPENDENCIES AND IN-HOUSE FUNCTIONS**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4141f75",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#import libraries and dependencies\n",
    "\n",
    "import pandas as pd\n",
    "import math\n",
    "import numpy as np\n",
    "\n",
    "from rdkit import Chem\n",
    "from chembl_structure_pipeline import standardizer\n",
    "import rdkit.Chem.MolStandardize.rdMolStandardize as rdMolStandardize\n",
    "from rdkit.Chem import PandasTools\n",
    "\n",
    "from rdkit.Chem.SaltRemover import SaltRemover\n",
    "from rdkit.Chem import inchi as rd_inchi\n",
    "\n",
    "from molvs import standardize_smiles\n",
    "from molvs import Standardizer\n",
    "from rdkit.Chem import Draw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7cd21c55",
   "metadata": {},
   "outputs": [],
   "source": [
    "#in-house functions\n",
    "def metal_atomic_numbers(at):\n",
    "    \"\"\" This function checks the atomic number of an atom \"\"\"\n",
    "    \n",
    "    n = at.GetAtomicNum()\n",
    "    return (n==13) or (n>=21 and n<=31) or (n>=39 and n<=50) or (n>=57 and n<=83) or (n>=89 and n<=115)\n",
    "\n",
    "def is_metal(smile):\n",
    "    \"\"\" This function checks if an atom is a metal based on its atomic number \"\"\"\n",
    "    mol = Chem.MolFromSmiles(smile)\n",
    "    rwmol = Chem.RWMol(mol)\n",
    "    rwmol.UpdatePropertyCache(strict=False)\n",
    "    metal = [at.GetSymbol() for at in rwmol.GetAtoms() if metal_atomic_numbers(at)]\n",
    "    return len(metal) == 1\n",
    "\n",
    "def smiles_preparator(smile):\n",
    "    \"\"\" This function prepares smiles by removing stereochemistry \"\"\"\n",
    "    smile1 = smile.replace('@','')\n",
    "    smile2 = smile1.replace('/','')\n",
    "    smile3 = smile2.replace(\"\\\\\",'')\n",
    "    return str(smile3)\n",
    "\n",
    "def salt_remover(mol):\n",
    "    \"\"\" This function removes salts, see complete list of possible salts in https://github.com/rdkit/rdkit/blob/master/Data/Salts.txt \"\"\"\n",
    "\n",
    "    salt_list = [None, \"[Cl,Br,I]\", \"[Li,Na,K,Ca,Mg]\", \"[O,N]\", \"[H]\", \"[Ba]\", \"[Al]\", \"[Cu]\", \"[Cs]\", \"[Zn]\", \n",
    "    \"[Mn]\", \"Cl[Cr]Cl\", \"COS(=O)(=O)[O-]\", \"[Sb]\", \"[Cr]\", \"[Ni]\", \"[B]\", \"CCN(CC)CC\", \"NCCO\", \"O=CO\", \"O=S(=O)([O-])C(F)(F)F\",\n",
    "    \"O=C(O)C(F)(F)F\"]\n",
    "\n",
    "    stripped = mol\n",
    "\n",
    "    for salt in salt_list:\n",
    "        remover = SaltRemover(defnData=salt)\n",
    "        stripped = remover.StripMol(stripped, dontRemoveEverything=True)\n",
    "    \n",
    "    return stripped\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "0987bb71",
   "metadata": {},
   "source": [
    "**SET PATH**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b704a1b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#choose a path to save\n",
    "savepath = r\"...\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e3448b12",
   "metadata": {},
   "outputs": [],
   "source": [
    "df0 = pd.read_excel(r\"...\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "d88684f1",
   "metadata": {},
   "source": [
    "**DATA PREPARATION**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1156deee",
   "metadata": {},
   "outputs": [],
   "source": [
    "df0 = df0.loc[:, ['PUBCHEM_EXT_DATASOURCE_SMILES', 'PUBCHEM_ACTIVITY_OUTCOME', 'PUBCHEM_CID']]\n",
    "df0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0210b445",
   "metadata": {},
   "outputs": [],
   "source": [
    "df0 = df0.rename(columns={'Canonical SMILES':'Molecule', 'GHS':'Outcome'})\n",
    "df0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfc2905d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def string_to_int(s):\n",
    "    mapping = {\"Active\": 1, \"Inactive\": 0}\n",
    "    return mapping.get(s, None)\n",
    "\n",
    "df0['Outcome'] = df0['Outcome'].apply(string_to_int)\n",
    "df0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c42668c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df0 = df0.dropna(subset=['Outcome'])\n",
    "df0 = df0.reset_index(drop=True)\n",
    "df0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dde6d50",
   "metadata": {},
   "outputs": [],
   "source": [
    "df0 = df0.dropna(subset=['Molecule'])\n",
    "df0 = df0.reset_index(drop=True)\n",
    "df0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb6e23f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df0.groupby('Outcome').size()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "d9f760ff",
   "metadata": {},
   "source": [
    "**ESTEREO REMOVAL**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c8e290a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#remove stereoisomers \n",
    "smiles = [smiles_preparator(str(smile)) for smile in df0['Molecule']]\n",
    "df0['SMILES_no_stereo'] = smiles\n",
    "\n",
    "#remove relation column\n",
    "df0"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "8d95540a",
   "metadata": {},
   "source": [
    "**REMOVE SALTS AND INVALID SMILES**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8212bf00",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#remove salts\n",
    "wrongSmiles = []\n",
    "new_smiles = []\n",
    "indexDropList_salts = []\n",
    "for index, smile in enumerate(df0['SMILES_no_stereo']):\n",
    "    try:\n",
    "        mol = Chem.MolFromSmiles(smile)\n",
    "        remov = salt_remover(mol)\n",
    "        if remov.GetNumAtoms() <= 2:\n",
    "            indexDropList_salts.append(index)\n",
    "        else:\n",
    "            new_smiles.append(Chem.MolToSmiles(remov, kekuleSmiles=True))\n",
    "        \n",
    "    except:\n",
    "        wrongSmiles.append(df0.iloc[[index]])\n",
    "        indexDropList_salts.append(index)\n",
    "\n",
    "\n",
    "if len(wrongSmiles) == 0:\n",
    "    print(\"no wrong smiles found\")\n",
    "    \n",
    "else:\n",
    "    #drop wrong smiles\n",
    "    df0 = df0.drop(df0.index[indexDropList_salts])\n",
    "    \n",
    "    print(f\"{len(indexDropList_salts)} wrong smiles found\")\n",
    "    \n",
    "    #save removes mixtures\n",
    "    wrongsmiles = pd.concat(wrongSmiles)\n",
    "    wrongsmiles.to_csv(f'{savepath}\\\\wrongsmiles_tryp.csv', sep=',', header=True, index=False)\n",
    "df0['SMILES_no_salts'] = new_smiles\n",
    "df0 \n",
    "\n",
    "df0 = df0.reset_index(drop=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "6ec4fbec",
   "metadata": {},
   "source": [
    "**REMOVE ORGANOMETALLICS**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d6cb5d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "organometals = []\n",
    "indexDropList_org = []\n",
    "for index, smile in enumerate(df0['SMILES_no_salts']):\n",
    "    if is_metal(smile) == True:\n",
    "        organometals.append(df0.iloc[[index]])\n",
    "        indexDropList_org.append(index)\n",
    "\n",
    "if len(indexDropList_org) == 0:\n",
    "    print(\"no organometallics found\")\n",
    "    \n",
    "else:\n",
    "    #drop organometallics\n",
    "    df0 = df0.drop(df0.index[indexDropList_org])\n",
    "    \n",
    "    print(f\"{len(indexDropList_org)} organometallics found\")\n",
    "    \n",
    "    #save droped organometallics\n",
    "    organmetal = pd.concat(organometals)\n",
    "    organmetal.to_csv(f'{savepath}\\\\organometallics_tryp.csv', sep=',', header=True, index=False)\n",
    "    \n",
    "df0\n",
    "\n",
    "df0 = df0.reset_index(drop=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "35937f65",
   "metadata": {},
   "source": [
    "**REMOVE MIXTURES**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32968b64",
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove mixtures\n",
    "mixtureList = []\n",
    "indexDropList_mix = []\n",
    "for index, smile in enumerate (df0['SMILES_no_salts']):\n",
    "    for char in smile:\n",
    "        if char == '.':\n",
    "            mixtureList.append(df0.iloc[[index]])\n",
    "            indexDropList_mix.append(index)\n",
    "            break\n",
    "\n",
    "            \n",
    "if len(indexDropList_mix) == 0:\n",
    "    print(\"no mixtures found\")\n",
    "    \n",
    "else:\n",
    "    #drop mixtures\n",
    "    df0 = df0.drop(df0.index[indexDropList_mix])\n",
    "    \n",
    "    print(f\"{len(indexDropList_mix)} mixtures found\")\n",
    "    \n",
    "    #save removes mixtures\n",
    "    mixtures = pd.concat(mixtureList)\n",
    "    mixtures.to_csv(f'{savepath}\\\\mixtures_tryp.csv', sep=',', header=True, index=False)\n",
    "df0 \n",
    "\n",
    "df0 = df0.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9735ce16",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Remove moléculas com peso molecular acima de 1000\n",
    "\n",
    "mols = [Chem.MolFromSmiles(smile) for smile in df0['SMILES_no_salts']]\n",
    "MWdroplist = []\n",
    "indexdroplist_MW = []\n",
    "for index, mol in enumerate(mols):\n",
    "    mw = Chem.rdMolDescriptors.CalcExactMolWt(mol)\n",
    "    if mw >= 1000: \n",
    "        MWdroplist.append(df0.iloc[[index]])\n",
    "        indexdroplist_MW.append(index)\n",
    "    else:\n",
    "        pass\n",
    "\n",
    "if len(indexdroplist_MW) == 0:\n",
    "    print(\"no molecule removed\")\n",
    "    \n",
    "else:\n",
    "    #drop MW >1000\n",
    "    df0 = df0.drop(df0.index[indexdroplist_MW])\n",
    "    \n",
    "    print(f\"{len(indexdroplist_MW)} polymer found\")\n",
    "    \n",
    "    #save removes \n",
    "    MWW = pd.concat(MWdroplist)\n",
    "    MWW.to_csv(f'{savepath}\\\\MW_tryp.csv', sep=',', header=True, index=False)\n",
    "df0 \n",
    "\n",
    "df0 = df0.reset_index(drop=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "72345974",
   "metadata": {},
   "source": [
    "**STANDARDISE**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40f1b07f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df0['final_smiles'] = [Chem.MolToSmiles(Chem.MolFromMolBlock(standardizer.standardize_molblock(Chem.MolToMolBlock(Chem.MolFromSmiles(smile, sanitize=True))))) for smile in df0['SMILES_no_salts']]\n",
    "df0 = df0.reset_index(drop=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "6c954840",
   "metadata": {},
   "source": [
    "**Remove Salts for the second time**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c2b1308",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#remove salts second time\n",
    "wrongSmiles = []\n",
    "new_smiles = []\n",
    "indexDropList_salts = []\n",
    "for index, smile in enumerate(df0['final_smiles']):\n",
    "    try:\n",
    "        mol = Chem.MolFromSmiles(smile)\n",
    "        remov = salt_remover(mol)\n",
    "        if remov.GetNumAtoms() <= 2:\n",
    "            indexDropList_salts.append(index)\n",
    "        else:\n",
    "            new_smiles.append(Chem.MolToSmiles(remov, kekuleSmiles=True))\n",
    "        \n",
    "    except:\n",
    "        wrongSmiles.append(df0.iloc[[index]])\n",
    "        indexDropList_salts.append(index)\n",
    "\n",
    "\n",
    "if len(wrongSmiles) == 0:\n",
    "    print(\"no wrong smiles found\")\n",
    "    \n",
    "else:\n",
    "    #drop wrong smiles\n",
    "    df0 = df0.drop(df0.index[indexDropList_salts])\n",
    "    \n",
    "    print(f\"{len(indexDropList_salts)} wrong smiles found\")\n",
    "    \n",
    "    #save removes mixtures\n",
    "    wrongsmiles = pd.concat(wrongSmiles)\n",
    "    wrongsmiles.to_csv(f'{savepath}\\\\wrongsmiles_after_std.csv', sep=',', header=True, index=False)\n",
    "df0['final_smiles_stand'] = new_smiles\n",
    "df0 \n",
    "\n",
    "df0 = df0.reset_index(drop=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "236d811b",
   "metadata": {},
   "source": [
    "### DUPLICATES REMOVAL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afa9222e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate the InChI\n",
    "inchi_list = []\n",
    "for smiles in df0['final_smiles_stand']:\n",
    "    mol = Chem.MolFromSmiles(smiles)\n",
    "    inchi = Chem.inchi.MolToInchi(mol)\n",
    "    inchi_list.append(inchi)\n",
    "\n",
    "# Adicionar a coluna de InChI no dataframe\n",
    "df0['InChI'] = inchi_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16e9bd58",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Binary\n",
    "\n",
    "df0_active = df0.query('Outcome == 1')\n",
    "df0_inactive = df0.query('Outcome == 0')\n",
    "\n",
    "df0_active = df0_active.drop_duplicates(subset=['InChI'], inplace=False)\n",
    "df0_inactive = df0_inactive.drop_duplicates(subset=['InChI'], inplace=False)\n",
    "\n",
    "df_no_dup_concord = pd.concat([df0_active, df0_inactive], axis=0)\n",
    "\n",
    "final_drop_dup = df_no_dup_concord.drop_duplicates(subset=['InChI'], keep=False, inplace=False)\n",
    "\n",
    "lastcount = final_drop_dup['InChI'].count()\n",
    "\n",
    "df_final = final_drop_dup\n",
    "\n",
    "df_final = df_final.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a39c2d18",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a279d27",
   "metadata": {},
   "outputs": [],
   "source": [
    "df0_dropped = df0[~df0['InChI'].isin(df_final['InChI'])]\n",
    "dropped_duplist = df0_dropped.loc[:, ['final_smiles_stand', 'Outcome', 'CAS No', 'InChI', 'Source']]\n",
    "dropped_duplist.to_csv(f'{savepath}\\\\dropped_duplicates.csv', sep=',', header=True, index=False)\n",
    "\n",
    "countdprem = df0_dropped['InChI'].count()\n",
    "print(f\"{countdprem} duplicates removed\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "d21c5941",
   "metadata": {},
   "source": [
    "### SAVE FINAL DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8067a61",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.groupby('Outcome').size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c53a7305",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.rename(columns={'final_smiles_stand': 'SMILES'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44d795e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cd333a34",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_final = df_final.loc[:, ['SMILES', 'Outcome', 'Source', 'CAS No']]\n",
    "df_final.to_csv(f'{savepath}\\curated_binary.csv', sep=',', header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "eeeb3d57",
   "metadata": {},
   "outputs": [],
   "source": [
    "def escrever_log(**contagens):\n",
    "    with open(f'{savepath}\\log.txt', 'w') as arquivo:\n",
    "        arquivo.write('Were removed:\\n')\n",
    "        for chave, valor in contagens.items():\n",
    "            arquivo.write(f'{chave}: {valor}\\n')\n",
    "\n",
    "# Exemplo de variáveis de contagem\n",
    "Wrong_Smiles = len(wrongSmiles)\n",
    "Salts = len(indexDropList_salts)\n",
    "Organometals = len(indexDropList_org)\n",
    "Mixtures = len(indexDropList_mix)\n",
    "Polymers = len(indexdroplist_MW)\n",
    "Duplicates = countdprem\n",
    "\n",
    "# Chamada da função para escrever o log\n",
    "escrever_log(Wrong_Smiles=Wrong_Smiles, Salts=Salts, Organometals=Organometals, Mixtures=Mixtures, Polymers=Polymers, Duplicates=Duplicates)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "my-rdkit-env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "abdac575baa5ca75846748934e69150e4d7d5b6c8b3f8d5f951a87886767e227"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}