clean_pdfs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "import os\n",
    "\n",
    "path = 'path-to-data-repository'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doi</th>\n",
       "      <th>bioActivity</th>\n",
       "      <th>collectionSpecie</th>\n",
       "      <th>collectionSite</th>\n",
       "      <th>collectionType</th>\n",
       "      <th>molType</th>\n",
       "      <th>molecularMass</th>\n",
       "      <th>monoisotropicMass</th>\n",
       "      <th>cLogP</th>\n",
       "      <th>tpsa</th>\n",
       "      <th>numberOfLipinskiViolations</th>\n",
       "      <th>numberOfH_bondAcceptors</th>\n",
       "      <th>numberOfH_bondDonors</th>\n",
       "      <th>numberOfRotableBonds</th>\n",
       "      <th>molecularVolume</th>\n",
       "      <th>smile</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Inhibition of Protease</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/AromaticDerivative</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Inhibition of Protease</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/ChemicalCompound</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Inhibition of Protease</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/ChemicalSubstance</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Antiviral</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/AromaticDerivative</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Antiviral</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/ChemicalCompound</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2386</th>\n",
       "      <td>10.1016/S0031-9422(00)94671-5</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2387</th>\n",
       "      <td>10.1016/S0031-9422(00)80444-6</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2388</th>\n",
       "      <td>10.1021/np50032a021</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2389</th>\n",
       "      <td>10.1016/S0031-9422(00)83467-6</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2390</th>\n",
       "      <td>10.1016/0031-9422(80)83208-0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2391 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 doi             bioActivity  \\\n",
       "0     10.1016/j.biortech.2010.01.041  Inhibition of Protease   \n",
       "1     10.1016/j.biortech.2010.01.041  Inhibition of Protease   \n",
       "2     10.1016/j.biortech.2010.01.041  Inhibition of Protease   \n",
       "3     10.1016/j.biortech.2010.01.041               Antiviral   \n",
       "4     10.1016/j.biortech.2010.01.041               Antiviral   \n",
       "...                              ...                     ...   \n",
       "2386   10.1016/S0031-9422(00)94671-5                           \n",
       "2387   10.1016/S0031-9422(00)80444-6                           \n",
       "2388             10.1021/np50032a021                           \n",
       "2389   10.1016/S0031-9422(00)83467-6                           \n",
       "2390    10.1016/0031-9422(80)83208-0                           \n",
       "\n",
       "                      collectionSpecie collectionSite  \\\n",
       "0     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "1     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "2     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "3     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "4     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "...                                ...            ...   \n",
       "2386                                                    \n",
       "2387                                                    \n",
       "2388                                                    \n",
       "2389                                                    \n",
       "2390                                                    \n",
       "\n",
       "                 collectionType                             molType  \\\n",
       "0     Biotransformation Product  http://nubbe.db/AromaticDerivative   \n",
       "1     Biotransformation Product    http://nubbe.db/ChemicalCompound   \n",
       "2     Biotransformation Product   http://nubbe.db/ChemicalSubstance   \n",
       "3     Biotransformation Product  http://nubbe.db/AromaticDerivative   \n",
       "4     Biotransformation Product    http://nubbe.db/ChemicalCompound   \n",
       "...                         ...                                 ...   \n",
       "2386                                                                  \n",
       "2387                                                                  \n",
       "2388                                                                  \n",
       "2389                                                                  \n",
       "2390                                                                  \n",
       "\n",
       "     molecularMass monoisotropicMass        cLogP         tpsa  \\\n",
       "0       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "1       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "2       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "3       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "4       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "...            ...               ...          ...          ...   \n",
       "2386                                                             \n",
       "2387                                                             \n",
       "2388                                                             \n",
       "2389                                                             \n",
       "2390                                                             \n",
       "\n",
       "     numberOfLipinskiViolations numberOfH_bondAcceptors numberOfH_bondDonors  \\\n",
       "0                           0.0                     4.0                  2.0   \n",
       "1                           0.0                     4.0                  2.0   \n",
       "2                           0.0                     4.0                  2.0   \n",
       "3                           0.0                     4.0                  2.0   \n",
       "4                           0.0                     4.0                  2.0   \n",
       "...                         ...                     ...                  ...   \n",
       "2386                                                                           \n",
       "2387                                                                           \n",
       "2388                                                                           \n",
       "2389                                                                           \n",
       "2390                                                                           \n",
       "\n",
       "     numberOfRotableBonds molecularVolume  \\\n",
       "0                     7.0     282,9689941   \n",
       "1                     7.0     282,9689941   \n",
       "2                     7.0     282,9689941   \n",
       "3                     7.0     282,9689941   \n",
       "4                     7.0     282,9689941   \n",
       "...                   ...             ...   \n",
       "2386                                        \n",
       "2387                                        \n",
       "2388                                        \n",
       "2389                                        \n",
       "2390                                        \n",
       "\n",
       "                                            smile  \n",
       "0     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1  \n",
       "1     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1  \n",
       "2     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1  \n",
       "3     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1  \n",
       "4     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1  \n",
       "...                                           ...  \n",
       "2386                                               \n",
       "2387                                               \n",
       "2388                                               \n",
       "2389                                               \n",
       "2390                                               \n",
       "\n",
       "[2391 rows x 16 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('{}query03-05.csv'.format(path))\n",
    "df = df.fillna('')\n",
    "df = df.astype({\n",
    "    'doi': 'str', 'bioActivity': 'str', 'molType': 'str', 'collectionSpecie': 'str', 'collectionSite': 'str',\n",
    "    'collectionType': 'str', 'molecularMass': 'str', 'monoisotropicMass': 'str', 'cLogP': 'str', 'tpsa': 'str',\n",
    "    'numberOfLipinskiViolations': 'str', 'numberOfH_bondAcceptors': 'str', 'numberOfH_bondDonors': 'str',\n",
    "    'numberOfRotableBonds': 'str', 'molecularVolume': 'str',\n",
    "})\n",
    "df = df[(df['molType'].str.contains('http://nubbe.db/')) | (df['molType'] == '')]\n",
    "df = df.drop_duplicates().reset_index(drop=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>smiles</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "      <td>3,4-dihydroxybenzoic acid geranyl ester</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>O=C(C1=CC(O)=C(C=C1)OC/C=C(CC/C=C(C)/C)\\C)O</td>\n",
       "      <td>4-O-geranyl-3-hydroxybenzoic acid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>OC1=C(OC/C=C(C)/CC/C=C(C)/C)C=C(C(O)=O)C=C1</td>\n",
       "      <td>3-O-geranyl-4-hydroxybenzoic acid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OC1=CC=C(C(OC/C=C(C)/CC/C=C(C)/C)=O)C=C1</td>\n",
       "      <td>p-hydroxybenzoic acid-geranyl ester</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>O=C(O)C1=CC=C(OC/C=C(C)/CC/C=C(C)/C)C=C1</td>\n",
       "      <td>p-O-geranyl-benzoic acid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2243</th>\n",
       "      <td>C(C=C)[C@@]12[C@H](C)[C@H]([C@H](C1O)[C@@H](C(...</td>\n",
       "      <td>(7R,8R,1'R,2'R,3'S,4'S)-(Δ8')-2',4'-Dihydroxy-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2244</th>\n",
       "      <td>C(C=C)[C@]12C(O)[C@H]([C@@H]([C@H]1C)c1cc3c(cc...</td>\n",
       "      <td>(7R,8R,1'R,2'R,3'S,4'S)-(Δ8')-2'-Hydroxy-4'-me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2245</th>\n",
       "      <td>C(C=C)[C@]12C(O)[C@H]([C@@H]([C@H]1C)c1cc3c(cc...</td>\n",
       "      <td>(7R,8R,1'R,2'R,3'S,4'S)-(Δ8')-2'-Hydroxy-4'-me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2246</th>\n",
       "      <td>C(C=C)C12C(C)C(C(C1=O)C(=O)C(C2)OC)c1cc(OC)c(c...</td>\n",
       "      <td>(7R,8R,1'S,3'S)-(Δ8')-4-Hydroxy-3,5'-dimethoxy...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2247</th>\n",
       "      <td>C(C=C)[C@@]12[C@H](C)[C@H]([C@H](C1=O)C(=O)C(=...</td>\n",
       "      <td>(7R,8R,1'R,3'S)-(Δ8')-4-Hydroxy-3,5'-dimethoxy...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2248 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 smiles  \\\n",
       "0           O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1   \n",
       "1           O=C(C1=CC(O)=C(C=C1)OC/C=C(CC/C=C(C)/C)\\C)O   \n",
       "2           OC1=C(OC/C=C(C)/CC/C=C(C)/C)C=C(C(O)=O)C=C1   \n",
       "3              OC1=CC=C(C(OC/C=C(C)/CC/C=C(C)/C)=O)C=C1   \n",
       "4              O=C(O)C1=CC=C(OC/C=C(C)/CC/C=C(C)/C)C=C1   \n",
       "...                                                 ...   \n",
       "2243  C(C=C)[C@@]12[C@H](C)[C@H]([C@H](C1O)[C@@H](C(...   \n",
       "2244  C(C=C)[C@]12C(O)[C@H]([C@@H]([C@H]1C)c1cc3c(cc...   \n",
       "2245  C(C=C)[C@]12C(O)[C@H]([C@@H]([C@H]1C)c1cc3c(cc...   \n",
       "2246  C(C=C)C12C(C)C(C(C1=O)C(=O)C(C2)OC)c1cc(OC)c(c...   \n",
       "2247  C(C=C)[C@@]12[C@H](C)[C@H]([C@H](C1=O)C(=O)C(=...   \n",
       "\n",
       "                                                   name  \n",
       "0               3,4-dihydroxybenzoic acid geranyl ester  \n",
       "1                     4-O-geranyl-3-hydroxybenzoic acid  \n",
       "2                     3-O-geranyl-4-hydroxybenzoic acid  \n",
       "3                   p-hydroxybenzoic acid-geranyl ester  \n",
       "4                              p-O-geranyl-benzoic acid  \n",
       "...                                                 ...  \n",
       "2243  (7R,8R,1'R,2'R,3'S,4'S)-(Δ8')-2',4'-Dihydroxy-...  \n",
       "2244  (7R,8R,1'R,2'R,3'S,4'S)-(Δ8')-2'-Hydroxy-4'-me...  \n",
       "2245  (7R,8R,1'R,2'R,3'S,4'S)-(Δ8')-2'-Hydroxy-4'-me...  \n",
       "2246  (7R,8R,1'S,3'S)-(Δ8')-4-Hydroxy-3,5'-dimethoxy...  \n",
       "2247  (7R,8R,1'R,3'S)-(Δ8')-4-Hydroxy-3,5'-dimethoxy...  \n",
       "\n",
       "[2248 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smiles_names = pd.read_csv('{}smiles_names03-05.csv'.format(path))\n",
    "smiles_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doi</th>\n",
       "      <th>bioActivity</th>\n",
       "      <th>collectionSpecie</th>\n",
       "      <th>collectionSite</th>\n",
       "      <th>collectionType</th>\n",
       "      <th>molType</th>\n",
       "      <th>molecularMass</th>\n",
       "      <th>monoisotropicMass</th>\n",
       "      <th>cLogP</th>\n",
       "      <th>tpsa</th>\n",
       "      <th>numberOfLipinskiViolations</th>\n",
       "      <th>numberOfH_bondAcceptors</th>\n",
       "      <th>numberOfH_bondDonors</th>\n",
       "      <th>numberOfRotableBonds</th>\n",
       "      <th>molecularVolume</th>\n",
       "      <th>smile</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Inhibition of Protease</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/AromaticDerivative</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "      <td>3,4-dihydroxybenzoic acid geranyl ester</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Inhibition of Protease</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/ChemicalCompound</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "      <td>3,4-dihydroxybenzoic acid geranyl ester</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Inhibition of Protease</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/ChemicalSubstance</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "      <td>3,4-dihydroxybenzoic acid geranyl ester</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Antiviral</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/AromaticDerivative</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "      <td>3,4-dihydroxybenzoic acid geranyl ester</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Antiviral</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/ChemicalCompound</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "      <td>3,4-dihydroxybenzoic acid geranyl ester</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2386</th>\n",
       "      <td>10.1016/S0031-9422(00)94671-5</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2387</th>\n",
       "      <td>10.1016/S0031-9422(00)80444-6</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2388</th>\n",
       "      <td>10.1021/np50032a021</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2389</th>\n",
       "      <td>10.1016/S0031-9422(00)83467-6</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2390</th>\n",
       "      <td>10.1016/0031-9422(80)83208-0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2395 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 doi             bioActivity  \\\n",
       "0     10.1016/j.biortech.2010.01.041  Inhibition of Protease   \n",
       "1     10.1016/j.biortech.2010.01.041  Inhibition of Protease   \n",
       "2     10.1016/j.biortech.2010.01.041  Inhibition of Protease   \n",
       "3     10.1016/j.biortech.2010.01.041               Antiviral   \n",
       "4     10.1016/j.biortech.2010.01.041               Antiviral   \n",
       "...                              ...                     ...   \n",
       "2386   10.1016/S0031-9422(00)94671-5                           \n",
       "2387   10.1016/S0031-9422(00)80444-6                           \n",
       "2388             10.1021/np50032a021                           \n",
       "2389   10.1016/S0031-9422(00)83467-6                           \n",
       "2390    10.1016/0031-9422(80)83208-0                           \n",
       "\n",
       "                      collectionSpecie collectionSite  \\\n",
       "0     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "1     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "2     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "3     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "4     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "...                                ...            ...   \n",
       "2386                                                    \n",
       "2387                                                    \n",
       "2388                                                    \n",
       "2389                                                    \n",
       "2390                                                    \n",
       "\n",
       "                 collectionType                             molType  \\\n",
       "0     Biotransformation Product  http://nubbe.db/AromaticDerivative   \n",
       "1     Biotransformation Product    http://nubbe.db/ChemicalCompound   \n",
       "2     Biotransformation Product   http://nubbe.db/ChemicalSubstance   \n",
       "3     Biotransformation Product  http://nubbe.db/AromaticDerivative   \n",
       "4     Biotransformation Product    http://nubbe.db/ChemicalCompound   \n",
       "...                         ...                                 ...   \n",
       "2386                                                                  \n",
       "2387                                                                  \n",
       "2388                                                                  \n",
       "2389                                                                  \n",
       "2390                                                                  \n",
       "\n",
       "     molecularMass monoisotropicMass        cLogP         tpsa  \\\n",
       "0       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "1       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "2       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "3       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "4       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "...            ...               ...          ...          ...   \n",
       "2386                                                             \n",
       "2387                                                             \n",
       "2388                                                             \n",
       "2389                                                             \n",
       "2390                                                             \n",
       "\n",
       "     numberOfLipinskiViolations numberOfH_bondAcceptors numberOfH_bondDonors  \\\n",
       "0                           0.0                     4.0                  2.0   \n",
       "1                           0.0                     4.0                  2.0   \n",
       "2                           0.0                     4.0                  2.0   \n",
       "3                           0.0                     4.0                  2.0   \n",
       "4                           0.0                     4.0                  2.0   \n",
       "...                         ...                     ...                  ...   \n",
       "2386                                                                           \n",
       "2387                                                                           \n",
       "2388                                                                           \n",
       "2389                                                                           \n",
       "2390                                                                           \n",
       "\n",
       "     numberOfRotableBonds molecularVolume  \\\n",
       "0                     7.0     282,9689941   \n",
       "1                     7.0     282,9689941   \n",
       "2                     7.0     282,9689941   \n",
       "3                     7.0     282,9689941   \n",
       "4                     7.0     282,9689941   \n",
       "...                   ...             ...   \n",
       "2386                                        \n",
       "2387                                        \n",
       "2388                                        \n",
       "2389                                        \n",
       "2390                                        \n",
       "\n",
       "                                            smile  \\\n",
       "0     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1   \n",
       "1     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1   \n",
       "2     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1   \n",
       "3     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1   \n",
       "4     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1   \n",
       "...                                           ...   \n",
       "2386                                                \n",
       "2387                                                \n",
       "2388                                                \n",
       "2389                                                \n",
       "2390                                                \n",
       "\n",
       "                                         name  \n",
       "0     3,4-dihydroxybenzoic acid geranyl ester  \n",
       "1     3,4-dihydroxybenzoic acid geranyl ester  \n",
       "2     3,4-dihydroxybenzoic acid geranyl ester  \n",
       "3     3,4-dihydroxybenzoic acid geranyl ester  \n",
       "4     3,4-dihydroxybenzoic acid geranyl ester  \n",
       "...                                       ...  \n",
       "2386                                           \n",
       "2387                                           \n",
       "2388                                           \n",
       "2389                                           \n",
       "2390                                           \n",
       "\n",
       "[2395 rows x 17 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.join(smiles_names.set_index('smiles'), on='smile')\n",
    "df = df.fillna('')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/_q/9djqcd1n39x3jty15jbvy2sm0000gn/T/ipykernel_65447/3664528758.py:4: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  all_files_df['doi'] = all_files_df['doi'].str.replace('.pdf','')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doi</th>\n",
       "      <th>bioActivity</th>\n",
       "      <th>collectionSpecie</th>\n",
       "      <th>collectionSite</th>\n",
       "      <th>collectionType</th>\n",
       "      <th>molType</th>\n",
       "      <th>molecularMass</th>\n",
       "      <th>monoisotropicMass</th>\n",
       "      <th>cLogP</th>\n",
       "      <th>tpsa</th>\n",
       "      <th>numberOfLipinskiViolations</th>\n",
       "      <th>numberOfH_bondAcceptors</th>\n",
       "      <th>numberOfH_bondDonors</th>\n",
       "      <th>numberOfRotableBonds</th>\n",
       "      <th>molecularVolume</th>\n",
       "      <th>smile</th>\n",
       "      <th>name</th>\n",
       "      <th>file_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Inhibition of Protease</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/AromaticDerivative</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "      <td>3,4-dihydroxybenzoic acid geranyl ester</td>\n",
       "      <td>10.1016@j.biortech.2010.01.041.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Inhibition of Protease</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/ChemicalCompound</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "      <td>3,4-dihydroxybenzoic acid geranyl ester</td>\n",
       "      <td>10.1016@j.biortech.2010.01.041.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Inhibition of Protease</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/ChemicalSubstance</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "      <td>3,4-dihydroxybenzoic acid geranyl ester</td>\n",
       "      <td>10.1016@j.biortech.2010.01.041.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Antiviral</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/AromaticDerivative</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "      <td>3,4-dihydroxybenzoic acid geranyl ester</td>\n",
       "      <td>10.1016@j.biortech.2010.01.041.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>Antiviral</td>\n",
       "      <td>Piper crassinervium (Piperaceae)</td>\n",
       "      <td>Araraquara/SP</td>\n",
       "      <td>Biotransformation Product</td>\n",
       "      <td>http://nubbe.db/ChemicalCompound</td>\n",
       "      <td>290,354187</td>\n",
       "      <td>290,1517944</td>\n",
       "      <td>4,660999775</td>\n",
       "      <td>66,76100159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>282,9689941</td>\n",
       "      <td>O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1</td>\n",
       "      <td>3,4-dihydroxybenzoic acid geranyl ester</td>\n",
       "      <td>10.1016@j.biortech.2010.01.041.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2299</th>\n",
       "      <td>10.1016/S0031-9422(00)94671-5</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>10.1016@S0031-9422(00)94671-5.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2300</th>\n",
       "      <td>10.1016/S0031-9422(00)80444-6</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>10.1016@S0031-9422(00)80444-6.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2301</th>\n",
       "      <td>10.1021/np50032a021</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>10.1021@np50032a021.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2302</th>\n",
       "      <td>10.1016/S0031-9422(00)83467-6</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>10.1016@S0031-9422(00)83467-6.pdf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2303</th>\n",
       "      <td>10.1016/0031-9422(80)83208-0</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>10.1016@0031-9422(80)83208-0.pdf</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2304 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 doi             bioActivity  \\\n",
       "0     10.1016/j.biortech.2010.01.041  Inhibition of Protease   \n",
       "1     10.1016/j.biortech.2010.01.041  Inhibition of Protease   \n",
       "2     10.1016/j.biortech.2010.01.041  Inhibition of Protease   \n",
       "3     10.1016/j.biortech.2010.01.041               Antiviral   \n",
       "4     10.1016/j.biortech.2010.01.041               Antiviral   \n",
       "...                              ...                     ...   \n",
       "2299   10.1016/S0031-9422(00)94671-5                           \n",
       "2300   10.1016/S0031-9422(00)80444-6                           \n",
       "2301             10.1021/np50032a021                           \n",
       "2302   10.1016/S0031-9422(00)83467-6                           \n",
       "2303    10.1016/0031-9422(80)83208-0                           \n",
       "\n",
       "                      collectionSpecie collectionSite  \\\n",
       "0     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "1     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "2     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "3     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "4     Piper crassinervium (Piperaceae)  Araraquara/SP   \n",
       "...                                ...            ...   \n",
       "2299                                                    \n",
       "2300                                                    \n",
       "2301                                                    \n",
       "2302                                                    \n",
       "2303                                                    \n",
       "\n",
       "                 collectionType                             molType  \\\n",
       "0     Biotransformation Product  http://nubbe.db/AromaticDerivative   \n",
       "1     Biotransformation Product    http://nubbe.db/ChemicalCompound   \n",
       "2     Biotransformation Product   http://nubbe.db/ChemicalSubstance   \n",
       "3     Biotransformation Product  http://nubbe.db/AromaticDerivative   \n",
       "4     Biotransformation Product    http://nubbe.db/ChemicalCompound   \n",
       "...                         ...                                 ...   \n",
       "2299                                                                  \n",
       "2300                                                                  \n",
       "2301                                                                  \n",
       "2302                                                                  \n",
       "2303                                                                  \n",
       "\n",
       "     molecularMass monoisotropicMass        cLogP         tpsa  \\\n",
       "0       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "1       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "2       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "3       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "4       290,354187       290,1517944  4,660999775  66,76100159   \n",
       "...            ...               ...          ...          ...   \n",
       "2299                                                             \n",
       "2300                                                             \n",
       "2301                                                             \n",
       "2302                                                             \n",
       "2303                                                             \n",
       "\n",
       "     numberOfLipinskiViolations numberOfH_bondAcceptors numberOfH_bondDonors  \\\n",
       "0                           0.0                     4.0                  2.0   \n",
       "1                           0.0                     4.0                  2.0   \n",
       "2                           0.0                     4.0                  2.0   \n",
       "3                           0.0                     4.0                  2.0   \n",
       "4                           0.0                     4.0                  2.0   \n",
       "...                         ...                     ...                  ...   \n",
       "2299                                                                           \n",
       "2300                                                                           \n",
       "2301                                                                           \n",
       "2302                                                                           \n",
       "2303                                                                           \n",
       "\n",
       "     numberOfRotableBonds molecularVolume  \\\n",
       "0                     7.0     282,9689941   \n",
       "1                     7.0     282,9689941   \n",
       "2                     7.0     282,9689941   \n",
       "3                     7.0     282,9689941   \n",
       "4                     7.0     282,9689941   \n",
       "...                   ...             ...   \n",
       "2299                                        \n",
       "2300                                        \n",
       "2301                                        \n",
       "2302                                        \n",
       "2303                                        \n",
       "\n",
       "                                            smile  \\\n",
       "0     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1   \n",
       "1     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1   \n",
       "2     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1   \n",
       "3     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1   \n",
       "4     O=C(OC/C=C(C)/CC/C=C(C)/C)C1=CC(O)=C(O)C=C1   \n",
       "...                                           ...   \n",
       "2299                                                \n",
       "2300                                                \n",
       "2301                                                \n",
       "2302                                                \n",
       "2303                                                \n",
       "\n",
       "                                         name  \\\n",
       "0     3,4-dihydroxybenzoic acid geranyl ester   \n",
       "1     3,4-dihydroxybenzoic acid geranyl ester   \n",
       "2     3,4-dihydroxybenzoic acid geranyl ester   \n",
       "3     3,4-dihydroxybenzoic acid geranyl ester   \n",
       "4     3,4-dihydroxybenzoic acid geranyl ester   \n",
       "...                                       ...   \n",
       "2299                                            \n",
       "2300                                            \n",
       "2301                                            \n",
       "2302                                            \n",
       "2303                                            \n",
       "\n",
       "                               file_name  \n",
       "0     10.1016@j.biortech.2010.01.041.pdf  \n",
       "1     10.1016@j.biortech.2010.01.041.pdf  \n",
       "2     10.1016@j.biortech.2010.01.041.pdf  \n",
       "3     10.1016@j.biortech.2010.01.041.pdf  \n",
       "4     10.1016@j.biortech.2010.01.041.pdf  \n",
       "...                                  ...  \n",
       "2299   10.1016@S0031-9422(00)94671-5.pdf  \n",
       "2300   10.1016@S0031-9422(00)80444-6.pdf  \n",
       "2301             10.1021@np50032a021.pdf  \n",
       "2302   10.1016@S0031-9422(00)83467-6.pdf  \n",
       "2303    10.1016@0031-9422(80)83208-0.pdf  \n",
       "\n",
       "[2304 rows x 18 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_files = glob.glob(os.path.join(\"{}pdfs/\".format(path), \"*.pdf\"))\n",
    "all_files_df = pd.DataFrame([file.replace(\"{}pdfs/\".format(path), '') for file in all_files], columns=['file_name'])\n",
    "all_files_df['doi'] = all_files_df['file_name'].str.replace('@', '/')\n",
    "all_files_df['doi'] = all_files_df['doi'].str.replace('.pdf','')\n",
    "dowloaded_df = df.join(all_files_df.set_index('doi'), on='doi', how='inner').reset_index(drop=True)\n",
    "dowloaded_df.to_parquet('{}file-name_query03-05.parquet'.format(path))\n",
    "dowloaded_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>not_downloaded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.1002/1099-1565(200101/02)12:1&lt;64::AID-PCA56...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.5897/AJB08.883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISSN 1424-6376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10.1002/1099-1565(200011/12)11:6&lt;383::AID-PCA5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>10.1002/cbdv.201000011.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10.3987/R-1981-03-0371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>10.1016/0031-9422(95)00696-8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>10.1211/jpp/61.02.0017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10.1002/(SICI)1521-4168(19980901)21:9&lt;513::AID...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10.1078/0944-7113-00016.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>10.1016/S0031-9422(00)94227-5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>10.1016/0031-9422(75)85305-3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>10.3987/R-1985-12-3111</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       not_downloaded\n",
       "0   10.1002/1099-1565(200101/02)12:1<64::AID-PCA56...\n",
       "1                                   10.5897/AJB08.883\n",
       "2                                      ISSN 1424-6376\n",
       "3   10.1002/1099-1565(200011/12)11:6<383::AID-PCA5...\n",
       "4                                                    \n",
       "5                             10.1002/cbdv.201000011.\n",
       "6                              10.3987/R-1981-03-0371\n",
       "7                        10.1016/0031-9422(95)00696-8\n",
       "8                              10.1211/jpp/61.02.0017\n",
       "9   10.1002/(SICI)1521-4168(19980901)21:9<513::AID...\n",
       "10                           10.1078/0944-7113-00016.\n",
       "11                      10.1016/S0031-9422(00)94227-5\n",
       "12                       10.1016/0031-9422(75)85305-3\n",
       "13                             10.3987/R-1985-12-3111"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# not downloaded but on query\n",
    "pd.DataFrame(df['doi'][~df['doi'].isin(all_files_df['doi'])].unique(), columns=['not_downloaded'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>downloaded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.1016/S0031-94220000323-X</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.1016/0031-94229500504-Z</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    downloaded\n",
       "0  10.1016/S0031-94220000323-X\n",
       "1   10.1016/0031-94229500504-Z"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# not on query but downloaded\n",
    "pd.DataFrame(all_files_df['doi'][~all_files_df['doi'].isin(df['doi'])].to_list(), columns=['downloaded'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doi</th>\n",
       "      <th>file_name</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.1016/j.biortech.2010.01.041</td>\n",
       "      <td>10.1016@j.biortech.2010.01.041.pdf</td>\n",
       "      <td>Geranylation of benzoic acid derivatives by en...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.1590/S0100-40422010000200017</td>\n",
       "      <td>10.1590@S0100-40422010000200017.pdf</td>\n",
       "      <td>Quim. Nova, Vol. 33, No. 2, 321-323, 2010\\nArt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10.1590/S0103-50532006000500017</td>\n",
       "      <td>10.1590@S0103-50532006000500017.pdf</td>\n",
       "      <td>J. Braz. Chem. Soc., Vol. 17, No. 5, 929-934, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10.1002/pca.818</td>\n",
       "      <td>10.1002@pca.818.pdf</td>\n",
       "      <td>FLAVONOID GLYCOSIDES AND STYRYLPYRONES FROM CR...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10.1002/ps.478</td>\n",
       "      <td>10.1002@ps.478.pdf</td>\n",
       "      <td>Biological activity of astilbin from ���������...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>385</th>\n",
       "      <td>10.1016/S0031-9422(00)94671-5</td>\n",
       "      <td>10.1016@S0031-9422(00)94671-5.pdf</td>\n",
       "      <td>Short Reports \\n1671 \\nDichrographe Mark III, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>10.1016/S0031-9422(00)80444-6</td>\n",
       "      <td>10.1016@S0031-9422(00)80444-6.pdf</td>\n",
       "      <td>Phytochemrstry, \\nVol. 23, No 6, pp. 1X33-1296...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>387</th>\n",
       "      <td>10.1021/np50032a021</td>\n",
       "      <td>10.1021@np50032a021.pdf</td>\n",
       "      <td>Journal of Natural Products \\nVol. 47. No. 2. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>388</th>\n",
       "      <td>10.1016/S0031-9422(00)83467-6</td>\n",
       "      <td>10.1016@S0031-9422(00)83467-6.pdf</td>\n",
       "      <td>Phytockmistry, \\nVol. 23, No. 8, pp. 1677-1679...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>389</th>\n",
       "      <td>10.1016/0031-9422(80)83208-0</td>\n",
       "      <td>10.1016@0031-9422(80)83208-0.pdf</td>\n",
       "      <td>Phytochemistry, 1980, Vol. 19, pp. 474-476. © ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>390 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 doi                            file_name  \\\n",
       "0     10.1016/j.biortech.2010.01.041   10.1016@j.biortech.2010.01.041.pdf   \n",
       "1    10.1590/S0100-40422010000200017  10.1590@S0100-40422010000200017.pdf   \n",
       "2    10.1590/S0103-50532006000500017  10.1590@S0103-50532006000500017.pdf   \n",
       "3                    10.1002/pca.818                  10.1002@pca.818.pdf   \n",
       "4                     10.1002/ps.478                   10.1002@ps.478.pdf   \n",
       "..                               ...                                  ...   \n",
       "385    10.1016/S0031-9422(00)94671-5    10.1016@S0031-9422(00)94671-5.pdf   \n",
       "386    10.1016/S0031-9422(00)80444-6    10.1016@S0031-9422(00)80444-6.pdf   \n",
       "387              10.1021/np50032a021              10.1021@np50032a021.pdf   \n",
       "388    10.1016/S0031-9422(00)83467-6    10.1016@S0031-9422(00)83467-6.pdf   \n",
       "389     10.1016/0031-9422(80)83208-0     10.1016@0031-9422(80)83208-0.pdf   \n",
       "\n",
       "                                                  text  \n",
       "0    Geranylation of benzoic acid derivatives by en...  \n",
       "1    Quim. Nova, Vol. 33, No. 2, 321-323, 2010\\nArt...  \n",
       "2    J. Braz. Chem. Soc., Vol. 17, No. 5, 929-934, ...  \n",
       "3    FLAVONOID GLYCOSIDES AND STYRYLPYRONES FROM CR...  \n",
       "4    Biological activity of astilbin from ���������...  \n",
       "..                                                 ...  \n",
       "385  Short Reports \\n1671 \\nDichrographe Mark III, ...  \n",
       "386  Phytochemrstry, \\nVol. 23, No 6, pp. 1X33-1296...  \n",
       "387  Journal of Natural Products \\nVol. 47. No. 2. ...  \n",
       "388  Phytockmistry, \\nVol. 23, No. 8, pp. 1677-1679...  \n",
       "389  Phytochemistry, 1980, Vol. 19, pp. 474-476. © ...  \n",
       "\n",
       "[390 rows x 3 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import fitz\n",
    "\n",
    "texts = []\n",
    "for file_name in dowloaded_df['file_name'].unique():\n",
    "    doc = fitz.open(f\"{path}pdfs/{file_name}\")\n",
    "    texts.append('')\n",
    "    for page_number in range(doc.page_count):\n",
    "        page = doc.load_page(page_number)\n",
    "        texts[-1] = texts[-1] + page.get_text(\"text\")\n",
    "texts_df = pd.DataFrame({'doi': dowloaded_df['doi'].unique(), 'file_name': dowloaded_df['file_name'].unique(), 'text': texts})\n",
    "texts_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'FLAVONOID GLYCOSIDES AND STYRYLPYRONES FROM CRYPTOCARYA SPP.\\n93\\nCopyright © 2005 John Wiley & Sons, Ltd.\\nPhytochem. Anal. 16: 93–97 (2005)\\nPHYTOCHEMICAL ANALYSIS\\nPhytochem. Anal. 16, 93–97 (2005)\\nPublished online in Wiley InterScience (www.interscience.wiley.com). DOI: 10.1002.pca.818\\nCopyright © 2005 John Wiley & Sons, Ltd.\\nReceived 2 September 2003\\nRevised 15 February 2004\\nAccepted 15 February 2004\\n* Correspondence to: A. J. Cavalheiro, Núcleo de Bioensaios, Bissíntese e\\nEcoﬁsiologia de Produtos Naturais, Instituto de Química, UNESP, Araraquara-\\nSP 14800-900, Brazil.\\nEmail: albjcava@iq.unesp.br\\nContract/grant sponsor: FAPESP.\\nAn HPLC-PAD Method to Analyse Flavonoid\\nGlycosides and Styrylpyrones from Cryptocarya\\nSpecies (Lauraceae)\\nCláudia J. Nehme, Wagner L. Bastos, Adelson J. de Araújo and Alberto J. Cavalheiro*\\nNúcleo de Bioensaios, Bissíntese e Ecoﬁsiologia de Produtos Naturais, Instituto de Química, UNESP, Araraquara-SP 14800-900, Brazil\\nAn HPLC-PAD method has been developed in order to evaluate simultaneously the main secondary meta-\\nbolites, ﬂavonoid glycosides and styrylpyrones, of leaves of Cryptocarya moschata. The sample preparation,\\nconsisting of extraction, liquid–liquid extraction and centrifugation, requires minimum sample manipulation but\\nproduces high yields with reproducibility, selectivity and simplicity. HPLC on a C18 column presents each class of\\nmetabolites grouped and with good resolution of the main compounds. The experimental conditions can be used\\nto study inter- and intra-speciﬁc variability of secondary metabolites in Cryptocarya spp. Copyright © 2005 John\\nWiley & Sons, Ltd.\\nKeywords: HPLC-PAD; retention index; ﬂavonoid glycosides; styrylpyrones; Cryptocarya moschata; Lauraceae.\\nArcanjo, São Paulo, Brazil). Leaves were dried at 40°C\\nand milled prior to solvent extraction.\\nSolvents used for extraction and ﬂash chromato-\\ngraphy were of AnalaR® grade, whilst those used in\\nHPLC analysis and sample preparation were of HPLC\\ngrade, and were from Mallinckrodt (St Louis, MO,\\nUSA). HPLC-grade water (18 mΩ) was prepared using\\na Barnstead (Dubuque IA, USA) E-pure system.\\nSample preparation. In order to obtain a rapid and cheap\\nprotocol by which to prepare extracts of leaves of C.\\nmoschata, a liquid–liquid extraction method was devel-\\noped to extract and clean up the samples. Milled leaves\\n(120 mg) were weighed into a centrifuge tube and\\nextracted with 4.0 mL of petroleum ether (60–68°C boil-\\ning point) for 30 min with sonication, followed by the\\naddition of 4.0 mL of methanol:10% aqueous acetic acid\\n(1:1, v/v) and sonication for a further 30 min. The result-\\ning mixture was stirred and centrifuged at 1200 g for\\n15 min and an aliquot (1.0 mL) taken from hydro-\\nalcoholic phase, ﬁltered on a 0.45 µm PTFE membrane\\nand submitted to HPLC analysis by injection of 20 µL.\\nIsolation and identiﬁcation of ﬂavonoid glycoside stand-\\nards. Powdered leaf material (400 g) was extracted with\\nmethanol:10% aqueous acetic acid (1:1, v/v) for 60 min\\nwith sonication. The crude extract (15 g) was washed\\nwith petroleum ether (60–80°C boiling point) to remove\\nthe lipophilic compounds, and the residue (10 g) was dis-\\nsolved in methanol:10% aqueous acetic acid (1:1, v/v)\\nand fractioned by ﬂash chromatography over RP18 (40–\\n63 µm) eluted using a step gradient from methanol:1%\\naqueous acetic acid (1:1, v/v) to pure methanol. Sub-\\nfractions were submitted to preparative HPLC using\\na Supelcosil (Belleforte, PA, USA) LC18 column (250 ×\\n22 mm i.d.; 12 µm) eluted with methanol:1% aqueous\\nacetic acid (3:7, v/v) to yield six ﬂavonoid glycosides (F1–\\nF6, Fig. 1) that were identiﬁed by comparison of their\\nUV and 1H- and 13C-NMR data (Table 1) with those pre-\\nviously published (Agrawal, 1989).\\nINTRODUCTION\\nCryptocarya moschata Nees is an arboreal species of\\nsome 25–30 m height that is widely spread in the Atlan-\\ntic forests of Brazil. It is recognised as an important food\\nsource for primates such as Brachyteles arachnoids E.\\nGeoffroy, 1806 (mono-carvoeiro or muriqui). A study of\\nthe genetic structure of four natural populations of this\\nspecies, using isoenzyme markers, showed a high level\\nof genetic diversity when compared with other tropical\\nangiosperm trees. The intra-populational genetic vari-\\nability was found to be more prominent than that exhib-\\nited between populations, and the estimated gene ﬂow\\ncorroborated the pronounced divergence found among\\npopulations (Moraes et al., 1999).\\nPhytochemical studies of the bark of C. moschata have\\nshown that styrylpyrones are the typical secondary meta-\\nbolites present in this species (Cavalheiro and Yoshida,\\n2000). Flavonoid glycosides, as well as styrylpyrones, have\\nbeen detected in leaves, and the qualitative and quantita-\\ntive intra-speciﬁc variability of these secondary meta-\\nbolites has also been determined (Nehme at al., 2002).\\nIn this paper, a protocol of extraction, clean-up and\\nHPLC analysis is described that may be employed to\\nestablish simultaneously the ﬂavonoid and styrylpyrone\\nproﬁles of the leaves of C. moschata with the aim of\\nidentifying chemotypes related to these metabolites.\\nEXPERIMENTAL\\nMaterials. Plant material of Cryptocarya moschata was\\ncollected in the Carlos Botelho State Park (São Miguel\\n94\\nC. J. NEHME ET AL.\\nCopyright © 2005 John Wiley & Sons, Ltd.\\nPhytochem. Anal. 16: 93–97 (2005)\\nFigure 1. Flavonoids glycosides (F1–F6) and styrylpyrones (S1–S8) identiﬁed in leaves from\\nCryptocarya moschata.\\nTable 1. NMR data for the ﬂavonoid glycosides F1–F6 isolated from leaves of Cryptocarya moschata\\nCompounda\\nF1\\nF2\\nF3\\nF4\\nF5\\nF6\\nC\\n13Cb\\n1Hb\\n13Cb\\n1Hb\\n13Cb\\n1Hb\\n13Cb\\n1Hb\\n13Cb\\n1Hb\\n13Cb\\n1Hb\\n2\\n156.3\\n—\\n156.0\\n—\\n156.1\\n—\\n156.5\\n—\\n156.3\\n—\\n156.6\\n—\\n3\\n133.6\\n—\\n133.2\\n—\\n133.1\\n—\\n133.2\\n—\\n133.8\\n—\\n134.2\\n—\\n4\\n177.5\\n—\\n177.2\\n—\\n177.2\\n—\\n177.5\\n—\\n177.6\\n—\\n177.7\\n—\\n5\\n161.3\\n12.59s\\n161.2\\n12.51s\\n161.2\\n12.55s\\n161.2\\n12.62s\\n161.2\\n12.60s\\n161.3\\n12.78s\\n(OH)\\n(OH)\\n(OH)\\n(OH)\\n(OH)\\n(OH)\\n6\\n99.0\\n6.19bs\\n99.2\\n6.20bs\\n99.1\\n6.17bs\\n99.1\\n6.16bs\\n98.6\\n6.81bs\\n99.0\\n6.25bs\\n7\\n164.9\\n—\\n166.0\\n—\\n165.5\\n—\\n165.8\\n—\\n164.0\\n—\\n165.0\\n—\\n8\\n93.7\\n6.40bs\\n93.8\\n6.41d\\n93.7\\n6.37bs\\n93.8\\n6.37bs\\n93.4\\n6.38bs\\n93.8\\n6.44bs\\n(2 Hz)\\n9\\n156.5\\n—\\n156.5\\n—\\n156.4\\n—\\n156.7\\n—\\n156.3\\n—\\n157.2\\n—\\n10\\n103.8\\n—\\n103.4\\n—\\n103.5\\n—\\n103.5\\n—\\n104.0\\n—\\n103.9\\n—\\n1′\\n121.2\\n—\\n121.1\\n—\\n120.9\\n—\\n120.9\\n—\\n121.3\\n—\\n121.1\\n—\\n2′\\n115.3\\n7.56s\\n115.3\\n7.62s\\n115.4\\n7.54s\\n115.5\\n7.48s\\n115.2\\n7.61d\\n115.7\\n7.36s\\n(2 Hz)\\n3′\\n145.0\\n—\\n145.0\\n—\\n145.1\\n—\\n145.3\\n—\\n144.7\\n—\\n145.3\\n—\\n4′\\n148.7\\n—\\n148.8\\n—\\n148.9\\n—\\n148.0\\n—\\n148.5\\n—\\n148.6\\n—\\n5′\\n116.1\\n6.83d\\n116.2\\n6.88s\\n116.1\\n6.85d\\n115.6\\n6.85s\\n116.2\\n6.83d\\n115.5\\n6.94s\\n(8 Hz)\\n(8 Hz)\\n(8 Hz)\\n6’\\n122.0\\n7.67d\\n121.6\\n7.63s\\n121.5\\n7.5d\\n121.6\\n7.54d\\n121.8\\n7.63dd\\n120.8\\n7.31d\\n(8 Hz)\\n(8 Hz)\\n(8 Hz)\\n(2, 8 Hz)\\n(8 Hz)\\n1″\\n102.0\\n5.37d\\n101.1\\n5.50d\\n101.8\\n5.34d\\n107.8\\n5.58s\\n101.4\\n5.25d\\n101.9\\n5.32s\\n(8 Hz)\\n(7 Hz)\\n(7 Hz)\\n(8 Hz)\\n2″\\n71.3\\n3.59m\\n74.2\\n3.30m\\n76.0\\n3.35m\\n82.1\\n4.16m\\n70.7\\n3.72m\\n70.4\\n3.60m\\n3″\\n73.3\\n3.40m\\n76.6\\n3.30m\\n73.6\\n3.22m\\n77.0\\n3.73m\\n71.6\\n3.52m\\n70.6\\n3.30m\\n4″\\n68.0\\n3.67m\\n70.0\\n3.30m\\n69.5\\n3.35m\\n85.9\\n3.59m\\n66.0\\n3.62m\\n71.3\\n3.27m\\n5″\\n75.9\\n3.34m\\n77.5\\n3.30m\\n66.0\\n3.65/2.98m\\n60.7\\n3.32m\\n75.2\\n3.84m\\n70.1\\n4.05m\\n6″\\n60.3\\n3.47m\\n61.0\\n3.30m\\n—\\n—\\n—\\n—\\n60.3\\n3.46/3.70m\\n17.6\\n0.89d\\n(6 Hz)\\na Compound numbering and identiﬁcation corresponds to that shown in Table 2.\\nb Spectra measured in DMSO-d6 at 125 (13C) or 500 MHz (1H).\\nFLAVONOID GLYCOSIDES AND STYRYLPYRONES FROM CRYPTOCARYA SPP.\\n95\\nCopyright © 2005 John Wiley & Sons, Ltd.\\nPhytochem. Anal. 16: 93–97 (2005)\\nStyrylpyrone standards. Standards of the styrylpyrones\\ncryptomoscatones E3 (S1), D1 (S2), D2 (S3), F1 (S4) and\\nE1 (S6), and cryptofolione (S7) (Fig. 1) were obtained\\npreviously from bark of C. moschata (Cavalheiro and\\nYoshida, 2000). Deacetylcryptocaryalactone (S5) and\\ngoniothalamin (S8) were obtained by ﬂash chromato-\\ngraphy and preparative HPLC of hydromethanolic ex-\\ntracts of C. moschata and C. aschersoniana, respectively,\\nand identiﬁed by UV and NMR data.\\nInstrumentation. A Varian (Palo Alto, CA, USA)\\nProStar system consisting of a ternary pump model 240,\\na diode array detector model 330 and an auto-sampler\\nmodel 410 was employed, controlled by a Star chroma-\\ntography workstation (version 5.3). The column used was\\na Supelcosil (Belleforte, PA, USA) LC18 (250 × 4.6 mm\\ni.d.; 5 µm) protected by a corresponding guard-column\\n(20 × 4.6 mm i.d.).\\nRESULTS AND DISCUSSION\\nSample extraction and clean-up\\nThe protocol for sample extraction and clean-up was\\nevaluated for efﬁciency of extraction and reproducibil-\\nity of retention time and peak area. The same sample was\\nsubmitted four times to the extraction procedure as de-\\nscribed above and the experiment was carried out in trip-\\nlicate. The efﬁciency of extraction with respect to each\\nchromatographic peak was calculated by considering as\\n100% the total of all four of the peak areas obtained for\\nthat compound following analysis of the four extractions\\n(Table 1 and Fig. 2). In the ﬁrst extraction, the yield cor-\\nresponding to the 12 main peaks was 86.0 ± 3.3%. No\\nﬂavonoid glycosides or styrylpyrones were detected in\\nthe petroleum ether phase showing the high selectivity\\nof the liquid–liquid extraction step that was used to\\neliminate lipophilic components.\\nDevelopment of the chromatographic protocol\\nIn a previous paper (Nehme et al., 2002) the intraspeciﬁc\\nvariability of secondary metabolites in leaves of C.\\nmoschata was determined using a Shimpack (Southall,\\nUK) C18 column (15 × 0.46 cm i.d.) eluted with phos-\\nphoric acid–triethylamine (pH 2.0; solvent A) and\\nacetonitrile (solvent B) with a linear gradient from 85:15\\n(A:B) to 60:40 in 45 min followed by isocratic elution\\nwith 60:40 (A:B) for 10 min, at a ﬂow rate of 0.7 mL/min.\\nAnalysis of the UV spectra obtained from each chroma-\\ntographic peak by PAD, together with retention time (tR)\\nvalues of standards, showed the presence of ﬂavonoid\\nglycosides (tR between 12 and 21 min) and styrylpyrones\\n(tR between 26 and 38 min.). The three main peaks were\\nidentiﬁed as quercitrin (tR = 20.70 min), cryptomoscatone\\nD2 (tR = 28.90 min) and cryptofolione (tR = 34.73 min).\\nHowever, under these chromatographic conditions sev-\\neral peaks remained unresolved (Nehme et al., 2002).\\nIn order to improve the resolution, a longer (25 cm)\\nC18 column was employed with the elution conditions\\nas described above except for the ﬂow rate, which was\\n1.0 mL/min. The chromatogram obtained in these condi-\\ntions showed 20 well-resolved peaks, with the ﬂavonoids\\ngrouped from 12 to 20 min, and the styrylpyrones\\ngrouped from 24 to 36 min. In addition, at least two un-\\nresolved peaks between 22 and 25 min were tentatively\\nattributed to aporphine alkaloids based on UV spectra\\n(Fig. 3). These chromatographic conditions permitted the\\nanalysis of three classes of secondary metabolites in the\\npolar extracts of leaves from C. moschata.\\nThe identities of the main peaks was established by\\ncomparison of tR values and UV spectra with standards\\ntogether with co-injection studies. In order to facilitate\\nthe identiﬁcation of peaks, a relative retention time with\\nrespect to quercitrin (as internal reference) was deter-\\nmined for each peak (Table 2).\\nThe methodology was applied to analyse the second-\\nary metabolites in leaves of four species of Cryptocarya\\ncollected in the Atlantic rain forest in the state of São\\nPaulo (Brazil). A comparison of the chromatographic\\nproﬁles (Fig. 3) revealed that the pattern of ﬂavonoids\\nwith peaks between 16 and 22 min was very similar\\namong the species with only quantitative differences.\\nC. saligna and C. aschersoniana showed an intense peak\\nfor quercetin-3-O-β-d-galactopyranoside (F1) whilst\\nC. saligna exhibited another intense peak (absent from\\nthe other species) at 24.7 min, attributed to an unidenti-\\nﬁed ﬂavonoid with absorptions at 220, 293 and 335 nm.\\nIn the chromatograms from extracts of C. mandioccana\\nand C. moschata, the most intense ﬂavonoid peak was\\nquercetin-3-O-α-D-rhamnopyranoside (F6). In addition,\\nanother unidentiﬁed ﬂavonoid, labelled as (F) with a tR\\nof 24.8 min, in the chromatograms of C. mandioccana\\nand C. moschata, showed UV spectra with absorptions\\nat 203, 260 and 340 nm, which suggested the presence of\\nthe aglycone quercetin in this ﬂavonoid. An additional\\nunidentiﬁed peak with a tR of 42.7 min in the chromato-\\ngram of C. aschersoniana showed absorptions at 235, 267\\nand 362 nm, which were indicative of a dihydrochalcone.\\nFurthermore, each species was shown to possess a dis-\\ntinguishable proﬁle of styrylpyrones. Thus, C. moschata\\nshowed higher diversity in this class of compounds; C.\\naschersoniana showed only one intense peak with a UV\\nspectra similar to styrylpyrone, which was assigned as\\ngoniothalamin (S8); C. mandioccana showed only small\\npeaks from 24 to 36 min with UV spectra similar to the\\nstyrylpyrones found in C. moschata, and C. saligna ex-\\nhibited no styrylpyrone peaks as based on UV spectra.\\nAll peaks labelled with (S) in the chromatogram in\\nFig. 3 had UV spectra very similar those from identiﬁed\\nstyrylpyrones.\\nFigure 2. Yields of the 13 main compounds obtained in four\\nsequential extractions of the same sample of leaves of Cryp-\\ntocarya moschata (mean values of triplicate experiments).\\n96\\nC. J. NEHME ET AL.\\nCopyright © 2005 John Wiley & Sons, Ltd.\\nPhytochem. Anal. 16: 93–97 (2005)\\nFigure 3. Chromatographic proﬁles of four species of Cryptocarya growing\\nin the Atlantic rain forest in the state of São Paulo, Brazil. Peaks labelled\\nF, A and S were associated with, respectively, ﬂavonoids, alkaloids and\\nstyrylpyrones; the peak labelled NI was not identiﬁed. The inserts show\\ntypical UV spectra (PAD) of compound classes F, A and S.\\nTable 2. Reproducibility of retention time, relative retention time, peak area and yield of ﬂavonoid glycosides and styrylpyrones\\n(data from ﬁrst extraction only) from leaves of Cryptocarya moschata obtained by liquid–liquid extraction\\nRelative\\nretention time\\nRetention\\n(in relation to\\nArea ± σ\\nYield ± σ\\nPeak\\nCompound\\ntime (min)\\nquercitrin)\\n(arbitrary units)\\n(%)\\nF1\\nQuercetin-3-O-β-D-galactopyranoside\\n17.09 ± 0.08\\n0.83\\n481,887 ± 12\\n75.6 ± 1.7\\nF2\\nQuercetin-3-O-β-D-glucopyranoside\\n17.52 ± 0.09\\n0.84\\n415,356 ± 28\\n88.1 ± 0.6\\nF3\\nQuercetin-3-O-β-D-xylopyranoside\\n18.82 ± 0.05\\n0.90\\n180,651 ± 10\\n87.4 ± 1.3\\nF4\\nQuercetin-3-O-α-D-arabinopyranoside\\n19.61 ± 0.05\\n0.94\\n295,680 ± 11\\n84.9 ± 1.1\\nF5\\nQuercetin-3-O-β-D-allopyranoside\\n20.29 ± 0.07\\n0.97\\n131,345 ± 9\\n89.7 ± 1.6\\nF6\\nQuercetin-3-O-α-D-rhamnopyranoside\\n20.84 ± 0.08\\n1.00\\n2,025,540 ± 137\\n88.7 ± 1.0\\n(quercitrin)\\nS1\\nCryptomoschatone E3\\n25.88 ± 0.09\\n1.24\\n1,007,390 ± 59\\n88.8 ± 1.2\\nS2\\nCryptomoschatone D1\\n28.09 ± 0.08\\n1.34\\n2,320,990 ± 148\\n87.5 ± 1.6\\nS3\\nCryptomoschatone D2\\n28.41 ± 0.08\\n1.36\\n314,828 ± 21\\n85.6 ± 0.8\\nS4\\nCryptomoschatone F1\\n30.22 ± 0.09\\n1.45\\n712,126 ± 49\\n88.3 ± 1.4\\nS5\\nDeacetyl-cryptocaryalactone\\n32.04 ± 0.08\\n1.53\\n714,667 ± 50\\n85.3 ± 1.3\\nS6 +++++ S7\\nCryptomoschatone E1 and cryptofolione\\n33.26 ± 0.13\\n1.59\\n1,480,692 ± 174\\n90.2 ± 1.7\\nFLAVONOID GLYCOSIDES AND STYRYLPYRONES FROM CRYPTOCARYA SPP.\\n97\\nCopyright © 2005 John Wiley & Sons, Ltd.\\nPhytochem. Anal. 16: 93–97 (2005)\\nIn conclusion, the proposed extraction and analytical\\nmethodology will be useful in studying the composi-\\ntion of ﬂavonoids and styrylpyrones in Cryptocarya spe-\\ncies, assisting in taxonomic and physiological studies,\\nincluding inter- and intra-speciﬁc variability of second-\\nary metabolites.\\nAcknowledgements\\nThe authors are grateful to FAPESP for ﬁnancial support and a PhD\\nscholarship to Cláudia J. Nehme and Wagner L. Bastos, and to CNPq\\nfor a PhD scholarship to Adelson J. de Araújo. The authors also wish\\nto acknowledge the Instituto Florestal for authorising the collection of\\nC. moschata samples at Carlos Botelho state park.\\nREFERENCES\\nAgrawal PK. 1989. Carbon-13 NMR of Flavonoids. Elsevier:\\nAmsterdam.\\nCavalheiro AJ, Yoshida, M. 2000. 6-[ω-Arylalkenyl]-5,6-\\ndihydro-α-pyrones \\nfrom \\nCryptocarya \\nmoschata\\n(Lauraceae). Phytochemistry 53: 811–819.\\nMoraes PLR, Monteiro R, Vencovsky R. 1999. Conservação\\ngenética de populações de Cryptocarya moschata Nees\\n(Lauraceae) na mata Atlântica do estado de São Paulo.\\nRev Bras Bot 22: 237–248.\\nNehme CJ, Moraes PLR, Cavalheiro AJ. 2002. Intra-\\npopulational variability of styrylpyrones in leaves of\\nCryptocarya moschata Nees (Lauraceae) from Carlos\\nBotelho state park. Biochem System Ecol 30: 613–\\n616.\\n'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts_df['text'].iloc[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts_df.to_parquet(\"{}texts_query03-05.parquet\".format(path))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "987c8e17c55c322126b93cdde51aa2aa416046dfb8d84df2dae521860233502c"
  },
  "kernelspec": {
   "display_name": "Python 3.8.13 ('biologist')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}