QA_Extractor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open file\n",
    "\n",
    "f = open('./adobe_extractor/adobe-dc-pdf-services-sdk-extract-python-samples/output/TravelProtection/premium_plus.json')\n",
    "data = json.load(f)\n",
    "filename = \"TravelProtectionPremiumPlus\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>Page</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>General terms and conditions of insurance of A...</td>\n",
       "      <td>The following customer information provides a ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Who is the insurer?</td>\n",
       "      <td>The insurer is AWP P&amp;C S.A., Saint-Ouen (Paris...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Who is the policyholder?</td>\n",
       "      <td>The policyholder is the person designated as s...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What risks are insured and what is the scope o...</td>\n",
       "      <td>Assumption of legal protection exclusively in ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Which persons are insured?</td>\n",
       "      <td>In the case of insurance policies with a durat...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Temporal and geographical scope of the insuran...</td>\n",
       "      <td>The insurance cover basically applies worldwid...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>What are the main exclusions?</td>\n",
       "      <td>assistance or towing has not been organised by...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Which obligations are incumbent on the policyh...</td>\n",
       "      <td>Any need for legal assistance must be reported...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>How much is the premium?</td>\n",
       "      <td>The amount of the premium depends on the risks...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>When does the insurance begin and end?</td>\n",
       "      <td>The beginning and end of the insurance policie...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Right of revocation</td>\n",
       "      <td>The policyholder may revoke the contract withi...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>How does Allianz Travel treat data?</td>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>When processing personal data essential to the...</td>\n",
       "      <td>Table of benefits</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Allianz Travel stores data electronically or p...</td>\n",
       "      <td>The Common provisions to all insurance compone...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1 Insured persons</td>\n",
       "      <td>Persons with permanent residence in Switzerlan...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2 Geographical scope</td>\n",
       "      <td>Unless otherwise specified in the Special prov...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>3 Non-insured events and benefits</td>\n",
       "      <td>Activities related to an insured event are not...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>4 Duties in the event of a claim</td>\n",
       "      <td>The claims forms are available on our website:...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>5 Breach of duties</td>\n",
       "      <td>If the entitled person is in breach of his/her...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>6 Definitions</td>\n",
       "      <td>If the insured or entitled person has been com...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>7</td>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>8 Limitation period</td>\n",
       "      <td>The Swiss Insurance Contract Act (ICA) applies...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>9</td>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>10 Hierarchy of regulations</td>\n",
       "      <td>In the event of linguistic differences between...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>11 Contact address</td>\n",
       "      <td>Insured events</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>The insured person is entitled to a substitute...</td>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>3 Insured benefit</td>\n",
       "      <td>Persons as per section I 1 are insured up to t...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>3 Temporal and geographical scope</td>\n",
       "      <td>K Luggage</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1 Insured amount</td>\n",
       "      <td>The insured personâ€™s luggage is insured incl...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>4 Insured events</td>\n",
       "      <td>delay in delivery/handover by a public transpo...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>5 Insured benefits</td>\n",
       "      <td>The insured person must pay an excess of CHF 2...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>6 Non-insured items</td>\n",
       "      <td>theft, loss and destruction of cash (see insur...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>7 Non-insured events and benefits (in addition...</td>\n",
       "      <td>In the event of late delivery of luggage, cost...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>8 Conduct-related duties when travelling</td>\n",
       "      <td>Valuable items such as privately owned mobile ...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>9 Duties in the event of a claim (in addition ...</td>\n",
       "      <td>L Cash and bank account protection</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>1 Insured amount</td>\n",
       "      <td>The insured amount can be seen in the table of...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2 Temporal and geographical scope</td>\n",
       "      <td>Insurance cover applies worldwide exclusively ...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>3 Insured events</td>\n",
       "      <td>Financial losses incurred by the insured perso...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>4 Insured items</td>\n",
       "      <td>The cash is insured that the insured person wi...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>5 Non-insured events and benefits (in addition...</td>\n",
       "      <td>Losses arising indirectly from misuse, e.g. lo...</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>6 Duties in the event of a claim (in addition ...</td>\n",
       "      <td>O Overview of services excluding assumption of...</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>1 Travel hotline</td>\n",
       "      <td>Notification service for relatives and employe...</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>2 24h medical advisory service</td>\n",
       "      <td>advice on questions related to symptoms of ill...</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>3 Advance payment to a hospital</td>\n",
       "      <td>If the insured person experiences a serious il...</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>4 Credit and customer card blocking service</td>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>The insured person can request assistance in t...</td>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Question  \\\n",
       "0   General terms and conditions of insurance of A...   \n",
       "1                                Who is the insurer?    \n",
       "2                           Who is the policyholder?    \n",
       "3   What risks are insured and what is the scope o...   \n",
       "4                         Which persons are insured?    \n",
       "5   Temporal and geographical scope of the insuran...   \n",
       "6                      What are the main exclusions?    \n",
       "7   Which obligations are incumbent on the policyh...   \n",
       "8                           How much is the premium?    \n",
       "9             When does the insurance begin and end?    \n",
       "10                               Right of revocation    \n",
       "11               How does Allianz Travel treat data?    \n",
       "12  When processing personal data essential to the...   \n",
       "13  Allianz Travel stores data electronically or p...   \n",
       "14                                 1 Insured persons    \n",
       "15                              2 Geographical scope    \n",
       "16                 3 Non-insured events and benefits    \n",
       "17                  4 Duties in the event of a claim    \n",
       "18                                5 Breach of duties    \n",
       "19                                     6 Definitions    \n",
       "20                                                 7    \n",
       "21                               8 Limitation period    \n",
       "22                                                 9    \n",
       "23                       10 Hierarchy of regulations    \n",
       "24                                11 Contact address    \n",
       "25  The insured person is entitled to a substitute...   \n",
       "26                                 3 Insured benefit    \n",
       "27                 3 Temporal and geographical scope    \n",
       "28                                  1 Insured amount    \n",
       "29                                  4 Insured events    \n",
       "30                                5 Insured benefits    \n",
       "31                               6 Non-insured items    \n",
       "32  7 Non-insured events and benefits (in addition...   \n",
       "33          8 Conduct-related duties when travelling    \n",
       "34  9 Duties in the event of a claim (in addition ...   \n",
       "35                                  1 Insured amount    \n",
       "36                 2 Temporal and geographical scope    \n",
       "37                                  3 Insured events    \n",
       "38                                   4 Insured items    \n",
       "39  5 Non-insured events and benefits (in addition...   \n",
       "40  6 Duties in the event of a claim (in addition ...   \n",
       "41                                  1 Travel hotline    \n",
       "42                    2 24h medical advisory service    \n",
       "43                   3 Advance payment to a hospital    \n",
       "44       4 Credit and customer card blocking service    \n",
       "45  The insured person can request assistance in t...   \n",
       "46                                                      \n",
       "\n",
       "                                               Answer  Page  \n",
       "0   The following customer information provides a ...     1  \n",
       "1   The insurer is AWP P&C S.A., Saint-Ouen (Paris...     1  \n",
       "2   The policyholder is the person designated as s...     1  \n",
       "3   Assumption of legal protection exclusively in ...     1  \n",
       "4   In the case of insurance policies with a durat...     1  \n",
       "5   The insurance cover basically applies worldwid...     1  \n",
       "6   assistance or towing has not been organised by...     2  \n",
       "7   Any need for legal assistance must be reported...     2  \n",
       "8   The amount of the premium depends on the risks...     2  \n",
       "9   The beginning and end of the insurance policie...     2  \n",
       "10  The policyholder may revoke the contract withi...     2  \n",
       "11                                                        2  \n",
       "12                                 Table of benefits      3  \n",
       "13  The Common provisions to all insurance compone...     4  \n",
       "14  Persons with permanent residence in Switzerlan...     4  \n",
       "15  Unless otherwise specified in the Special prov...     4  \n",
       "16  Activities related to an insured event are not...     4  \n",
       "17  The claims forms are available on our website:...     4  \n",
       "18  If the entitled person is in breach of his/her...     4  \n",
       "19  If the insured or entitled person has been com...     5  \n",
       "20                                                        5  \n",
       "21  The Swiss Insurance Contract Act (ICA) applies...     5  \n",
       "22                                                        5  \n",
       "23  In the event of linguistic differences between...     5  \n",
       "24                                    Insured events      8  \n",
       "25                                                        8  \n",
       "26  Persons as per section I 1 are insured up to t...     9  \n",
       "27                                         K Luggage     10  \n",
       "28  The insured personâ€™s luggage is insured incl...    10  \n",
       "29  delay in delivery/handover by a public transpo...    10  \n",
       "30  The insured person must pay an excess of CHF 2...    11  \n",
       "31  theft, loss and destruction of cash (see insur...    11  \n",
       "32  In the event of late delivery of luggage, cost...    11  \n",
       "33  Valuable items such as privately owned mobile ...    11  \n",
       "34                L Cash and bank account protection     11  \n",
       "35  The insured amount can be seen in the table of...    11  \n",
       "36  Insurance cover applies worldwide exclusively ...    11  \n",
       "37  Financial losses incurred by the insured perso...    11  \n",
       "38  The cash is insured that the insured person wi...    11  \n",
       "39  Losses arising indirectly from misuse, e.g. lo...    12  \n",
       "40  O Overview of services excluding assumption of...    13  \n",
       "41  Notification service for relatives and employe...    13  \n",
       "42  advice on questions related to symptoms of ill...    13  \n",
       "43  If the insured person experiences a serious il...    13  \n",
       "44                                                       13  \n",
       "45                                                       13  \n",
       "46                                                       13  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Finding out sequence number of titles (identified by blue underline)\n",
    "ele_positions = []\n",
    "for ele in range(len(data[\"elements\"])):\n",
    "    if \"attributes\" in list(data[\"elements\"][ele].keys()):\n",
    "        if \"SpaceAfter\" in list(data[\"elements\"][ele]['attributes'].keys()):\n",
    "            if data[\"elements\"][ele][\"attributes\"]['SpaceAfter']==18:\n",
    "                ele_positions.append(ele)\n",
    "#                 print(\"Question:\", ele)\n",
    "#                 print(data[\"elements\"][ele][\"Text\"])\n",
    "#                 print(\"Answer:\")\n",
    "#                 print(data[\"elements\"][ele+1][\"Text\"])\n",
    "\n",
    "# Retrieving the questions (identified by blue underline) and the answers in between \n",
    "QA_data = {\"Question\":[], \"Answer\":[], \"Page\":[]}\n",
    "\n",
    "for ele in range(len(ele_positions)):\n",
    "    question = str()\n",
    "    answer = str()\n",
    "    page_number = 0\n",
    "    if ele != len(ele_positions)-1:\n",
    "        start = ele_positions[ele] +1\n",
    "        end = ele_positions[ele+1] \n",
    "        question_asked = ele_positions[ele]\n",
    "        if \"Text\" in list(data[\"elements\"][question_asked].keys()):\n",
    "                question = (data[\"elements\"][question_asked][\"Text\"])\n",
    "\n",
    "        for items in (np.arange(start,end,1)):\n",
    "            if \"Text\" in list(data[\"elements\"][items].keys()):\n",
    "                answer = (data[\"elements\"][items][\"Text\"])\n",
    "                page = (data[\"elements\"][items][\"Page\"])\n",
    "    \n",
    "#     print(question)\n",
    "#     print(answer)\n",
    "    QA_data[\"Question\"].append(question)\n",
    "    QA_data[\"Answer\"].append(answer)\n",
    "    QA_data[\"Page\"].append(page)\n",
    "    \n",
    "QA = pd.DataFrame.from_dict(QA_data)\n",
    "QA.to_csv(str(filename) + \".csv\")\n",
    "QA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Table of benefits '"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "QA.iloc[12][\"Answer\"]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}