Python Demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d202b4b2-a570-46d2-8f8a-8d1023b84d92",
   "metadata": {},
   "source": [
    "### Requirements\n",
    "* `pip install pandas`\n",
    "* `pip install numpy`\n",
    "* `pip install pyvis`\n",
    "* `pip install networkx`\n",
    "* `pip install nltk`\n",
    "* `pip install scikit-learn`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2f6f2c83-a744-47f1-b88d-0d9a2cf91350",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/calummacdonald/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package punkt_tab to\n",
      "[nltk_data]     /Users/calummacdonald/nltk_data...\n",
      "[nltk_data]   Package punkt_tab is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from itertools import combinations\n",
    "from pyvis.network import Network\n",
    "import networkx as nx\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import copy\n",
    "import nltk\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "\n",
    "nltk.download('punkt')\n",
    "nltk.download('punkt_tab')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f876849-4f87-420a-a9f4-5970f807950d",
   "metadata": {},
   "source": [
    "### Load data\n",
    "\n",
    "For the hackathon, we have provided extracts of three types of \"entity\" data we store on the HDRUK gateway:\n",
    "* **datasets** - metadata about healthdata datasets\n",
    "* **publications** - healthdata publications\n",
    "* **tools** - analysis scripts, software packages, various other software based tools\n",
    "\n",
    "Load them up from the provided json files:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c75005bc-75d1-4805-bc68-1b0b2897c34d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1059, 3199, 243)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets = []\n",
    "publications = []\n",
    "tools = []\n",
    "with open ('./data/datasets.json') as f:\n",
    "    datasets = json.load(f)\n",
    "with open ('./data/publications.json') as f:\n",
    "    publications = json.load(f)\n",
    "with open ('./data/tools.json') as f:\n",
    "    tools = json.load(f)\n",
    "\n",
    "len(datasets), len(publications), len(tools)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd99738d-35f3-49ef-a0dd-8f3aa0fe2689",
   "metadata": {},
   "source": [
    "#### Example dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "700f52bb-f2e0-4589-b53a-4eb55c302bc1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "      \"id\": 11,\n",
      "      \"metadata\": {\n",
      "            \"observations\": [\n",
      "                  {\n",
      "                        \"measuredValue\": 1,\n",
      "                        \"measuredProperty\": \"Count\",\n",
      "                        \"observationDate\": \"2021-06-16\",\n",
      "                        \"disambiguatingDescription\": \"Count of completed (processed) COVID-19 antigen tests carried out in NHS hospitals in Northern Ireland\",\n",
      "                        \"observedNode\": \"Events\"\n",
      "                  }\n",
      "            ],\n",
      "            \"omics\": null,\n",
      "            \"tissuesSampleCollection\": [],\n",
      "            \"structuralMetadata\": [],\n",
      "            \"coverage\": {\n",
      "                  \"followUp\": \"Unknown\",\n",
      "                  \"datasetCompleteness\": null,\n",
      "                  \"typicalAgeRange\": \"0-150\",\n",
      "                  \"pathway\": \"Dataset relates to a single type of antigen testing\",\n",
      "                  \"spatial\": \"United Kingdom,Northern Ireland\"\n",
      "            },\n",
      "            \"summary\": {\n",
      "                  \"description\": \"This dataset is extracted from the Laboratory Information Systems in Northern Ireland hospitals on a daily basis. It contains details of COVID-19 antigen tests carried out in each of the hospital laboratories, including those processed on behalf of primary care, social care and community settings.\\n\\nhttps://www.gov.uk/government/publications/coronavirus-covid-19-testing-data-methodology/covid-19-testing-data-methodology-note\",\n",
      "                  \"doiName\": null,\n",
      "                  \"datasetType\": \"Health and disease\",\n",
      "                  \"shortTitle\": \"COVID antigen testing - Pillar 1\",\n",
      "                  \"title\": \"COVID antigen testing - Pillar 1\",\n",
      "                  \"inPipeline\": \"Not available\",\n",
      "                  \"contactPoint\": \"hbs@hscni.net\",\n",
      "                  \"controlledKeywords\": null,\n",
      "                  \"abstract\": \"Details of completed (processed) COVID-19 antigen tests carried out in NHS hospitals in Northern Ireland.\",\n",
      "                  \"datasetSubType\": \"Not applicable\",\n",
      "                  \"keywords\": \"Northern Ireland;,;COVID-19;,;Antigen;,;Pillar 1;,;NCS;,;Occupation;,;National Core Study\",\n",
      "                  \"populationSize\": -1,\n",
      "                  \"publisher\": {\n",
      "                        \"name\": \"Health And Social Care Northern Ireland  Honest Broker Service\",\n",
      "                        \"gatewayId\": \"054370d2-ed54-434f-9b33-bdbbbdb69481\"\n",
      "                  }\n",
      "            },\n",
      "            \"required\": {\n",
      "                  \"revisions\": [\n",
      "                        {\n",
      "                              \"url\": \"https://web.prod.hdruk.cloud//dataset/11?version=1.0.0\",\n",
      "                              \"version\": \"1.0.0\"\n",
      "                        }\n",
      "                  ],\n",
      "                  \"version\": \"4.0.0\",\n",
      "                  \"modified\": \"2024-10-08T11:27:59.139137Z\",\n",
      "                  \"gatewayPid\": \"64a2b13f-4eec-45b2-b3b5-32d1d20a9788\",\n",
      "                  \"gatewayId\": \"11\",\n",
      "                  \"issued\": \"2024-10-08T11:27:59.139123Z\"\n",
      "            },\n",
      "            \"provenance\": {\n",
      "                  \"origin\": {\n",
      "                        \"imageContrast\": \"Not stated\",\n",
      "                        \"purpose\": \"Care\",\n",
      "                        \"collectionSituation\": \"Secondary care - Outpatients;,;Services\",\n",
      "                        \"source\": \"LIMS\"\n",
      "                  },\n",
      "                  \"temporal\": {\n",
      "                        \"accrualPeriodicity\": \"Irregular\",\n",
      "                        \"distributionReleaseDate\": null,\n",
      "                        \"endDate\": null,\n",
      "                        \"startDate\": \"2020-01-05\",\n",
      "                        \"timeLag\": \"Not applicable\"\n",
      "                  }\n",
      "            },\n",
      "            \"linkage\": {\n",
      "                  \"dataUses\": null,\n",
      "                  \"publicationUsingDataset\": null,\n",
      "                  \"investigations\": null,\n",
      "                  \"tools\": null,\n",
      "                  \"associatedMedia\": null,\n",
      "                  \"syntheticDataWebLink\": null,\n",
      "                  \"isGeneratedUsing\": null,\n",
      "                  \"publicationAboutDataset\": null,\n",
      "                  \"datasetLinkage\": {\n",
      "                        \"isDerivedFrom\": null,\n",
      "                        \"isMemberOf\": null,\n",
      "                        \"linkedDatasets\": null,\n",
      "                        \"isPartOf\": null\n",
      "                  },\n",
      "                  \"isReferenceIn\": null\n",
      "            },\n",
      "            \"demographicFrequency\": null,\n",
      "            \"accessibility\": {\n",
      "                  \"usage\": {\n",
      "                        \"resourceCreator\": {\n",
      "                              \"name\": \"ACKNOWLEDGEMENT The authors would like to acknowledge the help provided by the staff of the Honest Broker Service (HBS) within the Business Services Organisation Northern Ireland (BSO). The HBS is funded by the BSO and the Department of Health (DoH). The authors alone are responsible for the interpretation of the data and any views or opinions presented are solely those of the author and do not necessarily represent those of the BSO.\",\n",
      "                              \"gatewayId\": null,\n",
      "                              \"rorId\": null\n",
      "                        },\n",
      "                        \"dataUseLimitation\": \"Research use only\",\n",
      "                        \"dataUseRequirement\": \"Project-specific restrictions\"\n",
      "                  },\n",
      "                  \"access\": {\n",
      "                        \"accessServiceCategory\": \"TRE/SDE\",\n",
      "                        \"accessRights\": \"https://bso.hscni.net/directorates/digital-operations/honest-broker-service/\",\n",
      "                        \"dataProcessor\": \"HSC Honest Broker Service\",\n",
      "                        \"jurisdiction\": \"GB-NIR\",\n",
      "                        \"deliveryLeadTime\": \"Not applicable\",\n",
      "                        \"accessRequestCost\": \"https://bso.hscni.net/wp-content/uploads/2023/09/Researcher-Pack.zip\",\n",
      "                        \"dataController\": \"HSC Health Trusts (Belfast, Northern, South Eastern, Southern and Western)\",\n",
      "                        \"accessService\": \"Remote access can now be granted via an HSCNI tenancy of the UK Secure e-Research Platform (UK SeRP), the HBS still offer the physical safe setting in Belfast for local research.\"\n",
      "                  },\n",
      "                  \"formatAndStandards\": {\n",
      "                        \"languages\": \"en\",\n",
      "                        \"conformsTo\": null,\n",
      "                        \"formats\": \"text;,;csv\",\n",
      "                        \"vocabularyEncodingSchemes\": \"LOCAL\"\n",
      "                  }\n",
      "            }\n",
      "      }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print (json.dumps(datasets[10],indent=6))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3b49921-bbc4-4f6a-b6d6-1a179ecaacec",
   "metadata": {},
   "source": [
    "This metadata conforms to the HDRUK standard Gateway Data Model (GWDM) version `2.0`\n",
    "\n",
    "To understand the meeting of the different fields in our schema, you can use our [HDR schemata](https://hdruk.github.io/schemata-2/GWDM/2.0/) documentation "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c599a87-901a-4850-a294-c0132fd77900",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "#### Example publication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "42f39321-a18b-4726-b3be-b852563f0468",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "      \"publication_type\": \"Research articles\",\n",
      "      \"paper_title\": \"The role of health and social factors in education outcome: A record-linked electronic birth cohort analysis.\",\n",
      "      \"id\": 1,\n",
      "      \"year_of_publication\": \"2019\",\n",
      "      \"journal_name\": \"PloS one\",\n",
      "      \"authors\": \"Evans A, Dunstan F, Fone DL, Bandyopadhyay A, Schofield B, Demmler JC, Rahman MA, Lyons RA, Paranjothy S.\",\n",
      "      \"paper_doi\": \"https://doi.org/10.1371/journal.pone.0220771\",\n",
      "      \"abstract\": \"<h4>Background and objective</h4>Health status in childhood is correlated with educational outcomes. Emergency hospital admissions during childhood are common but it is not known how these unplanned breaks from schooling impact on education outcomes. We hypothesised that children who had emergency hospital admissions had an increased risk of lower educational attainment, in addition to the increased risks associated with other health, social and school factors.<h4>Methods</h4>This record-linked electronic birth cohort, included children born in Wales between 1 January 1998 and 31 August 2001. We fitted multilevel logistic regression models grouped by schools, to determine whether emergency hospital inpatient admission before age 7 years was associated with the educational outcome of not attaining the expected level in a teacher-based assessment at age 7 years (KS1). We adjusted for pregnancy, perinatal, socio-economic, neighbourhood, pupil mobility and school-level factors.<h4>Results</h4>The cohort comprised 64 934 children. Overall, 4680 (7.2%) did not attain the expected educational level. Emergency admission to hospital was associated with poor educational attainment (OR 1.12 95% Credible Interval (CI) 1.05, 1.20 for all causes during childhood, OR 1.19 95%CI 1.07, 1.32 for injuries and external causes and OR 1.31 95%CI 1.04, 1.22 for admissions during infancy), after adjusting for known determinants of education outcomes such as extreme prematurity, being small for gestational age and socio-economic indicators, such as eligibility for free school meals.<h4>Conclusion</h4>Emergency inpatient hospital admission during childhood, particularly during infancy or for injuries and external causes was associated with an increased risk of lower education attainment at age 7 years, in addition to the effects of pregnancy factors (gestational age, birthweight) and social deprivation. These findings support the need for injury prevention measures and additional support in school for affected children to help them to achieve their potential.\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print (json.dumps(publications[0],indent=6))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c178656a-e03e-497e-b461-3a6d6daaedc4",
   "metadata": {},
   "source": [
    "#### Example Tool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "738a7d75-0553-4f53-9066-7cab73b1a710",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "      \"name\": \"Test Tool\",\n",
      "      \"id\": 1,\n",
      "      \"description\": \"This is a test\",\n",
      "      \"tech_stack\": null,\n",
      "      \"programming_languages\": [],\n",
      "      \"type_category\": [],\n",
      "      \"publications\": [],\n",
      "      \"url\": \"https://example.com\",\n",
      "      \"results_insights\": null,\n",
      "      \"category\": {\n",
      "            \"updated_at\": \"2024-10-08T11:35:24.000000Z\",\n",
      "            \"enabled\": true,\n",
      "            \"id\": 6,\n",
      "            \"created_at\": \"2024-10-08T11:35:24.000000Z\",\n",
      "            \"name\": \"Code snippet\"\n",
      "      },\n",
      "      \"tag\": [],\n",
      "      \"license\": {\n",
      "            \"updated_at\": \"2024-10-08T11:03:09.000000Z\",\n",
      "            \"origin\": \"HDR\",\n",
      "            \"valid_until\": null,\n",
      "            \"label\": \"Available upon request\",\n",
      "            \"created_at\": \"2024-10-08T11:03:09.000000Z\",\n",
      "            \"definition\": \"Access to the software/code/tool is made available via a request and approval mechanism. The entity posting the entry can provide more information on access limitations or use.\",\n",
      "            \"valid_since\": \"2024-04-15 00:00:00\",\n",
      "            \"deleted_at\": null,\n",
      "            \"verified\": 1,\n",
      "            \"id\": 1,\n",
      "            \"code\": \"HDR_CATEGORY_AVAILABLE_UPON_REQUEST\"\n",
      "      },\n",
      "      \"programming_packages\": []\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print (json.dumps(tools[0],indent=6))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "130d08e2-0850-4dab-97a4-af9eab551f92",
   "metadata": {},
   "source": [
    "#### Linkages\n",
    "\n",
    "We have also provided (in JSON) linakges that we have already curated for:\n",
    "\n",
    "* **datasets <--> datasets**\n",
    "* **datasets <--> publications**\n",
    "* **datasets <--> tools**\n",
    "* **publications <--> tools** \n",
    "\n",
    "These are given as links between a __source__ and a __target__, each entity (dataset, publication, tool) has a `id` that these refer to"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6ef68d20-f75e-41bb-bf9a-ac67c4e5be29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target_dataset_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_dataset_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>719</th>\n",
       "      <td>722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>809</th>\n",
       "      <td>808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811</th>\n",
       "      <td>808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>814</th>\n",
       "      <td>808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>912</th>\n",
       "      <td>687</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>736 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   target_dataset_id\n",
       "source_dataset_id                   \n",
       "16                                20\n",
       "19                                20\n",
       "31                                15\n",
       "51                                78\n",
       "54                               700\n",
       "...                              ...\n",
       "719                              722\n",
       "809                              808\n",
       "811                              808\n",
       "814                              808\n",
       "912                              687\n",
       "\n",
       "[736 rows x 1 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_to_dataset = pd.read_json(\"./data/linkages_dataset_to_dataset.json\").set_index('source_dataset_id')\n",
    "dataset_to_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "23be9342-ae13-4541-88a1-ceb65206ce2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target_publication_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_dataset_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>880</th>\n",
       "      <td>2386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>880</th>\n",
       "      <td>3015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881</th>\n",
       "      <td>2343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>892</th>\n",
       "      <td>2939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>990</th>\n",
       "      <td>3167</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>762 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   target_publication_id\n",
       "source_dataset_id                       \n",
       "11                                  1411\n",
       "14                                   523\n",
       "14                                  1537\n",
       "14                                  1664\n",
       "15                                  1406\n",
       "...                                  ...\n",
       "880                                 2386\n",
       "880                                 3015\n",
       "881                                 2343\n",
       "892                                 2939\n",
       "990                                 3167\n",
       "\n",
       "[762 rows x 1 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_to_publication = pd.read_json(\"./data/linkages_dataset_to_publication.json\").set_index('source_dataset_id')\n",
    "dataset_to_publication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e4c6d26f-3de6-4ba6-8fdb-bc9bf7aadae1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target_tool_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_dataset_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>778</th>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>413</th>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>728</th>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   target_tool_id\n",
       "source_dataset_id                \n",
       "778                            93\n",
       "23                            102\n",
       "283                           102\n",
       "413                           102\n",
       "728                           102"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_to_tool = pd.read_json(\"./data/linkages_dataset_to_tool.json\").set_index('source_dataset_id')\n",
    "dataset_to_tool.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "055551bb-d68a-44e4-b617-fd0605441b8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target_publication_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_tool_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>3166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>3166</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                target_publication_id\n",
       "source_tool_id                       \n",
       "234                               296\n",
       "242                              3166\n",
       "243                              3166"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tool_to_publication = pd.read_json(\"./data/linkages_tool_to_publication.json\").set_index('source_tool_id')\n",
    "tool_to_publication"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64a39ed7-7af3-4067-bdc0-774b33d4d5f7",
   "metadata": {},
   "source": [
    "### Building a Simple Network Graph\n",
    "\n",
    "Given the data we have provided, we can build a simple network graph to visualise the existing entities and linkages between them:\n",
    "\n",
    "First, add all datasets as nodes..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "94e0f243-b689-4243-8db2-d239031fe2a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1044"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "G = nx.Graph()\n",
    "\n",
    "for entry in datasets:\n",
    "    node_id = f\"dataset_{entry['id']}\"\n",
    "    short_title = entry.get('metadata', {}).get('summary', {}).get('shortTitle', f\"Dataset {node_id}\")\n",
    "    G.add_node(node_id, title=short_title, label=short_title, group=1)\n",
    "\n",
    "len(G.nodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd3e94a3-3cad-4968-b24d-b00c72bb2091",
   "metadata": {},
   "source": [
    "Add linkages as edges..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "17335d24-bf34-4afe-9dca-6ee4076857e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "for _, row in dataset_to_dataset.iterrows():\n",
    "    source = f\"dataset_{row.name}\"\n",
    "    target = f\"dataset_{row['target_dataset_id']}\"\n",
    "    G.add_edge(source, target, group=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98e8eecf-53d7-4ed6-8f1e-08d3b5159b47",
   "metadata": {},
   "source": [
    "Add publications and linkages between them and datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "700044c7-1ff4-499f-9705-7765dd6b9fc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4242"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for entry in publications:\n",
    "    node_id = f\"publication_{entry['id']}\"\n",
    "    if (doi := entry.get('paper_doi',None)):\n",
    "        doi = doi.replace(\"https://doi.org/\",\"\")\n",
    "        G.add_node(node_id, title=doi, label=doi, group=2)\n",
    "\n",
    "for _, row in dataset_to_publication.iterrows():\n",
    "    source = f\"dataset_{row.name}\"\n",
    "    target = f\"publication_{row['target_publication_id']}\"\n",
    "    G.add_edge(source, target, group=2)\n",
    "\n",
    "len(G.nodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a00c3b45-8aac-46ee-a3d5-96430f7b088f",
   "metadata": {},
   "source": [
    "Do the same for tools..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a4068d97-4da7-4dff-854a-9c43217eb172",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4485"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for entry in tools:\n",
    "    node_id = f\"tool_{entry['id']}\"\n",
    "    if (name := entry.get('name',None)):\n",
    "        G.add_node(node_id, title=name, label=name, group=3)\n",
    "\n",
    "for _, row in dataset_to_tool.iterrows():\n",
    "    source = f\"dataset_{row.name}\"\n",
    "    target = f\"tool_{row['target_tool_id']}\"\n",
    "    G.add_edge(source, target, group=3)\n",
    "\n",
    "for _, row in tool_to_publication.iterrows():\n",
    "    source = f\"tool_{row.name}\"\n",
    "    target = f\"publication_{row['target_publication_id']}\"\n",
    "    G.add_edge(source, target, group=3)\n",
    "\n",
    "len(G.nodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64f053a1-d37b-493a-9ca4-22c116dc63e8",
   "metadata": {},
   "source": [
    "Make a copy of the graph with all entity nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "74e34d6c-0344-4c1e-a949-5d4e37fd08a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "Gorig = copy.deepcopy(G) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e465667-e425-4cae-8813-27f2b0b0633a",
   "metadata": {},
   "source": [
    "Remove isolated nodes - in the gateway we have a lot of entities that have no linkages (yet!)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d47ad46e-cf1f-46b9-8b33-a577a052b657",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "801"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isolated_nodes = list(nx.isolates(G))\n",
    "G.remove_nodes_from(isolated_nodes)\n",
    "len(G.nodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b95ec02a-0385-4fd1-a7d3-83826093db7f",
   "metadata": {},
   "source": [
    "Draw a visualisation of this graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "14ca90ba-0d7c-4b37-b0a7-d368da73791e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: When  cdn_resources is 'local' jupyter notebook has issues displaying graphics on chrome/safari. Use cdn_resources='in_line' or cdn_resources='remote' if you have issues viewing graphics in a notebook.\n",
      "./public/direct_orginal_linkages.html\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"600px\"\n",
       "            src=\"./public/direct_orginal_linkages.html\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x17cea0fe0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = Network(notebook=True)\n",
    "net.from_nx(G)\n",
    "\n",
    "net.barnes_hut(\n",
    "    gravity=-2000,\n",
    "    central_gravity=0.3,\n",
    "    spring_length=95,\n",
    "    spring_strength=0.04,\n",
    "    damping=0.09,\n",
    "    overlap=0.1\n",
    ")\n",
    "\n",
    "net.show(\"./public/direct_orginal_linkages.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a096b0c-22ae-41dd-be4a-6db53de8a5b8",
   "metadata": {},
   "source": [
    "### Finding additional linkages\n",
    "\n",
    "Now we can explore trying to build up more indirect linkages\n",
    "\n",
    "Firstly, in a simple example, we can extract all authors from the publications "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bccdad0c-045e-40b0-93bd-4ac696a240ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Bandyopadhyay A',\n",
       " 'Demmler JC',\n",
       " 'Dunstan F',\n",
       " 'Evans A',\n",
       " 'Fone DL',\n",
       " 'Lyons RA',\n",
       " 'Paranjothy S.',\n",
       " 'Rahman MA',\n",
       " 'Schofield B'}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "publication_authors = [\n",
    "    set([\n",
    "        auth.lstrip()\n",
    "        for auth in \n",
    "        pub[\"authors\"].split(\",\")\n",
    "    ]) \n",
    "    for pub in publications\n",
    "]\n",
    "publication_authors[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90cef695-9061-445b-9c2d-88bd2e253a7b",
   "metadata": {},
   "source": [
    "Build a similarity matrix based on how many overlapping authors two papers have.."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "aed2e1f1-a7ff-4e0c-8c48-34a5882537e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       ...,\n",
       "       [0.        , 0.        , 0.1875    , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.07692308, 0.        , ..., 0.15384615, 0.        ,\n",
       "        0.        ],\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n = len(publication_authors)\n",
    "similarity_matrix = np.zeros((n, n))\n",
    "\n",
    "for i, j in combinations(range(n), 2):\n",
    "    common_authors = publication_authors[i] & publication_authors[j]  \n",
    "    min_authors = min(len(publication_authors[i]), len(publication_authors[j])) \n",
    "    similarity_matrix[i, j] = similarity_matrix[j, i] = len(common_authors) / min_authors if min_authors > 0 else 0\n",
    "\n",
    "similarity_matrix = np.tril(similarity_matrix)\n",
    "np.fill_diagonal(similarity_matrix, 0)\n",
    "    \n",
    "similarity_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "987108c2-992d-4957-a25f-42681d7af586",
   "metadata": {},
   "source": [
    "Find those publications in which there is a >80% overlap in the authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "32359600-66b0-4f1f-9a3b-b541c7dced18",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "high_similarity_indices = np.argwhere((similarity_matrix >= 0.8))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42ae32a9-c655-4305-a6df-5134fa099150",
   "metadata": {},
   "source": [
    "For example..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "411a4ff4-ff93-4424-bada-5cd0276aa933",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([160,  35])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similar = high_similarity_indices[10]\n",
    "similar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "47162723-e185-4ec4-b4b9-56674c6c080e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Patel H, Dobson RJB, Newhouse SJ.'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "publications[similar[0]]['authors']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3484ab35-76c6-4a11-a830-d60f1770d4fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Patel H, Iniesta R, Stahl D, Dobson RJB, Newhouse SJ.'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "publications[similar[1]]['authors']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c8edbfe-1c60-49b4-adc1-f903a941bad5",
   "metadata": {},
   "source": [
    "Now add these assumed linkaged from authorship overlap as an additional linkage between publications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "db986217-0a6e-4c3a-a944-a1db9b4446b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "801"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "G2 = copy.deepcopy(Gorig)\n",
    "for i,j in high_similarity_indices:\n",
    "    source = f\"publication_{publications[i]['id']}\"\n",
    "    target = f\"publication_{publications[j]['id']}\"    \n",
    "    G2.add_edge(source, target, group=4)\n",
    "\n",
    "isolated_nodes = list(nx.isolates(G2))\n",
    "G2.remove_nodes_from(isolated_nodes)\n",
    "len(G.nodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "685a38fa-5784-4677-b48f-e8be1090bb73",
   "metadata": {},
   "source": [
    "Draw the new graph with these added relationships"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d3629b1f-6730-41e1-a8bf-72eee6adb63a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: When  cdn_resources is 'local' jupyter notebook has issues displaying graphics on chrome/safari. Use cdn_resources='in_line' or cdn_resources='remote' if you have issues viewing graphics in a notebook.\n",
      "./public/direct_linkages_with_authors.html\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"600px\"\n",
       "            src=\"./public/direct_linkages_with_authors.html\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x17f350560>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = Network(notebook=True)\n",
    "net.from_nx(G2)\n",
    "\n",
    "net.barnes_hut(\n",
    "    gravity=-2000,\n",
    "    central_gravity=0.3,\n",
    "    spring_length=95,\n",
    "    spring_strength=0.04,\n",
    "    damping=0.09,\n",
    "    overlap=0.1\n",
    ")\n",
    "net.show(\"./public/direct_linkages_with_authors.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "851bd463-2dd5-44b6-b046-fea024b4af86",
   "metadata": {},
   "source": [
    "#### Additional dataset linkages\n",
    "\n",
    "The dataset metadata contains an abstract about the datasets - we can use this and some simple NLP to try and extract some indirect linkages\n",
    "\n",
    "For example, an abstract may look like this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cebafc7a-7863-4815-9a16-654a9f0424af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The UK Cystic Fibrosis Registry Culture is made up of many data items relating to microbiology and cultures grown from samples taken during CF patient encounters at NHS hospitals in UK.'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets[0]['metadata']['summary']['abstract']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e03ff885-2d9e-45d1-8da0-66a6c99f53df",
   "metadata": {},
   "source": [
    "Extract the abstracts:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "78008092-bb6d-4a0b-93c2-8730c67a1208",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_abstracts = [\n",
    "    dataset['metadata']['summary'][\"abstract\"] \n",
    "    if dataset['metadata']['summary'][\"abstract\"] \n",
    "    else \"\" \n",
    "    for dataset in datasets\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da2a6fa2-e08c-4890-8c8c-c88393c0c698",
   "metadata": {},
   "source": [
    "Tokenise the words in the abstracts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "16f6105f-5923-452a-878b-7ccf9753f64d",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_abstracts = [\n",
    "    \" \".join(word_tokenize(abstract.lower())) \n",
    "    for abstract in dataset_abstracts\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f338ca9d-1d1d-47b1-9df8-54b8a04fa582",
   "metadata": {},
   "source": [
    "Vectorise and calcuated a simularity matrix between all the dataset abstracts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d92d84af-bc67-4f9d-b865-2d1e1b5004a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.58587943, 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.34776338, 0.32731339, 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       ...,\n",
       "       [0.00748172, 0.0068858 , 0.00593754, ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.0707266 , 0.06509324, 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.04141645, 0.03811764, 0.        , ..., 0.        , 0.09169122,\n",
       "        0.        ]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = TfidfVectorizer(stop_words='english', max_features=5000)  \n",
    "tfidf_matrix = vectorizer.fit_transform(tokenized_abstracts)\n",
    "\n",
    "similarity_matrix = np.tril(cosine_similarity(tfidf_matrix))\n",
    "np.fill_diagonal(similarity_matrix, 0)\n",
    "similarity_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "5e3dac02-1eef-4282-adce-a82280148267",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1059, 1059), 1059)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_matrix.shape,len(datasets)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "644f1056-ebed-4731-a2c2-fe89ed0b4281",
   "metadata": {},
   "source": [
    "Find some examples of high overlap in the abstracts:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4192a1dd-8aa8-4ca6-bb73-dc1715c55cfa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([165, 163])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "high_similarity_indices = np.argwhere((similarity_matrix >= 0.75) & (similarity_matrix < 0.98))\n",
    "similar = high_similarity_indices[12]\n",
    "similar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "83f99257-ed94-441f-a74b-0cb57e6033a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.7874033102505323)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_matrix[similar[0],similar[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "8c37585b-caa1-42f8-a2a9-46a420d5e840",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Chronic Obstructive Pulmonary Disease (COPD) patients admitted or using hospital services. Granular care pathways. Multi-morbidity, investigations, interventions, treatments. Serial physiology, blood biomarkers, physiotherapy, outcome. Deeply phenotyped.'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_abstracts[similar[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "96934d93-095e-45b3-bd76-aa1c16f5e6e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Patients admitted with a COPD related event. Granular care pathways. Multi-morbidity, investigations, interventions and treatments.   Serial physiology, blood biomarkers, physiotherapy, outcome. Deeply phenotyped.'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_abstracts[similar[1]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac81bf5c-7656-4d05-8036-0fb3c8b7e107",
   "metadata": {},
   "source": [
    "Construct a third network graph, to add in these additional indirect linkages based on the abstracts.."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "0a422cf1-8301-44ee-a3a4-b950c255671b",
   "metadata": {},
   "outputs": [],
   "source": [
    "G3 = copy.deepcopy(G2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "a0d31a79-02f2-43e1-b1c5-cd0e7c8470dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2041"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i,j in high_similarity_indices:\n",
    "    source = f\"dataset_{datasets[i]['id']}\"\n",
    "    target = f\"dataset_{datasets[j]['id']}\"\n",
    "    G3.add_edge(source, target, color='red')\n",
    "\n",
    "isolated_nodes = list(nx.isolates(G3))\n",
    "G3.remove_nodes_from(isolated_nodes)\n",
    "len(G3.nodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "730e3431-4244-40e2-b468-e6f416e9bc1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: When  cdn_resources is 'local' jupyter notebook has issues displaying graphics on chrome/safari. Use cdn_resources='in_line' or cdn_resources='remote' if you have issues viewing graphics in a notebook.\n",
      "./public/direct_linkages_and_indirect.html\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"600px\"\n",
       "            src=\"./public/direct_linkages_and_indirect.html\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x17f385640>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = Network(notebook=True)\n",
    "net.from_nx(G3)\n",
    "\n",
    "net.barnes_hut(\n",
    "    gravity=-2000,\n",
    "    central_gravity=0.3,\n",
    "    spring_length=95,\n",
    "    spring_strength=0.04,\n",
    "    damping=0.09,\n",
    "    overlap=0.1\n",
    ")\n",
    "net.show(\"./public/direct_linkages_and_indirect.html\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}