Automate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b49a10af-8deb-49c4-8958-76d7fb62da7b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "import re\n",
    "import time\n",
    "import os\n",
    "import shutil\n",
    "import pprint\n",
    "\n",
    "import titlecase\n",
    "import bibtexparser\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5b85fbc-f9eb-4405-8423-29357e871708",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# If you haven't installed chromedriver already\n",
    "import chromedriver_autoinstaller\n",
    "_p = chromedriver_autoinstaller.install()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f4c1ab9-5cd0-4524-850a-3a133d3c6c95",
   "metadata": {},
   "source": [
    "Steps:\n",
    "* Download the current list of bibliography entries from DBLP.\n",
    "* Figure out the new entries and deleted entries since last update.\n",
    "* Add new entries to the website and removed deleted entries automatically using Selenium.\n",
    "\n",
    "Notes:\n",
    "* Necessary: A correct map from the DBLP IDs to the DSG website Node IDs. This is maintained in the `dblp_to_web_mappings.txt` file.\n",
    "* When needed: Selenium can also be used to mass update existing entries. Adapt from the code for adding new entries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2893840d-cdff-4f26-9bcb-cbbf1e799efd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# List of DSG faculty for whom publications should be downloaded.\n",
    "authors_list = [\n",
    "     # filename, dblp url id, expected author names, normalized author name\n",
    "     ('CharlieClarke', '96/3666', ['Charles L. A. Clarke'], 'Charles Clarke'),\n",
    "     ('GordonVCormack', 'c/GVCormack', ['Gordon V. Cormack'], 'Gordon Cormack'),\n",
    "     ('KhuzaimaDaudjee', 'd/KhuzaimaDaudjee', ['Khuzaima Daudjee', 'Khuzaima S. Daudjee'], 'Khuzaima Daudjee'),\n",
    "     ('LukaszGolab', '42/3296', ['Lukasz Golab'], 'Lukasz Golab'),\n",
    "     ('MauraGrossman', '122/5875', ['Maura R. Grossman'], 'Maura Grossman'),\n",
    "     ('XiHe', '28/949-1', ['Xi He'], 'Xi He'),\n",
    "     ('IhabFIlyas', 'i/IhabFIlyas', ['Ihab F. Ilyas', 'Ihab Francis Ilyas'], 'Ihab Ilyas'),\n",
    "     ('JimmyLin', '00/7739', ['Jimmy J. Lin', 'Jimmy Lin'], 'Jimmy Lin'),\n",
    "     ('TamerOzsu', 'o/MTamerOzsu', ['M. Tamer Özsu'], 'Tamer Özsu'),\n",
    "     ('KennethSalem', 's/KennethSalem', ['Kenneth Salem', 'Ken Salem'], 'Ken Salem'),\n",
    "     ('SemihSalihoglu', '55/6560', ['Semih Salihoglu'], 'Semih Salihoglu'),\n",
    "     ('MarkDSmucker', '07/801', ['Mark D. Smucker'], 'Mark Smucker'),\n",
    "     ('DavidToman', 't/DavidToman', ['David Toman'], 'David Toman'),\n",
    "     ('FrankTompa', 't/FrankWmTompa', ['Frank Wm. Tompa'], 'Frank Tompa'),\n",
    "     ('GrantWeddell', '81/5447', ['Grant E. Weddell'], 'Grant Weddell'),\n",
    "     ('XiaoHu', '19/1374-5', ['Xiao Hu'], 'Xiao Hu'),\n",
    "     ('SujayaMaiyya', '224/6467', ['Sujaya Maiyya'], 'Sujaya Maiyya'),\n",
    "     ('ReneeMiller', 'm/RJMiller', ['Renée J. Miller'], 'Renée Miller')\n",
    "    \n",
    "]\n",
    "# Individual entries to avoid.\n",
    "bibid_blacklist = [\n",
    "    'DBLP:journals/sigmod/Ozsu04',\n",
    "    'DBLP:journals/sigmod/Ozsu04a',\n",
    "    'DBLP:journals/sigmod/Ozsu04b',\n",
    "    'DBLP:journals/sigmod/Ozsu03',\n",
    "    'DBLP:journals/sigmod/Ozsu03a',\n",
    "    'DBLP:journals/sigmod/Ozsu02',\n",
    "    'DBLP:journals/sigmod/Ozsu02a',\n",
    "    'DBLP:journals/sigmod/Ozsu02b',\n",
    "    'DBLP:journals/sigmod/Ozsu01',\n",
    "    'DBLP:journals/sigmod/Ozsu01a',\n",
    "    'DBLP:books/mc/19/Ozsu19',\n",
    "    'DBLP:journals/anthology/SnodgrassO01',\n",
    "    'DBLP:conf/ntcir/YilmazC16',\n",
    "    'DBLP:journals/vldb/Ozsu05',\n",
    "    'DBLP:journals/dpd/BertinoO94',\n",
    "    'DBLP:journals/pvldb/BonczS16',\n",
    "]\n",
    "# Bibliography types to avoid.\n",
    "btype_blacklist = [\n",
    "    'proceedings'\n",
    "]\n",
    "# Fields we are interested in. `common_fields` is just for reference but not used directly. Those fields are parsed individually below.\n",
    "common_fields = ['title', 'author', 'year', 'doi', 'url']\n",
    "extra_fields = {\n",
    "    'article': ['journal', 'volume', 'number', 'pages'],\n",
    "    'inproceedings': ['booktitle'],\n",
    "    'incollection': ['booktitle', 'publisher'],\n",
    "    'inproceedings': ['booktitle'],\n",
    "    'book': ['publisher'],\n",
    "    'phdthesis': ['school'],\n",
    "}\n",
    "\n",
    "# Helper function to parse dblp id for remote url.\n",
    "def get_remote_id(bibid):\n",
    "    parts = bibid.split(\"/\")\n",
    "    parts[0] = parts[0].replace(\"DBLP:\",\"\")\n",
    "    return  \"/\".join(parts[:-1])\n",
    "\n",
    "data_dir = \"data\"\n",
    "authors_dirs_previous = f\"{data_dir}/authors\"\n",
    "authors_dirs_new = f\"{data_dir}/authors_new\"\n",
    "mappings_file = f\"{data_dir}/dblp_to_web_mappings.txt\"\n",
    "\n",
    "def get_parsed_files(dir):\n",
    "    print(f\"Working version = {dir}\")\n",
    "    parsed = []\n",
    "    for author in authors_list:\n",
    "        with open(f\"{dir}/{author[0]}.bib\") as f:\n",
    "            content = f.read()\n",
    "            parser = bibtexparser.bparser.BibTexParser()\n",
    "            parser.customization = bibtexparser.customization.convert_to_unicode # Convert latex code to unicode. Not perfect.\n",
    "            bibliography = bibtexparser.loads(content, parser=parser)\n",
    "            parsed.append((author[0], bibliography))\n",
    "    return parsed\n",
    "\n",
    "def combine(dir):\n",
    "    count = 0\n",
    "    all_entries = {}\n",
    "    for author,collection in get_parsed_files(dir):\n",
    "        print(\"len(%s) = %s\" % (author, len(collection.entries)))\n",
    "        count += len(collection.entries)\n",
    "        for entry in collection.entries:\n",
    "            bibid = entry['ID']\n",
    "            if bibid not in all_entries and bibid not in bibid_blacklist and entry['ENTRYTYPE'] not in btype_blacklist:\n",
    "                all_entries[bibid] = entry\n",
    "    print(\"total entries = %s, deduplicated and filtered entries = %s\" % (count, len(all_entries)))\n",
    "\n",
    "    # Separate entries by bibliography type for easier processing.\n",
    "    by_type = {}\n",
    "    for entry in all_entries.values():\n",
    "        by_type.setdefault(entry['ENTRYTYPE'],list()).append(entry)\n",
    "    count = 0\n",
    "    for btype, entry in by_type.items():\n",
    "        print(btype, len(entry))\n",
    "        count += len(entry)\n",
    "    print(\"total =\", count)\n",
    "\n",
    "    return all_entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02fe688c-6051-41b1-84a0-e4bdd2d8c930",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Download bib files for individual authors and save to disk.\n",
    "assert os.path.exists(authors_dirs_previous), \"'{authors_dirs_previous}' missing\"\n",
    "if not os.path.exists(authors_dirs_new):\n",
    "    os.makedirs(authors_dirs_new)\n",
    "print(f\"Downloading to '{authors_dirs_new}':\")\n",
    "for author in authors_list:\n",
    "    print(\">\",author)\n",
    "    req = urllib.request.Request(\"https://dblp.uni-trier.de/pid/%s.bib?param=1\" % author[1])\n",
    "    response = urllib.request.urlopen(req)\n",
    "    content = response.read()\n",
    "    with open(\"%s/%s.bib\" % (authors_dirs_new,author[0]),\"wb\") as f:\n",
    "        f.write(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a61fdb26-cabf-458c-b3ba-09c7707d473e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Combine authors and deduplicate entries for latest and previous set of downloads.\n",
    "all_entries = combine(authors_dirs_new)\n",
    "print()\n",
    "previous_all_entries = combine(authors_dirs_previous)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "154225ed-ba2b-4aa3-90d9-c8fc91cd28a4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Find new entries that did not exist\n",
    "new_ids = {}\n",
    "deleted_ids = {}\n",
    "for bid in all_entries:\n",
    "    if bid not in previous_all_entries:\n",
    "        new_ids[bid] = all_entries[bid]\n",
    "for bid in previous_all_entries:\n",
    "    if bid not in all_entries:\n",
    "        deleted_ids[bid] = previous_all_entries[bid]\n",
    "print(\"New =\", len(new_ids),\"Deleted = \", len(deleted_ids))\n",
    "assert(len(new_ids) - len(deleted_ids) == len(all_entries) - len(previous_all_entries))\n",
    "\n",
    "by_type = {}\n",
    "for entry in new_ids.values():\n",
    "    by_type.setdefault(entry['ENTRYTYPE'],list()).append(entry)\n",
    "for btype, entry in by_type.items():\n",
    "    print(btype, len(entry))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35cd6385-c401-475f-a2bb-db2d9816a726",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Full names for conferences and journals. Edits should be done in the `edited_strings` variable below.\n",
    "raw_strings = {\n",
    "    \"journals/tois\": \"ACM Transactions on Information Systems (TOIS)\",\n",
    "    \"conf/chiir\": \"Conference on Human Information Interaction and Retrieval (CHIIR)\",\n",
    "    \"conf/sigir\": \"Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR)\",\n",
    "    \"conf/cikm\": \"International Conference on Information and Knowledge Management (CIKM)\",\n",
    "    \"conf/ictir\": \"International Conference on the Theory of Information Retrieval (ICTIR)\",\n",
    "    \"conf/trec\": \"Text Retrieval Conference (TREC)\",\n",
    "    \"journals/corr\": \"Computing Research Repository (CoRR)\",\n",
    "    \"conf/ntcir\": \"NTCIR Conference on Evaluation of Information Access Technologies (NTCIR)\",\n",
    "    \"journals/csur\": \"ACM Computing Surveys\",\n",
    "    \"journals/sigir\": \"SIGIR Forum\",\n",
    "    \"journals/tkde\": \"IEEE Transactions on Knowledge and Data Engineering\",\n",
    "    \"conf/wsdm\": \"Web Search and Data Mining (WSDM)\",\n",
    "    \"reference/db\": \"Encyclopedia of Database Systems\",\n",
    "    \"conf/chi\": \"ACM Conference on Human Factors in Computing Systems (CHI)\",\n",
    "    \"conf/icccrea\": \"International Conference on Computational Creativity (ICCC)\",\n",
    "    \"conf/www\": \"The Web Conference (WWW)\",\n",
    "    \"journals/internet\": \"IEEE Internet Computing\",\n",
    "    \"journals/ir\": \"Information Retrieval Journal\",\n",
    "    \"conf/adcs\": \"Australasian Document Computing Symposium (ADCS)\",\n",
    "    \"conf/acl\": \"Annual Meeting of the Association for Computational Linguistics (ACL)\",\n",
    "    \"conf/ecir\": \"European Conference on Information Retrieval (ECIR)\",\n",
    "    \"conf/russir\": \"Russian Summer School on Information Retrieval (RuSSIR)\",\n",
    "    \"journals/tist\": \"ACM Transactions on Intelligent Systems and Technology (TIST)\",\n",
    "    \"conf/iiix\": \"International Conference on Information Interaction in Context (IIiX)\",\n",
    "    \"journals/kais\": \"Knowledge and Information Systems (KAIS)\",\n",
    "    \"journals/spe\": \"Software - Practice and Experience (SPE)\",\n",
    "    \"conf/icdm\": \"IEEE International Conference on Data Mining (ICDM)\",\n",
    "    \"conf/eurohcir\": \"European Workshop on Human-Computer Interaction and Information Retrieval (EuroHCIR)\",\n",
    "    \"conf/hcir\": \"Symposium on Human-Computer Interaction and Information Retrieval (HCIR)\",\n",
    "    \"conf/ceas\": \"International Conference on Email and Anti-Spam (CEAS)\",\n",
    "    \"books/daglib\": \"Books from the Dagstuhl Library\",\n",
    "    \"conf/graphicsinterface\": \"Graphics Interface\",\n",
    "    \"conf/riao\": \"Open research Areas in Information Retrieval (OAIR)\",\n",
    "    \"conf/inex\": \"INitiative for the Evaluation of XML Retrieval (INEX)\",\n",
    "    \"conf/webi\": \"IEEE/WIC/ACM International Conference on Web Intelligence (WI)\",\n",
    "    \"journals/usenix-login\": \"login - The Usenix Magazine\",\n",
    "    \"conf/clef\": \"Conference and Labs of the Evaluation Forum (CLEF)\",\n",
    "    \"conf/fast\": \"USENIX Conference on File and Storage Technologies (FAST)\",\n",
    "    \"conf/uist\": \"ACM Symposium on User Interface Software and Technology (UIST)\",\n",
    "    \"journals/entcs\": \"Electronic Notes in Theoretical Computer Science (ENTCS)\",\n",
    "    \"conf/coling\": \"International Conference on Computational Linguistics (COLING)\",\n",
    "    \"conf/iwpc\": \"IEEE International Conference on Program Comprehension (ICPC)\",\n",
    "    \"conf/naacl\": \"North American Chapter of the Association for Computational Linguistics (NAACL)\",\n",
    "    \"conf/icsm\": \"IEEE International Conference on Software Maintenance and Evolution (ICSME)\",\n",
    "    \"journals/ipm\": \"Information Processing and Management\",\n",
    "    \"conf/apsec\": \"Asia-Pacific Software Engineering Conference (APSEC)\",\n",
    "    \"conf/wflp\": \"International Workshop on Functional and (Constraint) Logic Programming (WFLP)\",\n",
    "    \"conf/cascon\": \"Conference of the Centre for Advanced Studies on Collaborative Research (CASCON)\",\n",
    "    \"journals/toplas\": \"ACM Transactions on Programming Languages and Systems (TOPLAS)\",\n",
    "    \"journals/cj\": \"The Computer Journal\",\n",
    "    \"journals/jdiq\": \"Journal of Data and Information Quality\",\n",
    "    \"conf/doceng\": \"ACM Symposium on Document Engineering (DocEng)\",\n",
    "    \"conf/kdd\": \"ACM SIGKDD Conference on Knowledge Discovery and Data Mining (KDD)\",\n",
    "    \"journals/cacm\": \"Communications of the ACM\",\n",
    "    \"conf/ppdp\": \"ACM-SIGPLAN International Conference on Principles and Practice of Declarative Programming (PPDP)\",\n",
    "    \"journals/ftir\": \"Foundations and Trends in Information Retrieval\",\n",
    "    \"journals/iie\": \"Informatics in Education\",\n",
    "    \"journals/jmlr\": \"Journal of Machine Learning Research (JMLR)\",\n",
    "    \"conf/issep\": \"International Conference on Informatics in Secondary Schools (ISSEP)\",\n",
    "    \"journals/ipl\": \"Information Processing Letters\",\n",
    "    \"conf/cscw\": \"Conference on Computer Supported Cooperative Work (CSCW)\",\n",
    "    \"journals/acta\": \"Acta Informatica\",\n",
    "    \"conf/podc\": \"ACM SIGACT-SIGOPS Symposium on Principles of Distributed Computing (PODC)\",\n",
    "    \"journals/cl\": \"Computer Languages, Systems &amp; Structures\",\n",
    "    \"conf/coodbse\": \"Colloquium on Object Orientation in Databases and Software Engineering (COODBSE)\",\n",
    "    \"conf/dcc\": \"Data Compression Conference (DCC)\",\n",
    "    \"conf/icpp\": \"International Conference on Parallel Processing (ICPP)\",\n",
    "    \"conf/pldi\": \"ACM-SIGPLAN Symposium on Programming Language Design and Implementation (PLDI)\",\n",
    "    \"journals/sigplan\": \"ACM SIGPLAN Notices\",\n",
    "    \"conf/asplos\": \"International Conference on Architectural Support for Programming Languages and Operating Systems (ASPLOS)\",\n",
    "    \"conf/sigmod\": \"ACM SIGMOD Conference (SIGMOD)\",\n",
    "    \"conf/ssdbm\": \"International Conference on Statistical and Scientific Database Management (SSDBM)\",\n",
    "    \"journals/dpd\": \"Distributed and Parallel Databases\",\n",
    "    \"journals/pvldb\": \"Proceedings of the VLDB Endowment\",\n",
    "    \"conf/debs\": \"Distributed Event-Based Systems (DEBS)\",\n",
    "    \"conf/icde\": \"IEEE International Conference on Data Engineering (ICDE)\",\n",
    "    \"journals/vldb\": \"The VLDB Journal\",\n",
    "    \"conf/conext\": \"Conference on Emerging Network Experiment and Technology (CoNEXT)\",\n",
    "    \"conf/sigcse\": \"Technical Symposium on Computer Science Education (SIGCSE)\",\n",
    "    \"conf/edbt\": \"International Conference on Extending Database Technology (EDBT)\",\n",
    "    \"conf/icdcs\": \"IEEE International Conference on Distributed Computing Systems (ICDCS)\",\n",
    "    \"conf/middleware\": \"International Middleware Conference (Middleware)\",\n",
    "    \"conf/edcc\": \"European Dependable Computing Conference (EDCC)\",\n",
    "    \"conf/nca\": \"IEEE International Symposium on Network Computing and Applications (NCA)\",\n",
    "    \"journals/jpdc\": \"Journal of Parallel and Distributed Computing\",\n",
    "    \"journals/ppna\": \"Peer-to-Peer Networking and Applications\",\n",
    "    \"conf/cloud\": \"ACM Symposium on Cloud Computing (SoCC)\",\n",
    "    \"conf/vldb\": \"Very Large Data Bases Conference (VLDB)\",\n",
    "    \"conf/semweb\": \"International Semantic Web Conference (ISWC)\",\n",
    "    \"conf/spaa\": \"ACM Symposium on Parallelism in Algorithms and Architectures (SPAA)\",\n",
    "    \"conf/icdcsw\": \"International Conference on Distributed Computing\\nSystems (ICDCS) - Workshops\",\n",
    "    \"conf/ieeehpcs\": \"International Conference on High Performance Computing &amp; Simulation (HPCS)\",\n",
    "    \"conf/ht\": \"ACM Conference on Hypertext and Social Media (HT)\",\n",
    "    \"conf/ipccc\": \"IEEE International Performance, Computing, and Communications Conference  (IPCCC)\",\n",
    "    \"conf/otm\": \"OnTheMove Federated Conferences &amp; Workshops (OTM)\",\n",
    "    \"conf/p2p\": \"IEEE International Conference on Peer-to-Peer Computing\",\n",
    "    \"conf/icpads\": \"International Conference on Parallel and Distributed Systems (ICPADS)\",\n",
    "    \"journals/re\": \"Requirements Engineering\",\n",
    "    \"conf/dolap\": \"International Workshop on Data Warehousing and OLAP (DOLAP)\",\n",
    "    \"journals/computing\": \"Computing\",\n",
    "    \"journals/ijnm\": \"International Journal of Network Management\",\n",
    "    \"conf/ai\": \"Canadian Conference on Artificial Intelligence (AI)\",\n",
    "    \"conf/edm\": \"Educational Data Mining (EDM)\",\n",
    "    \"conf/europar\": \"European Conference on Parallel Processing (Euro-Par)\",\n",
    "    \"conf/icbc2\": \"IEEE International Conference on Blockchain and Cryptocurrency (ICBC)\",\n",
    "    \"conf/amw\": \"Alberto Mendelzon International Workshop on Foundations of Data Management (AMW)\",\n",
    "    \"conf/eenergy\": \"Energy-Efficient Computing and Networking (e-Energy)\",\n",
    "    \"conf/wassa\": \"Workshop on Computational Approaches to Subjectivity, Sentiment and Social Media Analysis (WASSA)\",\n",
    "    \"reference/bdt\": \"Encyclopedia of Big Data Technologies\",\n",
    "    \"series/synthesis\": \"Synthesis\",\n",
    "    \"journals/debu\": \"IEEE Data Engineering Bulletin\",\n",
    "    \"journals/tits\": \"IEEE Transactions on Intelligent Transportation Systems\",\n",
    "    \"journals/is\": \"Information Systems\",\n",
    "    \"journals/tods\": \"ACM Transactions on Database Systems (TODS)\",\n",
    "    \"conf/bigdataconf\": \"IEEE International Conference on Big Data (IEEE BigData)\",\n",
    "    \"journals/cn\": \"Computer Networks\",\n",
    "    \"conf/iwcmc\": \"International Conference on Wireless Communications and Mobile Computing (IWCMC)\",\n",
    "    \"books/sp/13\": \"************ failed\",\n",
    "    \"conf/pam\": \"Passive and Active Network Measurement Conference (PAM)\",\n",
    "    \"journals/mst\": \"Theory of Computing Systems\",\n",
    "    \"conf/cidr\": \"Conference on Innovative Data Systems Research (CIDR)\",\n",
    "    \"phd/basesearch\": \"************ failed\",\n",
    "    \"conf/edbtw\": \"International Conference on Extending Database Technology (EDBT) - Workshops\",\n",
    "    \"journals/sigmod\": \"SIGMOD Record\",\n",
    "    \"conf/imc\": \"ACM/SIGCOMM Internet Measurement Conference (IMC)\",\n",
    "    \"conf/mie\": \"Medical Informatics Europe (MIE)\",\n",
    "    \"conf/isit\": \"International Symposium on Information Theory (ISIT)\",\n",
    "    \"journals/popets\": \"Proceedings on Privacy Enhancing Technologies (PoPETs)\",\n",
    "    \"reference/sp\": \"Springer Handbooks\",\n",
    "    \"conf/ccs\": \"Conference on Computer and Communications Security (CCS)\",\n",
    "    \"conf/mlsys\": \"Conference on Machine Learning and Systems (MLSys)\",\n",
    "    \"books/acm\": \"************ failed\",\n",
    "    \"conf/aistats\": \"International Conference on Artificial Intelligence and Statistics (AISTATS)\",\n",
    "    \"conf/btw\": \"Datenbanksysteme f&uuml;r Business, Technologie und Web (BTW)\",\n",
    "    \"conf/icdt\": \"International Conference on Database Theory (ICDT)\",\n",
    "    \"conf/uai\": \"Conference on Uncertainty in Artificial Intelligence (UAI)\",\n",
    "    \"books/mc/19\": \"************ failed\",\n",
    "    \"journals/ml\": \"Machine Learning\",\n",
    "    \"journals/ftdb\": \"Foundations and Trends in Databases\",\n",
    "    \"journals/www\": \"World Wide Web (WWW)\",\n",
    "    \"conf/seco\": \"SeCO Workshops (SeCO)\",\n",
    "    \"journals/sadm\": \"Statistical Analysis and Data Mining\",\n",
    "    \"conf/dagstuhl\": \"Dagstuhl Publications\",\n",
    "    \"journals/mms\": \"Multimedia Systems\",\n",
    "    \"conf/icac\": \"IEEE International Conference on Autonomic Computing (ICAC)\",\n",
    "    \"conf/mmdb\": \"ACM International Workshop on Multimedia Databases (MMDB)\",\n",
    "    \"conf/mis\": \"Workshop on Multimedia Information Systems (MIS)\",\n",
    "    \"journals/jiis\": \"Journal of Intelligent Information Systems (JIIS)\",\n",
    "    \"journals/dhq\": \"Digital Humanities Quarterly\",\n",
    "    \"journals/envsoft\": \"Environmental Modelling and Software\",\n",
    "    \"conf/aaai\": \"AAAI Conference on Artificial Intelligence (AAAI)\",\n",
    "    \"conf/eacl\": \"Conference of the European Chapter of the Association for Computational Linguistics (EACL)\",\n",
    "    \"journals/oclc\": \"Digital Library Perspectives\",\n",
    "    \"journals/scientometrics\": \"Scientometrics\",\n",
    "    \"conf/birws\": \"International Workshop on Bibliometric-enhanced Information Retrieval (BIR)\",\n",
    "    \"conf/emnlp\": \"Conference on Empirical Methods in Natural Language Processing (EMNLP)\",\n",
    "    \"conf/icml\": \"International Conference on Machine Learning (ICML)\",\n",
    "    \"conf/jcdl\": \"ACM/IEEE Joint Conference on Digital Libraries (JCDL)\",\n",
    "    \"conf/rep4nlp\": \"Workshop on Representation Learning for NLP (RepL4NLP)\",\n",
    "    \"conf/acl-deeplo\": \"Workshop on Deep Learning Approaches for Low-Resource Natural Language Processing (DeepLo)\",\n",
    "    \"conf/amia\": \"American Medical Informatics Association Annual Symposium (AMIA)\",\n",
    "    \"conf/iui\": \"International Conference on Intelligent User Interfaces (IUI)\",\n",
    "    \"conf/IEEEcloud\": \"IEEE International Conference on Cloud Computing (CLOUD)\",\n",
    "    \"conf/desires\": \"Biennial Conference on Design of Experimental Search &amp; Information Retrieval Systems (DESIRES)\",\n",
    "    \"conf/hotcloud\": \"USENIX Workshop on Hot Topics in Cloud Computing (HotCloud)\",\n",
    "    \"conf/icassp\": \"IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP)\",\n",
    "    \"journals/jocch\": \"ACM Journal on Computing and Cultural Heritage\",\n",
    "    \"conf/grades\": \"International Workshop on Graph Data Management Experiences and Systems (GRADES)\",\n",
    "    \"conf/isbi\": \"IEEE International Symposium on Biomedical Imaging (ISBI)\",\n",
    "    \"books/sp/17\": \"************ failed\",\n",
    "    \"conf/ccnc\": \"Consumer Communications and Networking Conference (CCNC)\",\n",
    "    \"conf/dihu\": \"Digital Humanities Conference (DH)\",\n",
    "    \"conf/mm\": \"ACM International Conference on Multimedia (MM)\",\n",
    "    \"conf/sac\": \"ACM Symposium on Applied Computing (SAC)\",\n",
    "    \"conf/semeval\": \"International Workshop on Semantic Evaluation (SemEval )\",\n",
    "    \"journals/tacl\": \"Transactions of the Association for Computational Linguistics\",\n",
    "    \"conf/medinfo\": \"World Congress on Medical and Health (Medical) Informatics (MedInfo)\",\n",
    "    \"conf/dexaw\": \"DEXA Workshops\",\n",
    "    \"conf/gis\": \"ACM SIGSPATIAL International Workshop on Advances in Geographic Information Systems (GIS)\",\n",
    "    \"conf/ieeevast\": \"IEEE Conference on Visual Analytics Science and Technology (VAST)\",\n",
    "    \"conf/lats\": \"ACM Conference on Learning @ Scale (L@S)\",\n",
    "    \"journals/bigdata\": \"Big Data\",\n",
    "    \"conf/icwsm\": \"International Conference on Web and Social Media (ICWSM)\",\n",
    "    \"conf/wmt\": \"Conference on Machine Translation (WMT)\",\n",
    "    \"journals/sigkdd\": \"SIGKDD Explorations\",\n",
    "    \"conf/iconference\": \"iConference\",\n",
    "    \"conf/cloudcom\": \"International Conference on Cloud Computing (CloudCom)\",\n",
    "    \"conf/mlg\": \"Mining and Learning with Graphs (MLG)\",\n",
    "    \"journals/bmcbi\": \"BMC Bioinformatics\",\n",
    "    \"journals/firstmonday\": \"First Monday\",\n",
    "    \"journals/jasis\": \"Journal of the Association for Information Science and Technology (JASIST)\",\n",
    "    \"journals/lre\": \"Language Resources and Evaluation (LRE)\",\n",
    "    \"journals/mta\": \"Multimedia Tools and Applications\",\n",
    "    \"conf/tac\": \"Text Analysis Conference (TAC)\",\n",
    "    \"journals/coling\": \"Computational Linguistics\",\n",
    "    \"conf/latech\": \"Joint SIGHUM Workshop on Computational Linguistics for Cultural Heritage, Social Sciences, Humanities and Literature (LaTeCH)\",\n",
    "    \"conf/asist\": \"ASIS&amp;T Annual Meeting (ASIST)\",\n",
    "    \"conf/bionlp\": \"Workshop on Biomedical Natural Language Processing (BioNLP)\",\n",
    "    \"conf/eamt\": \"European Association for Machine Translation Conferences/Workshops (EAMT)\",\n",
    "    \"phd/ndltd\": \"************ failed\",\n",
    "    \"conf/ndqa\": \"New Directions in Question Answering\",\n",
    "    \"conf/acl-iwp\": \"International Workshop on Paraphrasing (IWP)\",\n",
    "    \"conf/dgo\": \"(Inter)National Conference on Digital Government Research (DG.O)\",\n",
    "    \"conf/interact\": \"IFIP TC13 International Conference on Human-Computer Interaction (INTERACT)\",\n",
    "    \"conf/acl-xml\": \"Workshop on NLP and XML\",\n",
    "    \"conf/coopis\": \"International Conference on Cooperative Information Systems (CoopIS)\",\n",
    "    \"conf/lrec\": \"International Conference on Language Resources and Evaluation (LREC)\",\n",
    "    \"conf/nldb\": \"International Conference on Applications of Natural Language to Data Bases (NLDB)\",\n",
    "    \"journals/ijst\": \"International Journal of Speech Technology\",\n",
    "    \"conf/icmcs\": \"IEEE International Conference on Multimedia and Expo (ICME)\",\n",
    "    \"journals/tlsdkcs\": \"Transactions on Large-Scale Data- and Knowledge-Centered Systems\",\n",
    "    \"books/sp\": \"************ failed\",\n",
    "    \"journals/tpds\": \"IEEE Transactions on Parallel and Distributed Systems (TPDS)\",\n",
    "    \"conf/dsc\": \"International Conference on Data Science in Cyberspace (DSC)\",\n",
    "    \"conf/ppopp\": \"ACM SIGPLAN Symposium on Principles &amp; Practice of Parallel Programming (PPoPP)\",\n",
    "    \"conf/dasfaa\": \"International Conference on Database Systems for Advanced Applications (DASFAA)\",\n",
    "    \"journals/dase\": \"Data Science and Engineering\",\n",
    "    \"journals/fcsc\": \"Frontiers of Computer Science\",\n",
    "    \"conf/ideas\": \"International Database Engineering and Applications Symposium (IDEAS)\",\n",
    "    \"books/crc/chb\": \"************ failed\",\n",
    "    \"conf/wbdb\": \"Workshop on Big Data Benchmarking (WBDB)\",\n",
    "    \"conf/birthday\": \"Festschrifts: Birthday, In Memory of ...\",\n",
    "    \"conf/xsym\": \"International XML Database Symposium (XSym)\",\n",
    "    \"conf/waim\": \"Interational Conference on Web-Age Information Management (WAIM)\",\n",
    "    \"conf/recsys\": \"ACM Conference on Recommender Systems (RecSys)\",\n",
    "    \"conf/adbis\": \"Symposium on Advances in Databases and Information Systems (ADBIS)\",\n",
    "    \"journals/ijdsn\": \"International Journal of Distributed Sensor Networks\",\n",
    "    \"conf/mmm\": \"Conference on Multimedia Modeling (MMM)\",\n",
    "    \"conf/wise\": \"International Conference on Web Information Systems Engineering (WISE)\",\n",
    "    \"conf/cvdb\": \"Computer Vision meets Databases (CVDB)\",\n",
    "    \"conf/mir\": \"International Conference on Multimedia Retrieval (ICMR)\",\n",
    "    \"journals/ijig\": \"International Journal of Image and Graphics\",\n",
    "    \"conf/bda\": \"Journ&eacute;es Bases de Donn&eacute;es Avanc&eacute;es  (BDA)\",\n",
    "    \"conf/icann\": \"International Conference on Artificial Neural Networks and Machine Learning (ICANN)\",\n",
    "    \"conf/iscis\": \"International Symposium on Computer and Information Sciences (ISCIS)\",\n",
    "    \"conf/spieSR\": \"Storage and Retrieval Methods and Applications for Multimedia\",\n",
    "    \"conf/diweb\": \"International Workshop on Data Integration over the Web (DIWeb)\",\n",
    "    \"conf/eextt\": \"Efficiency and Effectiveness of XML Tools and Techniques (EEXTT)\",\n",
    "    \"conf/icip\": \"IEEE International Conference on Image Processing (ICIP)\",\n",
    "    \"reference/ap\": \"Academic Press Reference\",\n",
    "    \"conf/caise\": \"International Conference on Advanced Information Systems Engineering (CAiSE)\",\n",
    "    \"journals/anthology\": \"************ failed\",\n",
    "    \"conf/ride\": \"International Workshop on Research Issues in Data Engineering (RIDE)\",\n",
    "    \"conf/wiiw\": \"Workshop on Information Integration on the Web (WIIW)\",\n",
    "    \"books/mk/dittrichG01\": \"************ failed\",\n",
    "    \"journals/dr\": \"ACM SIGMOD Digital Review\",\n",
    "    \"books/ph\": \"************ failed\",\n",
    "    \"conf/ds\": \"IFIP Working Conference on Database Semantics (DS)\",\n",
    "    \"journals/dke\": \"Data &amp; Knowledge Engineering\",\n",
    "    \"conf/adl\": \"Advances in Digital Libraries (ADL)\",\n",
    "    \"conf/tools\": \"International Conference on Software Technology: Methods and Tools (TOOLS)\",\n",
    "    \"conf/vdb\": \"Visual Database Systems (VDB)\",\n",
    "    \"conf/iadt\": \"Issues and Applications of Database Technology (IADT)\",\n",
    "    \"journals/ibmsj\": \"IBM Systems Journal\",\n",
    "    \"conf/er\": \"International Conference on Conceptual Modeling (ER)\",\n",
    "    \"books/crc/tucker97\": \"************ failed\",\n",
    "    \"conf/iwmmdbms\": \"International Workshop on Multi-Media Database Management Systems (IW-MMDBMS)\",\n",
    "    \"conf/icccn\": \"International Conference on Computer Communications and Networks (ICCCN)\",\n",
    "    \"books/acm/kim95\": \"************ failed\",\n",
    "    \"conf/nato\": \"NATO Advanced Study Institute (NATO ASI)\",\n",
    "    \"conf/oodbs\": \"International Symposium on Objects and Databases (SODB)\",\n",
    "    \"conf/icci\": \"International Conference on Computing and Information (ICCI)\",\n",
    "    \"books/mk/elmagarmid92\": \"************ failed\",\n",
    "    \"journals/computer\": \"IEEE Computer\",\n",
    "    \"conf/dood\": \"International Conference on Deductive and Object-Oriented Databases (DOOD)\",\n",
    "    \"tr/gte\": \"************ failed\",\n",
    "    \"conf/compsac\": \"Annual International Computer Software and Applications Conference (COMPSAC)\",\n",
    "    \"conf/oopsla\": \"ACM SIGPLAN International Conference on Systems, Programming, Languages and Applications: Software for Humanity (SPLASH)\",\n",
    "    \"journals/tse\": \"IEEE Transactions on Software Engineering (TSE)\",\n",
    "    \"conf/damon\": \"International Workshop on Data Management on New Hardware (DaMoN)\",\n",
    "    \"conf/hotstorage\": \"USENIX Workshop on Hot Topics in Storage and File Systems (HotStorage)\",\n",
    "    \"conf/srds\": \"IEEE International Symposium on Reliable Distributed Systems (SRDS)\",\n",
    "    \"conf/dnis\": \"Databases in Networked Information Systems (DNIS)\",\n",
    "    \"conf/eurosys\": \"European Conference on Computer Systems (EuroSys)\",\n",
    "    \"journals/jcss\": \"Journal of Computer and System Sciences (JCSS)\",\n",
    "    \"journals/tc\": \"IEEE Transactions on Computers\",\n",
    "    \"journals/tocs\": \"ACM Transactions on Computer Systems (TOCS)\",\n",
    "    \"conf/usenix\": \"USENIX Annual Technical Conference (USENIX ATC)\",\n",
    "    \"conf/compcon\": \"IEEE Computer Society International Conference (COMPCON)\",\n",
    "    \"conf/pods\": \"ACM SIGACT-SIGMOD-SIGART Symposium on Principles of Database Systems (PODS)\",\n",
    "    \"conf/db-workshops\": \"Data Base Workshops\",\n",
    "    \"conf/hpts\": \"High Performance Transaction Systems Workshop (HPTS)\",\n",
    "    \"conf/complexnetworks\": \"International Workshop on Complex Networks &amp; Their Applications (COMPLEX NETWORKS)\",\n",
    "    \"conf/cgo\": \"IEEE/ACM International Symposium on Code Generation and Optimization (CGO)\",\n",
    "    \"books/others/11\": \"************ failed\",\n",
    "    \"conf/eScience\": \"IEEE International Conference on e-Science (E-Science)\",\n",
    "    \"conf/ecal\": \"European Conference on Artificial Life (ECAL)\",\n",
    "    \"journals/ki\": \"K&uuml;nstliche Intelligenz (KI)\",\n",
    "    \"conf/dlog\": \"International Workshop on Description Logics (DL)\",\n",
    "    \"conf/ausai\": \"Australian Joint Conference on Artificial Intelligence (AUS-AI)\",\n",
    "    \"conf/pricai\": \"Pacific Rim International Conference on Artificial Intelligence (PRICAI)\",\n",
    "    \"conf/ekaw\": \"International Conference Knowledge Engineering and Knowledge Management (EKAW)\",\n",
    "    \"conf/kr\": \"International Conference on Principles of Knowledge Representation and Reasoning (KR)\",\n",
    "    \"journals/jlp\": \"Journal of Logic Programming\",\n",
    "    \"conf/ijcai\": \"International Joint Conference on Artificial Intelligence (IJCAI)\",\n",
    "    \"conf/lpar\": \"International Conference on Logic Programming and Automated Reasoning (LPAR)\",\n",
    "    \"journals/ws\": \"Journal of Web Semantics\",\n",
    "    \"conf/tableaux\": \"International Conference on Theorem Proving with Analytic Tableaux and Related Methods (TABLEAUX)\",\n",
    "    \"journals/jar\": \"Journal of Automated Reasoning\",\n",
    "    \"conf/sebd\": \"Sistemi Evoluti per Basi di Dati (SEBD)\",\n",
    "    \"conf/dsd\": \"Euromicro Symposium on Digital Systems Design (DSD)\",\n",
    "    \"conf/owled\": \"W3C Web Ontology Language (OWL) Experiences and Directions Workshop (OWLED)\",\n",
    "    \"journals/iandc\": \"Information and Computation\",\n",
    "    \"conf/cade\": \"Conference on Automated Deduction (CADE)\",\n",
    "    \"conf/iclp\": \"International Conference on Logic Programming (ICLP)\",\n",
    "    \"conf/time\": \"International Symposium/Workshop on Temporal Representation and Reasoning (TIME)\",\n",
    "    \"conf/stdbm\": \"International Workshop on Spatio-Temporal Database Management (STDBM)\",\n",
    "    \"journals/tcs\": \"Theoretical Computer Science\",\n",
    "    \"reference/fai\": \"Foundations of Artificial Intelligence\",\n",
    "    \"conf/ssd\": \"International Symposium on Spatial and Temporal Databases (SSTD)\",\n",
    "    \"conf/dbtel\": \"Databases in Telecommunications (DBTel)\",\n",
    "    \"conf/cl\": \"International Conference on Computational Logic (CL)\",\n",
    "    \"conf/cdb\": \"International Symposium on the Applications of Constraint Databases (CDB)\",\n",
    "    \"journals/constraints\": \"Constraints - An International Journal\",\n",
    "    \"conf/slp\": \"Joint International Conference and Symposium on Logic Programming (JICSLP)\",\n",
    "    \"journals/jis\": \"Journal of Information Science\",\n",
    "    \"conf/sacmat\": \"ACM Symposium on Access Control Models and Technologies (SACMAT)\",\n",
    "    \"conf/mkm\": \"International Conference on Intelligent Computer Mathematics (CICM)\",\n",
    "    \"conf/enlg\": \"European Workshop on Natural Language Generation (ENLG)\",\n",
    "    \"journals/lalc\": \"Digital Scholarship in the Humanities (DSH)\",\n",
    "    \"conf/policy\": \"IEEE International Symposium on Policies for Distributed Systems and Networks (POLICY)\",\n",
    "    \"conf/webdb\": \"International Workshop on the Web and Databases (WebDB)\",\n",
    "    \"conf/re\": \"IEEE International Requirements Engineering Conference (RE)\",\n",
    "    \"conf/sdmw\": \"Secure Data Management (VLDB Workshop) (SDM)\",\n",
    "    \"journals/csse\": \"Computer Systems: Science &amp; Engineering\",\n",
    "    \"journals/tapos\": \"TAPOS - Theory and Practice of Object Systems\",\n",
    "    \"conf/w3c\": \"W3C Workshops (W3C)\",\n",
    "    \"journals/csi\": \"Computer Standards &amp; Interfaces\",\n",
    "    \"conf/adb\": \"Applications of Databases (ADB)\",\n",
    "    \"journals/isci\": \"Information Sciences\",\n",
    "    \"conf/iccal\": \"International Conference on Computers and Learning (ICCAL)\",\n",
    "    \"journals/ijmms\": \"International Journal of Human-Computer Studies\",\n",
    "    \"journals/algorithmica\": \"Algorithmica\",\n",
    "    \"conf/siggraph\": \"International Conference on Computer Graphics and Interactive Techniques (SIGGRAPH)\",\n",
    "    \"journals/ac\": \"Advances in Computers\",\n",
    "    \"journals/logcom\": \"Journal of Logic and Computation\",\n",
    "    \"conf/fois\": \"Formal Ontology in Information Systems (FOIS)\",\n",
    "    \"journals/tkdd\": \"ACM Transactions on Knowledge Discovery from Data\",\n",
    "    \"conf/fab\": \"International Symposium on Foundations and Applications of Blockchain (FAB) \",\n",
    "}\n",
    "# Edits and cleanups of above entries where required.\n",
    "edited_strings = {\n",
    "    # These keys should not exist.\n",
    "    \"phd/basesearch\": None,\n",
    "    \"books/acm\": None,\n",
    "    \"books/sp\": None,\n",
    "    \"books/ph\": None,\n",
    "    \"phd/us\": None,\n",
    "    \"phd/ca\": None,\n",
    "    # Override entries, e.g., for missing keys or to make string shorter.\n",
    "    \"conf/sigir\": \"International Conference on Research and Development in Information Retrieval (SIGIR)\",\n",
    "    \"journals/corr\": \"ArXiv\",\n",
    "    \"conf/ntcir\": \"Conference on Evaluation of Information Access Technologies (NTCIR)\",\n",
    "    \"journals/tkde\": \"IEEE Transactions on Knowledge and Data Engineering (TKDE)\",\n",
    "    \"conf/acl\": \"Association for Computational Linguistics (ACL)\",\n",
    "    \"journals/cl\": \"Computer Languages, Systems & Structures\",\n",
    "    \"conf/ieeehpcs\": \"International Conference on High Performance Computing & Simulation (HPCS)\",\n",
    "    \"conf/otm\": \"OnTheMove Federated Conferences & Workshops (OTM)\",\n",
    "    \"conf/btw\": \"Datenbanksysteme für Business, Technologie und Web(BTW)\",\n",
    "    \"conf/icdcsw\": \"International Conference on Distributed Computing Systems (ICDCS) - Workshops\",\n",
    "    \"conf/desires\": \"Conference on Design of Experimental Search & Information Retrieval Systems (DESIRES)\",\n",
    "    \"conf/asist\": \"ASIS&T Annual Meeting (ASIST)\",\n",
    "    \"conf/ppopp\": \"ACM Symposium on Principles & Practice of Parallel Programming (PPoPP)\",\n",
    "    \"conf/bda\": \"Journées Bases de Données Avancées (BDA)\",\n",
    "    \"journals/dke\": \"Data & Knowledge Engineering (DKE)\",\n",
    "    \"conf/complexnetworks\": \"International Workshop on Complex Networks & Their Applications\",\n",
    "    \"journals/ki\": \"German Journal of Artificial Intelligence (KI)\",\n",
    "    \"journals/csse\": \"Computer Systems: Science & Engineering\",\n",
    "    \"journals/csi\": \"Computer Standards & Interfaces\",\n",
    "    \"conf/sigmod\": \"ACM International Conference on Management of Data (SIGMOD)\",\n",
    "    \"journals/pvldb\": \"Proceedings of the VLDB Endowment (PVLDB)\",\n",
    "    \"conf/ipccc\": \"IEEE International Performance, Computing, and Communications Conference (IPCCC)\",\n",
    "    \"conf/p2p\": \"IEEE International Conference on Peer-to-Peer Computing (P2P)\",\n",
    "    \"books/sp/13\": \"Handbook of Data Quality\",\n",
    "    \"books/mc/19\": \"Making Databases Work: the Pragmatic Wisdom of Michael Stonebraker\",\n",
    "    \"books/sp/17\": \"Algorithms for Next-Generation Sequencing Data, Techniques, Approaches, and Applications\",\n",
    "    \"conf/semeval\": \"International Workshop on Semantic Evaluation (SemEval)\",\n",
    "    \"conf/dexaw\": \"International Conference on Database and Expert Systems Applications (DEXA) - Workshops\",\n",
    "    \"phd/ndltd\": \"Massachusetts Institute of Technology, USA\",\n",
    "    \"books/crc/chb\": \"Computing Handbook: Information Systems and Information Technology\",\n",
    "    \"conf/birthday\": \"Description Logic, Theory Combination, and All That - Essays Dedicated to Franz Baader\",\n",
    "    \"journals/anthology\": \"ACM SIGMOD Anthology\",\n",
    "    \"books/mk/dittrichG01\": \"Component Database Systems\",\n",
    "    \"books/crc/tucker97\": \"The Computer Science and Engineering Handbook\",\n",
    "    \"books/acm/kim95\": \"Modern Database Systems: The Object Model, Interoperability, and Beyond\",\n",
    "    \"books/mk/elmagarmid92\": \"Database Transaction Models for Advanced Applications\",\n",
    "    \"tr/gte\": \"GTE Laboratories Incorporated\",\n",
    "    \"conf/usenix\": \"USENIX Annual Technical Conference (ATC)\",\n",
    "    \"conf/pods\": \"ACM Symposium on Principles of Database Systems (PODS)\",\n",
    "    \"books/others/11\": \"Interactive Information Seeking, Behaviour and Retrieval\",\n",
    "    \"journals/dpd\": \"Distributed and Parallel Databases\",\n",
    "    \"conf/sigada\": \"SIGAda Conference\",\n",
    "    \"conf/blackboxnlp\": \"Workshop on Analyzing and Interpreting Neural Networks for NLP (BlackboxNLP)\",\n",
    "    \"journals/health\": \"ACM Transactions on Computing for Healthcare (HEALTH)\",\n",
    "    \"conf/um\": \"User Modeling, Adaptation, and Personalization (UMAP)\",\n",
    "    \"conf/cec\": \"IEEE Congress on Evolutionary Computation (CEC)\",\n",
    "    \"conf/wadl\": \"Web Archiving and Digital Libraries Workshop (WADL)\",\n",
    "    \"journals/toit\": \"ACM Transactions on Internet Technology (TOIT)\",\n",
    "    \"conf/ruleml\": \"International Web Rule Symposium (RuleML)\",\n",
    "    \"journals/cbm\": \"Computers in Biology and Medicine\",\n",
    "    \"conf/IEEEpact\": \"International Conference on Parallel Architectures and Compilation Techniques (PACT)\",\n",
    "    \"conf/acl-louhi\": \"International Workshop on Health Text Mining and Information Analysis (Louhi)\",\n",
    "    \"conf/evia\": \"International Workshop on Evaluating Information Access (EVIA)\",\n",
    "    \"conf/atal\": \"International Joint Conference on Autonomous Agents &amp; Multiagent Systems (AAMAS)\",\n",
    "    \"conf/nips\": \"Conference on Neural Information Processing Systems (NeurIPS)\",\n",
    "    \"journals/pacmmod\": \"Proceedings of the ACM on Management of Data\",\n",
    "    \"journals/tweb\": \"ACM Transactions on the Web\",\n",
    "    \"conf/sigir-ap\": \"ACM SIGIR Conference on Research and Development in Information Retrieval in the Asia Pacific Region (SIGIR-AP)\",\n",
    "    \"conf/fire\": \"Forum for Information Retrieval Evaluation (FIRE)\",\n",
    "    \"conf/csci\": \"International Conference on Computational Science and Computational Intelligence (CSCI)\",\n",
    "    \"conf/guide-ai\": \"Workshop in Governance, Understanding and Integration of Data for Effective and Responsible AI (GUIDE-AI)\",\n",
    "    \"journals/eninf\": \"Energy Informatics\",\n",
    "    \"journals/jacm\": \"Journal of the ACM\",\n",
    "    \"conf/wads\": \"International Symposium on Algorithms and Data Structures (WADS)\",\n",
    "    \"conf/icalp\": \"International Colloquium on Automata, Languages and Programming (ICALP)\",\n",
    "    \"journals/iacr\": \"IACR Cryptology ePrint Archive\",\n",
    "    \"conf/uss\": \"USENIX Security Symposium\",\n",
    "    \"conf/sofsem\": \"Conference on Current Trends in Theory and Practice of Computer Science (SOFSEM)\",\n",
    "    \"journals/tcci\": \"Transactions on Computational Collective Intelligence (TCCI)\",\n",
    "    \"conf/comad\": \"Joint International Conference on Data Science & Management of Data (COMAD)\",\n",
    "    \"journals/semweb\": \"Semantic Web\",\n",
    "    \"conf/tapp\": \"Workshop on the Theory and Practice of Provenance (TaPP)\",\n",
    "    \"conf/birte\": \"Real-Time Business Intelligence and Analytics (BIRTE)\",\n",
    "    \"books/sp/12\": \"Semantic Search over the Web\",\n",
    "    \"conf/iq\": \"International Conference on Information Quality (ICIQ)\",\n",
    "    \"conf/i-semantics\": \"International Conference on Semantic Systems (SEMANTiCS)\",\n",
    "    \"conf/iqis\": \"International Workshop on Quality in Databases (QDB)\",\n",
    "    \"conf/dmkd\": \"Workshop on Research Issues on Data Mining and Knowledge Discovery (DMKD)\",\n",
    "    \"journals/ijseke\": \"International Journal of Software Engineering and Knowledge Engineering (IJSEKE)\",\n",
    "    \"journals/comcom\": \"Computer Communications\",\n",
    "    \"journals/ieeemm\": \"IEEE MultiMedia\",\n",
    "    \"conf/md\": \"IEEE Metadata Conference (MD)\",\n",
    "}\n",
    "# Combine the two above.\n",
    "final_strings = {}\n",
    "for k,v in edited_strings.items():\n",
    "    if v is not None:\n",
    "        final_strings[k] = v                               # Add overridden entries first.\n",
    "for k,v in raw_strings.items():\n",
    "    if k in edited_strings and edited_strings[k] is None:\n",
    "        continue                                           # Ignore entries that are explicitly set to None.\n",
    "    if k not in final_strings:\n",
    "        final_strings[k] = v\n",
    "\n",
    "# Replace publisher names.\n",
    "publisher_strings = {\n",
    "    'Springer US': 'Springer',\n",
    "    'Morgan & Claypool Publishers': 'Morgan & Claypool',\n",
    "}\n",
    "\n",
    "# Override titles, e.g., when titles contain LATEX.\n",
    "title_strings = {\n",
    "    'DBLP:conf/ausai/TomanW15': 'On the Krom Extension of CFDI^{∀−}_{nc}',\n",
    "    'DBLP:conf/ijcai/JacquesTW16': 'Object-Relational Queries over CFDI^{∀−}_{nc} Knowledge Bases: OBDA for the SQL-Literate',\n",
    "    'DBLP:conf/pricai/TomanW14': 'On Adding Inverse Features to the Description Logic CFD^{∀}_{nc}',\n",
    "    'DBLP:conf/ausai/TomanW13': 'Conjunctive Query Answering in CFD_{nc}: A PTIME Description Logic with Functional Constraints and Disjointness',\n",
    "    'DBLP:journals/dpd/HerodotouCCHDWC22': 'Introduction to the special issue on self‑managing and hardware‑optimized database systems 2020',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04e02f73-c3b7-4aef-9cc5-a8741b2340f1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Retrieve nomalized names of journals, conferences, and books.\n",
    "def get_names(bibids, found=set()):\n",
    "    new = False\n",
    "    for bibid in bibids:\n",
    "        remote_id = get_remote_id(bibid)\n",
    "        if edited_strings.get(remote_id, \"\") is None:\n",
    "            continue\n",
    "        if remote_id not in found:\n",
    "            try:\n",
    "                req = urllib.request.Request(\"https://dblp.org/db/%s/index.html\" % remote_id)\n",
    "                response = urllib.request.urlopen(req)\n",
    "                content = str(response.read())\n",
    "                m = re.search(\"<h1>(.*)</h1>\", content)\n",
    "                full_name = m.group(1)\n",
    "            except:\n",
    "                full_name = \"************ failed: %s\" % bibid\n",
    "            print('\"%s\": \"%s\",' % (remote_id, full_name))\n",
    "            found.add(remote_id)\n",
    "            new = True\n",
    "    return new\n",
    "\n",
    "# get_names(all_entries.keys()) # Uncomment to get names for all entries. Replace `raw_strings` with the output. Normally not required.\n",
    "\n",
    "if get_names(new_ids.keys(),set(final_strings.keys())):\n",
    "    raise Exception(\"Missing entries found. Add them to the `edited_strings` map.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "710d51ff-b272-4f0b-8c63-00b2919b3d81",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Finally create the combined and cleaned-up list of bibliographies ready to be added to the website.\n",
    "cleaned_entries = {}\n",
    "for entry in new_ids.values():\n",
    "    btype = entry['ENTRYTYPE']\n",
    "    efields = extra_fields[btype]\n",
    "    try:\n",
    "        cleaned_entry = {}\n",
    "        bibid = entry['ID']\n",
    "\n",
    "        cleaned_entry['id'] = bibid\n",
    "        cleaned_entry['btype'] = btype\n",
    "\n",
    "        if bibid in title_strings:\n",
    "            cleaned_entry['title'] = title_strings[bibid] # Use manually set titles\n",
    "        else:\n",
    "            cleaned_entry['title'] = titlecase.titlecase(entry['title']).replace(\"\\n\",' ').replace(\"\\\\emph\",'').replace(\"\\\\Emph\",'').replace(\"\\\\(±\\\\)\",\"\") # Cleanup stray escape characters.\n",
    "        if '\\\\' in cleaned_entry['title']:\n",
    "            raise Exception(\"Error: Make sure no escape character is present in the title by overriding name in `title_strings`: \" + cleaned_entry['title'])\n",
    "\n",
    "        authors = (entry['author'] if 'author' in entry else entry['editor']) if btype == 'book' else entry['author']\n",
    "        found = False\n",
    "        for author in authors_list:\n",
    "            for variant in author[2]:\n",
    "                if variant in authors:\n",
    "                    authors = authors.replace(variant, author[3]) # Normalize author names.\n",
    "                    found = True\n",
    "        if not found:\n",
    "            raise Exception(\"Error: Make sure atleast one of the authors belongs to the DSG group by normalizing in `authors_list`.\")\n",
    "        cleaned_entry['author'] = authors.split(\" and\\n\")\n",
    "\n",
    "        cleaned_entry['year'] = entry['year']\n",
    "        cleaned_entry['doi'] = entry.get('doi','')\n",
    "        cleaned_entry['url'] = entry.get('url','')\n",
    "\n",
    "        # Process fields specific to each bibliography type.\n",
    "        for efield in efields:\n",
    "            if efield in ['number','pages']:\n",
    "                data = entry.get(efield,'') # Optional.\n",
    "            elif efield == \"volume\" and \"journals/iacr\" in entry[\"biburl\"]:\n",
    "                data = \"\" # No volumes.\n",
    "            else:\n",
    "                data = entry[efield]\n",
    "            data = data.replace(\"\\n\",' ')\n",
    "\n",
    "            # Normalize conference, journal, and publisher names.\n",
    "            if efield in ['journal','booktitle']:\n",
    "                remote_id = get_remote_id(bibid)\n",
    "                if remote_id not in final_strings:\n",
    "                    raise Exception(\"Key missing. See previous cell and add missing entries to `final_strings`.\")\n",
    "                data = final_strings[remote_id]\n",
    "            elif efield == 'publisher':\n",
    "                if data in publisher_strings:\n",
    "                    data = publisher_strings[data]\n",
    "\n",
    "            cleaned_entry[efield] = data\n",
    "        cleaned_entries[bibid] = cleaned_entry\n",
    "    except Exception as e:\n",
    "        print(entry)\n",
    "        raise Exception(\"Error!\")\n",
    "print(len(cleaned_entries))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4413e38c-ee55-4e88-ac66-3f5eb147dd80",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Persist to disk in csv format.\n",
    "fields = ['id','btype','title','author','year','journal','volume','number','pages','booktitle','publisher','school','doi','url']\n",
    "separator = '\\t'\n",
    "bibfile = {}\n",
    "with open(f\"{data_dir}/combined.csv\",\"w\") as f:\n",
    "    f.write(separator.join(fields))\n",
    "    f.write('\\n')\n",
    "    for bibid,entry in cleaned_entries.items():\n",
    "        data = []\n",
    "        for field in fields:\n",
    "            if field == 'author':\n",
    "                data.append(\" and \".join(entry[field]))\n",
    "            else:\n",
    "                data.append(entry.get(field,''))\n",
    "        \n",
    "        f.write(separator.join(data))\n",
    "        f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d117232-66ab-4bbb-90f8-3adbdd56fbb6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load info about entries already added to the website. Contains mapping from DBLP ID to Node ID on the website.\n",
    "# This is needed to automate modifying and deleting existing entries.\n",
    "# IMPORTANT: Make sure this file is correctly maintained.\n",
    "def load_mappings():\n",
    "    mappings = {}\n",
    "    with open(mappings_file) as f:\n",
    "        for line in f:\n",
    "            parts = line.rstrip().split(\",\")\n",
    "            mappings[parts[0]] = int(parts[1])\n",
    "    return mappings\n",
    "\n",
    "mappings = load_mappings()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca73dd03-119d-4be7-b5a7-7a80424ea806",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Login to 'My Workbench' and copy the cookie value of key 'SSESSc2214f6938283aa908e28c4bfa176f5b'\n",
    "# DANGER: since the repo is public, make sure not to commit the cookie value, otherwise anyone on the internet would be able to use it to login to Workbench.\n",
    "cookie_value = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68541719-4cb7-4450-83dc-dacd0da0180c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Open a test page using the provided cookies. If you get an \"Access denied\" error, the cookie value was incorrect.\n",
    "assert len(cookie_value) != 0\n",
    "opts = webdriver.ChromeOptions()\n",
    "opts.binary_location = \"/run/current-system/sw/bin/chromium\"\n",
    "service = webdriver.ChromeService(executable_path=\"/run/current-system/sw/bin/chromedriver\")\n",
    "browser = webdriver.Chrome(options=opts,service=service)\n",
    "browser.get('https://uwaterloo.ca')\n",
    "browser.add_cookie({'name' : 'SSESSc2214f6938283aa908e28c4bfa176f5b', 'value' : cookie_value}) \n",
    "browser.add_cookie({'name' : 'Drupal.tableDrag.showWeight', 'value' : '0'})\n",
    "browser.add_cookie({'name' : 'cookie-agreed-version', 'value' : '1.0.0'}) \n",
    "browser.add_cookie({'name' : 'has_js', 'value' : '1'}) \n",
    "browser.get('https://uwaterloo.ca/data-systems-group/node/add/biblio')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e856f06-3453-4971-84a4-fbd4bbb4dcb5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Helper functions to set bibliography details, specific to each type.\n",
    "# Note: Might have to fiddle with sleep() and scrollBy() values if selenium throws any error.\n",
    "\n",
    "def post_author(entry):\n",
    "    next_click = 4\n",
    "    for i,author in enumerate(entry['author']):\n",
    "        if i == next_click:\n",
    "            browser.find_element(\"xpath\",'//*[@id=\"edit-add-more\"]').click()\n",
    "            next_click += 2\n",
    "            browser.execute_script(\"window.scrollBy(0,100)\")\n",
    "            time.sleep(2)\n",
    "        browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-contributors-%d-name\"]' % i).send_keys(author)\n",
    "    browser.execute_script(\"window.scrollTo(0,10)\")\n",
    "\n",
    "def post_doi_url(entry):\n",
    "    if entry.get('doi','') != '':\n",
    "        browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-doi\"]').send_keys(entry['doi'])\n",
    "    url = entry.get('url','')\n",
    "    if entry.get('doi','') == '' or 'doi.org' not in url:\n",
    "        browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-url\"]').send_keys(url)\n",
    "\n",
    "def save_page():\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-submit\"]').click()\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-state\"]/option[text()=\"Published\"]').click()\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-submit\"]').click()\n",
    "    try:\n",
    "        href = browser.find_element(\"xpath\",'//*[@id=\"main\"]/div/div[2]/ul/li[2]/a').get_attribute('href')\n",
    "        pid = int(href.split(\"/\")[-2])\n",
    "    except:\n",
    "        print(href)\n",
    "        raise Exception(\"Error\")\n",
    "    return pid\n",
    "\n",
    "def post_conference_paper(entry):\n",
    "    browser.get('https://uwaterloo.ca/data-systems-group/node/add/biblio')\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-type\"]/option[text()=\"Conference Paper\"]').click()\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-title\"]').send_keys(entry['title'])\n",
    "    post_author(entry)\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[4]/a').click()\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-year\"]').send_keys(entry['year'])\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-secondary-title\"]').send_keys(entry['booktitle'])\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[7]/a').click()\n",
    "    post_doi_url(entry)\n",
    "\n",
    "    return save_page()\n",
    "\n",
    "def post_article(entry):\n",
    "    browser.get('https://uwaterloo.ca/data-systems-group/node/add/biblio')\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-type\"]/option[text()=\"Journal Article\"]').click()\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-title\"]').send_keys(entry['title'])\n",
    "    post_author(entry)\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[4]/a').click()\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-year\"]').send_keys(entry['year'])\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-secondary-title\"]').send_keys(entry['journal'])\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-volume\"]').send_keys(entry['volume'])\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-issue\"]').send_keys(entry['number'])\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-pages\"]').send_keys(entry['pages'])\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[6]/a').click()\n",
    "    post_doi_url(entry)\n",
    "\n",
    "    return save_page()\n",
    "\n",
    "def post_book(entry):\n",
    "    browser.get('https://uwaterloo.ca/data-systems-group/node/add/biblio')\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-type\"]/option[text()=\"Book\"]').click()\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-title\"]').send_keys(entry['title'])\n",
    "    post_author(entry)\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[4]/a').click()\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-year\"]').send_keys(entry['year'])\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[5]/a').click()\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-publisher\"]').send_keys(entry.get('publisher',''))\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[7]/a').click()\n",
    "    post_doi_url(entry)\n",
    "\n",
    "    return save_page()\n",
    "\n",
    "def post_book_chapter(entry):\n",
    "    browser.get('https://uwaterloo.ca/data-systems-group/node/add/biblio')\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-type\"]/option[text()=\"Book Chapter\"]').click()\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-title\"]').send_keys(entry['title'])\n",
    "    post_author(entry)\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[4]/a').click()\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-year\"]').send_keys(entry['year'])\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-secondary-title\"]').send_keys(entry['booktitle'])\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[5]/a').click()\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-publisher\"]').send_keys(entry.get('publisher',''))\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[7]/a').click()\n",
    "    post_doi_url(entry)\n",
    "\n",
    "    return save_page()\n",
    "\n",
    "def post_thesis(entry):\n",
    "    browser.get('https://uwaterloo.ca/data-systems-group/node/add/biblio')\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-type\"]/option[text()=\"Thesis\"]').click()\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-title\"]').send_keys(entry['title'])\n",
    "    post_author(entry)\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[4]/a').click()\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-year\"]').send_keys(entry['year'])\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[5]/a').click()\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-biblio-publisher\"]').send_keys(entry['school'])\n",
    "\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"biblio-node-form\"]/div/div[3]/ul/li[7]/a').click()\n",
    "    post_doi_url(entry)\n",
    "\n",
    "    return save_page()\n",
    "\n",
    "def delete_entry(nodeid):\n",
    "    browser.get('https://uwaterloo.ca/data-systems-group/node/%s/delete' % nodeid)\n",
    "    browser.find_element(\"xpath\",'//*[@id=\"edit-submit\"]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acebe66b-e422-47b5-8a29-d8f5db3612f6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Create a deterministic list of keys from the dict to make continuing from errors easier.\n",
    "keys = list(cleaned_entries.keys())\n",
    "print(len(keys))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b511efe-9dee-4f45-b958-2fa74aa4eb7a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set to the index of `'keys' from which to start uploading.\n",
    "# Should be set to `0` on a fresh run or to a particular index after recovering from an error.\n",
    "index = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3105d2dc-4675-4b3d-9cd0-3dad2fc7ce12",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Publish new entries to the website.\n",
    "# Note: Do not try to scroll or interact with the running browser as it may cause an error in Selenium.\n",
    "try:\n",
    "    # Simultaneously write successful ids to disk to prevent posting duplicate entries on the next run.\n",
    "    with open(mappings_file,\"a\") as f:\n",
    "        while index < len(keys):\n",
    "            bibid = keys[index]\n",
    "\n",
    "            if bibid in mappings:\n",
    "                raise Exception(f\"'{bibid}' present in mappings. Was it already posted?\")\n",
    "\n",
    "            entry = cleaned_entries[bibid]\n",
    "            btype = entry['btype']\n",
    "            if btype == \"article\":\n",
    "                post_id = post_article(entry)\n",
    "            elif btype == \"inproceedings\":\n",
    "                post_id = post_conference_paper(entry)\n",
    "            elif btype == \"incollection\":\n",
    "                post_id = post_book_chapter(entry)\n",
    "            elif btype == \"book\":\n",
    "                post_id = post_book(entry)\n",
    "            elif btype == \"phdthesis\":\n",
    "                post_id = post_thesis(entry)\n",
    "            else:\n",
    "                raise Exception(\"Not found\")\n",
    "\n",
    "            mappings[bibid] = post_id\n",
    "            f.write(\"%s,%s\\n\" % (bibid, post_id))\n",
    "            f.flush()\n",
    "            print(\"Posted:\",index, entry['title'],bibid,post_id)\n",
    "            index += 1\n",
    "            time.sleep(1)\n",
    "        print(\"Done!\")\n",
    "except:\n",
    "    pp = pprint.PrettyPrinter()\n",
    "    pp.pprint(entry)\n",
    "    raise Exception(\"Error in index\", index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "878d6695-3a42-4027-b267-a99f7e2200bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Delete old entries no longer on DBLP\n",
    "for bibid,entry in deleted_ids.items():\n",
    "    nodeid = mappings[bibid]\n",
    "    print(\"Deleting: %s | %s | %s | %s | %s\" % (nodeid, bibid, entry['title'].replace('\\n',' '), entry['author'].replace('\\n',' '), entry['biburl']))\n",
    "    delete_entry(nodeid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8ae6d03-000c-4d6d-9d26-e03e305b8b57",
   "metadata": {},
   "outputs": [],
   "source": [
    "mappings = load_mappings()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a287b40-9520-4f84-91cc-88a37ae2ff6d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Make sure the mappings for all the `bibid`s are present on disk and correctly reflects the ids posted online.\n",
    "mappings_check = load_mappings()\n",
    "for bibid in keys:\n",
    "    print(mappings[bibid], bibid, cleaned_entries[bibid]['title'])\n",
    "    assert bibid in mappings_check, \"'{bibid}' not present in mappings file\"\n",
    "    assert mappings[bibid] == mappings_check[bibid], f\"'{mappings[bibid]}' != '{mappings_check[bibid]}'\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d366bc6c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Replace old data with new\n",
    "shutil.rmtree(authors_dirs_previous)\n",
    "os.rename(authors_dirs_new, authors_dirs_previous)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe442ee7-c18b-4601-bb9b-f6151adb0843",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 1. IMPORTANT: Delete the cookie value above and rerun cell.\n",
    "assert len(cookie_value) == 0\n",
    "# 2. make sure to reset index to 0 if it was changed, to avoid missing entries in the next run. Just rerun the `index = ?` cell.\n",
    "assert index == 0\n",
    "# 3. Clear all cells outputs.\n",
    "# 4. Save the notebook.\n",
    "# 5. Commit to git and push."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb7b5a77-5e8a-48a6-8b77-534021fc2e12",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sample code to bulk delete entries\n",
    "\n",
    "# browser.get('https://uwaterloo.ca/data-systems-group/admin/workbench/archived')\n",
    "# while True:\n",
    "#     browser.find_element(\"xpath\",'//*[@id=\"block-system-main\"]/div/div/div[2]/table/tbody/tr[1]/td[5]/ul/li[4]/a').click()\n",
    "#     browser.find_element(\"xpath\",'//*[@id=\"edit-submit\"]').click()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}