notebooks/1_download_twitter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import twint\n",
    "import nest_asyncio\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data directory\n",
    "data_dir = Path(\"../data\")\n",
    "\n",
    "input_dir = data_dir / \"input\"\n",
    "raw_dir = data_dir / \"raw\"\n",
    "\n",
    "twitter_dir = raw_dir / \"twitter\"\n",
    "\n",
    "# load news sources\n",
    "twitter_feeds = pd.read_csv(input_dir / \"twitter_feeds.csv\")\n",
    "\n",
    "shortnames = twitter_feeds[\"short_name\"].tolist()\n",
    "twitter_usernames = twitter_feeds[\"username\"].tolist()\n",
    "existing_files = list(twitter_dir.glob(\"*.csv\"))\n",
    "existing_files = [f.name for f in existing_files]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This scrape has already been conducted.\n",
      "This scrape has already been conducted.\n"
     ]
    }
   ],
   "source": [
    "# config\n",
    "since_date = \"2021-03-01\"\n",
    "until_date = \"2021-05-02\"\n",
    "\n",
    "for username, shortname in zip(twitter_usernames, shortnames):\n",
    "    filename = f\"{shortname}_{since_date}_{until_date}.csv\"\n",
    "    if filename in existing_files:\n",
    "        print(\"This scrape has already been conducted.\")\n",
    "        continue\n",
    "    else:\n",
    "        print(f\"Collecting {username} from {since_date} until {until_date}\")\n",
    "        c = twint.Config()\n",
    "        c.Username = username\n",
    "        c.Since = since_date\n",
    "        c.Until = until_date\n",
    "\n",
    "        c.Hide_output = True\n",
    "        c.Pandas = True\n",
    "\n",
    "        # Run\n",
    "        twint.run.Search(c)\n",
    "\n",
    "        df = twint.storage.panda.Tweets_df\n",
    "        df.to_csv(twitter_dir / filename)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "altmetric-news",
   "language": "python",
   "name": "altmetric-news"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}