bot_datagen.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Listing all Commits for author-to-commit map for using the commands in README\n",
    "import gzip\n",
    "com = []\n",
    "with gzip.open('/da4_data/play/botDetection/paper_a2c.gz', 'rt', encoding = 'iso-8859-15') as f:\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        parts = line.split(';')\n",
    "        for i in range(1,len(parts)):\n",
    "            com.append(parts[i])\n",
    "\n",
    "with gzip.open('/da4_data/play/botDetection/paper_commits.gz', 'wt') as f:\n",
    "    f.write('\\n'.join(com))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This script assumes the following files are available:\n",
    "1. author-to-commit (a2c) map for the suspected bots ('/da4_data/play/botDetection/paper_a2c.gz')\n",
    "2. commit-to-content (c2cc) map for the commits in question ('/da4_data/play/botDetection/paper_cnt.gz')\n",
    "3. commit-to-project (c2p) map for the commits. ('/da4_data/play/botDetection/paper_c2p.gz')\n",
    "4. commit-to-file(c2f) map for the commits. ('/da4_data/play/botDetection/paper_c2f.gz')\n",
    "\n",
    "See `README` for corresponding commands using World of Code tool.\n",
    "\n",
    "## This script creates the following data:\n",
    "1. Data for running BICA ('/da4_data/play/botDetection/test_Info_paper.csv.gz')\n",
    "2. Data after running BIM ('/da4_data/play/botDetection/paper_template.out')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from alignment.sequence import Sequence\n",
    "from alignment.vocabulary import Vocabulary\n",
    "from alignment.sequencealigner import SimpleScoring, GlobalSequenceAligner\n",
    "import ast\n",
    "import json\n",
    "import gzip\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adding Timeout\n",
    "import signal\n",
    "\n",
    "class timeout:\n",
    "    def __init__(self, seconds=1, error_message='Timeout'):\n",
    "        self.seconds = seconds\n",
    "        self.error_message = error_message\n",
    "    def handle_timeout(self, signum, frame):\n",
    "        raise TimeoutError(self.error_message)\n",
    "    def __enter__(self):\n",
    "        signal.signal(signal.SIGALRM, self.handle_timeout)\n",
    "        signal.alarm(self.seconds)\n",
    "    def __exit__(self, type, value, traceback):\n",
    "        signal.alarm(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepapring Author - Commit Message Map for Generating Data for BIM\n",
    "pauthdict = {}\n",
    "with gzip.open('/da4_data/play/botDetection/paper_a2c.gz', 'rt', encoding = 'iso-8859-15') as f:\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        parts = line.split(';')\n",
    "        pauthdict[parts[0]] = {'commits':parts[1:], 'message':[]}\n",
    "\n",
    "pcc = {}        \n",
    "with gzip.open('/da4_data/play/botDetection/paper_cnt.gz', 'rt', encoding = 'iso-8859-15') as f:\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        parts = line.split(';')\n",
    "        pcc[parts[0]] = ';'.join(parts[3:])\n",
    "        \n",
    "    \n",
    "for key in pauthdict.keys():\n",
    "    commits = pauthdict[key]['commits']\n",
    "    for com in commits:\n",
    "        try:\n",
    "            pauthdict[key]['message'].append(pcc[com])\n",
    "        except:\n",
    "            continue   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generating Data for BIM\n",
    "from collections import defaultdict\n",
    "bin_threshold = [40]\n",
    "id_threshold = 0.5 # 50 percent\n",
    "max_bot_bin = 500\n",
    "# nbhumans = defaultdict(list)\n",
    "# nbbots = defaultdict(list)\n",
    "\n",
    "fw = open('/da4_data/play/botDetection/paper_template.out','wb')\n",
    "\n",
    "for threshold in bin_threshold:\n",
    "    for key in pauthdict.keys():\n",
    "        author, msgs = key, pauthdict[key]['message']\n",
    "        print (author, len(msgs))\n",
    "\n",
    "        if len (msgs) == 1:\n",
    "            ost = ';'.join([author, str(len(msgs)), str(1), str(1)])+'\\n'\n",
    "            fw.write(ost.encode('utf-8'))\n",
    "            continue\n",
    "        elif len (msgs) > 100000:\n",
    "            ost = ';'.join([author, str(len(msgs)), str(1), str(0)])+'\\n'\n",
    "            fw.write(ost.encode('utf-8'))\n",
    "            continue\n",
    "        bins = {}\n",
    "        bratio = 0\n",
    "        i = 0\n",
    "        try:\n",
    "            with timeout(seconds=60):                \n",
    "                for commit in msgs:\n",
    "                    i += 1\n",
    "                    if len(bins) == 0:\n",
    "                        bins[0] = [(commit, 100)]\n",
    "                    elif len(commit) >= 200:\n",
    "                        bins[len(bins)] = [(commit, 100)]\n",
    "                        continue\n",
    "                    else: \n",
    "                        '''\n",
    "                        # Create sequences to be aligned.\n",
    "                        b = Sequence('what a beautiful day'.split())\n",
    "                        a = Sequence('what a disappointingly bad day'.split())\n",
    "                        '''\n",
    "                        a = Sequence(commit.split())\n",
    "                        added = False\n",
    "                        brflag = False\n",
    "                        for key in bins:\n",
    "                            b = Sequence(bins[key][0][0].split()) #first eleman of the tuple in the list\n",
    "                            # Create a vocabulary and encode the sequences.\n",
    "                            v = Vocabulary()   \n",
    "                            try:                     \n",
    "                                aEncoded = v.encodeSequence(a)\n",
    "                                bEncoded = v.encodeSequence(b)\n",
    "\n",
    "                                # Create a scoring and align the sequences using global aligner.\n",
    "                                scoring = SimpleScoring(2, -1)\n",
    "                                aligner = GlobalSequenceAligner(scoring, -2)                    \n",
    "                                score, encodeds = aligner.align(aEncoded, bEncoded, backtrace=True)                        \n",
    "\n",
    "                                # Iterate over optimal alignments and print them.\n",
    "                                pi_max = 0\n",
    "                                score_ = 0\n",
    "                                for encoded in encodeds:                               \n",
    "                                    alignment = v.decodeSequenceAlignment(encoded)\n",
    "                                    score_ = alignment.score\n",
    "                                    percentIdentity =  alignment.percentIdentity()\n",
    "                                    if percentIdentity > pi_max : pi_max = percentIdentity                    \n",
    "\n",
    "                                if pi_max > threshold:\n",
    "            #                         print (pi_max)\n",
    "                                    bins[key].append((commit, percentIdentity)) # add b and similarity\n",
    "                                    added = True\n",
    "                                    break\n",
    "                            except KeyboardInterrupt:\n",
    "                                print ('KeyboardInterrupt')\n",
    "                                break\n",
    "                            except:\n",
    "                                brflag = True\n",
    "                                break\n",
    "                        if brflag:\n",
    "                            bins = {}\n",
    "                            break\n",
    "                        if added == False:\n",
    "                            bins[len(bins)] = [(commit, 100)]\n",
    "                        if len(bins) > max_bot_bin:                            \n",
    "                            bratio = 1\n",
    "                            break\n",
    "                            \n",
    "        except KeyboardInterrupt:\n",
    "            print ('KeyboardInterrupt')\n",
    "            break\n",
    "        except TimeoutError:\n",
    "            print ('Timeout')\n",
    "            ost = ';'.join([author, str(i), str(len(bins.keys())), str(ratio)])+'\\n'\n",
    "            fw.write(ost.encode('utf-8'))\n",
    "            continue\n",
    "        except Exception as e:\n",
    "            print (e)\n",
    "            break\n",
    "                \n",
    "        num_commits = len(msgs)\n",
    "        ratio = max(len(bins.keys()) / num_commits, bratio)\n",
    "        ost = ';'.join([author, str(num_commits), str(len(bins.keys())), str(ratio)])+'\\n'\n",
    "        fw.write(ost.encode('utf-8'))\n",
    "        \n",
    "fw.close()   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This Concludes Data Generation for BIM\n",
    "\n",
    "Format of output data:\n",
    "\n",
    "`Author ID; No. of Commits; No. of Bins; Ratio`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converting Clock Time of commit to standardized time with time zone correction\n",
    "fc = ''\n",
    "from datetime import datetime\n",
    "wf = gzip.open('/da4_data/play/botDetection/paper_cnt2.gz', 'wt', encoding = 'iso-8859-15')\n",
    "with gzip.open('/da4_data/play/botDetection/paper_cnt.gz', 'rt', encoding = 'iso-8859-15') as f:\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        parts = line.split(';')\n",
    "        time = parts[2]\n",
    "        del(parts[2])\n",
    "        old = int(time.split()[0])\n",
    "        epoch = int(time.split()[0])\n",
    "        tz = time.split()[1]\n",
    "        if tz[0] == '+':\n",
    "            epoch = epoch + (int(tz[1:3])*3600 + int(tz[3:5])*60)\n",
    "        elif  tz[0] == '-':\n",
    "            epoch = epoch - (int(tz[1:3])*3600 + int(tz[3:5])*60)\n",
    "\n",
    "        ts =  datetime.fromtimestamp(epoch)\n",
    "        oldts = datetime.fromtimestamp(old)\n",
    "\n",
    "        fc = ';'.join(parts) +';'+str(oldts) +';'+str(ts) + ';'+str(tz)+'\\n'\n",
    "        wf.write(fc)\n",
    " \n",
    "wf.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating required dicts for data generation for BICA\n",
    "\n",
    "fc2content = dict()\n",
    "with gzip.open('/da4_data/play/botDetection/paper_cnt2.gz', 'rt', encoding = 'iso-8859-15') as f:\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        parts = line.split(';')\n",
    "        fc2content[parts[0]] = {'message':parts[2], 'clock.time':parts[3], 'timezone':parts[5]}\n",
    "\n",
    "\n",
    "with gzip.open('/da4_data/play/botDetection/paper_c2f.gz', 'rt', encoding = 'iso-8859-15') as f:\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        parts = line.split(';')\n",
    "        try:\n",
    "            fc2content[parts[0]]['files'] = list(set(parts[1:]) )\n",
    "        except:\n",
    "            try:\n",
    "                fc2content[parts[0]]['files'] = []\n",
    "            except:\n",
    "                continue\n",
    "                \n",
    "                \n",
    "with gzip.open('/da4_data/play/botDetection/paper_c2p.gz', 'rt', encoding = 'iso-8859-15') as f:\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        parts = line.split(';')\n",
    "        try:\n",
    "            fc2content[parts[0]]['projects'] = list(set(parts[1:]) )\n",
    "        except:\n",
    "            try:\n",
    "                fc2content[parts[0]]['projects'] = []\n",
    "            except:\n",
    "                continue     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create Final Dataset for BICA - no aliasing\n",
    "import statistics as stat\n",
    "from datetime import datetime, timedelta\n",
    "from scipy.stats import circvar\n",
    "from collections import Counter\n",
    "from scipy.stats import iqr\n",
    "\n",
    "\n",
    "\n",
    "out = 'Author, No.Commit, Days.Active, Avg.Commit.pYear, Median.Commit.pYear, Activity.Hours, \\\n",
    "Spike.Hours, Circ.Variance.Hour, Tot.uniq.FilesChanged, Tot.FilesChanged, Uniq.File.Exten,\\\n",
    "Avg.File.pCommit, Std.File.pCommit, No.Timezones, Std.Timezones, Tot.Projects, Tot.uniq.Projects, \\\n",
    "Median.Project.pCommit, Std.Project.pCommit \\n'\n",
    "\n",
    "f= gzip.open('/da4_data/play/botDetection/test_Info_paper.csv.gz','wt',  encoding = 'iso-8859-15') \n",
    "f.write(out)\n",
    "#i = 0\n",
    "with gzip.open('/da4_data/play/botDetection/paper_a2c.gz', 'rt', encoding = 'iso-8859-15') as cf:\n",
    "    for line in cf:\n",
    "        line = line.strip()\n",
    "        parts = line.split(';')\n",
    "        auth = parts[0]\n",
    "        msgs, times, tz, files, n_files, proj, n_proj = ([] for i in range(7))\n",
    "        commits = parts[1:]\n",
    "        nc = len(commits) \n",
    "        print(auth, nc)\n",
    "        for com in commits:\n",
    "            try:\n",
    "                cont = fc2content[com]\n",
    "            except:\n",
    "                continue\n",
    "\n",
    "            #msgs.append(cont['message'])\n",
    "            times.append(datetime.strptime(cont['clock.time'], '%Y-%m-%d %H:%M:%S'))\n",
    "            ptz = cont['timezone']\n",
    "            tz.append((timedelta(hours=12)+timedelta(hours=int(ptz[0:2]),minutes=int(ptz[2:])) \\\n",
    "                       if '-' not in ptz else timedelta(hours=12)-timedelta(hours=int(ptz[1:3]),\\\n",
    "                                                                            minutes=int(ptz[3:]))).seconds/3600.0)\n",
    "            files += cont['files']\n",
    "            n_files.append( len(cont['files']))\n",
    "            proj += cont['projects']\n",
    "            n_proj.append(len(cont['projects']))\n",
    "\n",
    "        if len(times)  == 0:\n",
    "            continue\n",
    "        #File\n",
    "        tuf = len(set(files))\n",
    "        tf = len(files)\n",
    "        ufe = len(set([x.split('.')[-1] for x in files]))\n",
    "        \n",
    "        try:\n",
    "            afpc = stat.mean(n_files)\n",
    "            vfpc = stat.stdev(n_files)\n",
    "        except:\n",
    "            afpc = 0\n",
    "            vfpc = 0\n",
    "        #tz\n",
    "        ntz = len(set(tz))\n",
    "        try:\n",
    "            vtz = stat.stdev(tz)\n",
    "        except:\n",
    "            vtz = 0\n",
    "        #proj\n",
    "        tp = len(proj)\n",
    "        tup = len(set(proj))\n",
    "        \n",
    "        try:\n",
    "            mppc = stat.median(n_proj)\n",
    "            vppc = stat.stdev(n_proj)\n",
    "        except:\n",
    "            mppc = 0\n",
    "            vppc = 0\n",
    "        #time\n",
    "        yatd = max(times) - min(times)\n",
    "        ya = yatd.days+1\n",
    "        y_times = Counter([x.year for x in times])\n",
    "        acpy = stat.mean(list(y_times.values()))\n",
    "        mcpy = stat.median(list(y_times.values()))\n",
    "        h_times = Counter([x.hour for x in times])\n",
    "        cvhour = circvar(list(h_times.values()))\n",
    "        ih = iqr(list(h_times.values()))\n",
    "        ahour = len(h_times.keys())\n",
    "        sphour = len([x for x in h_times.values() if x > 1.5*ih])\n",
    "\n",
    "        # join\n",
    "        out = ','.join(str(x) for x in (auth.replace(',',''), nc,ya,acpy,mcpy,\\\n",
    "                                        ahour,sphour,cvhour,tuf,tf,ufe,afpc,vfpc,ntz,vtz,tp,tup,mppc,vppc))+'\\n'\n",
    "        f.write(out)\n",
    "\n",
    "    \n",
    "    \n",
    "\n",
    "f.close()        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This concludes data generation for BICA\n",
    "\n",
    "Not all the variables are used in the model we have, but you might try with other variables as well."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}