SEC EDGAR Data Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "from nltk.tokenize import RegexpTokenizer, sent_tokenize\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mda_regex = r\"item[^a-zA-Z\\n]*\\d\\s*\\.\\s*management\\'s discussion and analysis.*?^\\s*item[^a-zA-Z\\n]*\\d\\s*\\.*\"\n",
    "qqd_regex = r\"item[^a-zA-Z\\n]*\\d[a-z]?\\.?\\s*Quantitative and Qualitative Disclosures about \" \\\n",
    "            r\"Market Risk.*?^\\s*item\\s*\\d\\s*\"\n",
    "riskfactor_regex = r\"item[^a-zA-Z\\n]*\\d[a-z]?\\.?\\s*Risk Factors.*?^\\s*item\\s*\\d\\s*\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopWordsFile = (r'C:\\Users\\Jinu Augustine\\Documents\\BlackCoffer/StopWords_Generic.txt')\n",
    "positiveWordsFile = (r'C:\\Users\\Jinu Augustine\\Documents\\BlackCoffer/PositiveWords.txt')\n",
    "nagitiveWordsFile = (r'C:\\Users\\Jinu Augustine\\Documents\\BlackCoffer/NegativeWords.txt')\n",
    "uncertainty_dictionaryFile = (r'C:\\Users\\Jinu Augustine\\Documents\\BlackCoffer/uncertainty_dictionary.txt')\n",
    "constraining_dictionaryFile = (r'C:\\Users\\Jinu Augustine\\Documents\\BlackCoffer/constraining_dictionary.txt')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rawdata_extract(path, cikListFile):\n",
    "    html_regex = re.compile(r'<.*?>')\n",
    "    extraxted_data=[]\n",
    "    \n",
    "    \n",
    "    cikListFile = pd.read_csv(cikListFile)\n",
    "    for index, row in cikListFile.iterrows():\n",
    "        processingFile=row['SECFNAME'].split('/')\n",
    "        inputFile = processingFile[3]\n",
    "        cik=row['CIK']\n",
    "        coname=row['CONAME']\n",
    "        fyrmo=row['FYRMO']\n",
    "        fdate = row['FDATE']\n",
    "        form = row['FORM']\n",
    "        secfname=row['SECFNAME']\n",
    "        for fileName in os.listdir(path):\n",
    "            filenameopen = os.path.join(path, fileName)\n",
    "            dirFileName = filenameopen.split('\\\\')\n",
    "            currentFile=dirFileName[1]\n",
    "\n",
    "            if os.path.isfile(filenameopen) and currentFile == inputFile :\n",
    "                resultdict = dict()\n",
    "                resultdict['CIK'] = cik\n",
    "                resultdict['CONAME'] = coname\n",
    "                resultdict['FYRMO'] = fyrmo\n",
    "                resultdict['FDATE'] = fdate\n",
    "                resultdict['FORM'] = form\n",
    "                resultdict['SECFNAME'] = secfname\n",
    "                \n",
    "                with open(filenameopen, 'r', encoding='utf-8', errors=\"replace\") as in_file:\n",
    "                    content = in_file.read()\n",
    "                    content = re.sub(html_regex,'',content)\n",
    "                    content = content.replace('&nbsp;','')\n",
    "                    content = re.sub(r'&#\\d+;', '', content)\n",
    "                    matches_mda = re.findall(mda_regex, content, re.IGNORECASE | re.DOTALL | re.MULTILINE)\n",
    "                    if matches_mda:\n",
    "                        result = max(matches_mda, key=len)\n",
    "                        result = str(result).replace('\\n', '')\n",
    "                        resultdict['mda_extract'] = result\n",
    "                    else:\n",
    "                        resultdict['mda_extract'] = \"\"\n",
    "                    match_qqd = re.findall(qqd_regex, content, re.IGNORECASE | re.DOTALL | re.MULTILINE)\n",
    "                    if match_qqd:\n",
    "                        result_qqd = max(match_qqd, key=len)\n",
    "                        result_qqd = str(result_qqd).replace('\\n','')\n",
    "                        resultdict['qqd_extract']= result_qqd\n",
    "                    else:\n",
    "                        resultdict['qqd_extract'] = \"\"\n",
    "                    match_riskfactor = re.findall(riskfactor_regex, content, re.IGNORECASE | re.DOTALL | re.MULTILINE)\n",
    "                    if match_riskfactor:\n",
    "                        result_riskfactor = max(match_riskfactor, key=len)\n",
    "                        result_riskfactor = str(result_riskfactor).replace('\\n', '')\n",
    "                        resultdict['riskfactor_extract'] = result_riskfactor\n",
    "                    else:\n",
    "                        resultdict['riskfactor_extract'] = \"\"\n",
    "                    extraxted_data.append(resultdict)\n",
    "\n",
    "                in_file.close()\n",
    "\n",
    "    return extraxted_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(stopWordsFile ,'r') as stop_words:\n",
    "    stopWords = stop_words.read().lower()\n",
    "stopWordList = stopWords.split('\\n')\n",
    "stopWordList[-1:] = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenizer(text):\n",
    "    text = text.lower()\n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    tokens = tokenizer.tokenize(text)\n",
    "    filtered_words = list(filter(lambda token: token not in stopWordList, tokens))\n",
    "    return filtered_words\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(positiveWordsFile,'r') as posfile:\n",
    "    positivewords=posfile.read().lower()\n",
    "positiveWordList=positivewords.split('\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(nagitiveWordsFile ,'r') as negfile:\n",
    "    negativeword=negfile.read().lower()\n",
    "negativeWordList=negativeword.split('\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def positive_score(text):\n",
    "    numPosWords = 0\n",
    "    rawToken = tokenizer(text)\n",
    "    for word in rawToken:\n",
    "        if word in positiveWordList:\n",
    "            numPosWords  += 1\n",
    "    \n",
    "    sumPos = numPosWords\n",
    "    return sumPos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def negative_word(text):\n",
    "    numNegWords=0\n",
    "    rawToken = tokenizer(text)\n",
    "    for word in rawToken:\n",
    "        if word in negativeWordList:\n",
    "            numNegWords -=1\n",
    "    sumNeg = numNegWords \n",
    "    sumNeg = sumNeg * -1\n",
    "    return sumNeg\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def polarity_score(positiveScore, negativeScore):\n",
    "    pol_score = (positiveScore - negativeScore) / ((positiveScore + negativeScore) + 0.000001)\n",
    "    return pol_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average_sentence_length(text):\n",
    "    sentence_list = sent_tokenize(text)\n",
    "    tokens = tokenizer(text)\n",
    "    totalWordCount = len(tokens)\n",
    "    totalSentences = len(sentence_list)\n",
    "    average_sent = 0\n",
    "    if totalSentences != 0:\n",
    "        average_sent = totalWordCount / totalSentences\n",
    "    \n",
    "    average_sent_length= average_sent\n",
    "    \n",
    "    return round(average_sent_length)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def percentage_complex_word(text):\n",
    "    tokens = tokenizer(text)\n",
    "    complexWord = 0\n",
    "    complex_word_percentage = 0\n",
    "    \n",
    "    for word in tokens:\n",
    "        vowels=0\n",
    "        if word.endswith(('es','ed')):\n",
    "            pass\n",
    "        else:\n",
    "            for w in word:\n",
    "                if(w=='a' or w=='e' or w=='i' or w=='o' or w=='u'):\n",
    "                    vowels += 1\n",
    "            if(vowels > 2):\n",
    "                complexWord += 1\n",
    "    if len(tokens) != 0:\n",
    "        complex_word_percentage = complexWord/len(tokens)\n",
    "    \n",
    "    return complex_word_percentage\n",
    "                        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fog_index(averageSentenceLength, percentageComplexWord):\n",
    "    fogIndex = 0.4 * (averageSentenceLength + percentageComplexWord)\n",
    "    return fogIndex\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def complex_word_count(text):\n",
    "    tokens = tokenizer(text)\n",
    "    complexWord = 0\n",
    "    \n",
    "    for word in tokens:\n",
    "        vowels=0\n",
    "        if word.endswith(('es','ed')):\n",
    "            pass\n",
    "        else:\n",
    "            for w in word:\n",
    "                if(w=='a' or w=='e' or w=='i' or w=='o' or w=='u'):\n",
    "                    vowels += 1\n",
    "            if(vowels > 2):\n",
    "                complexWord += 1\n",
    "    return complexWord"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def total_word_count(text):\n",
    "    tokens = tokenizer(text)\n",
    "    return len(tokens)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(uncertainty_dictionaryFile ,'r') as uncertain_dict:\n",
    "    uncertainDict=uncertain_dict.read().lower()\n",
    "uncertainDictionary = uncertainDict.split('\\n')\n",
    "\n",
    "def uncertainty_score(text):\n",
    "    uncertainWordnum =0\n",
    "    rawToken = tokenizer(text)\n",
    "    for word in rawToken:\n",
    "        if word in uncertainDictionary:\n",
    "            uncertainWordnum +=1\n",
    "    sumUncertainityScore = uncertainWordnum \n",
    "    \n",
    "    return sumUncertainityScore\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(constraining_dictionaryFile ,'r') as constraining_dict:\n",
    "    constrainDict=constraining_dict.read().lower()\n",
    "constrainDictionary = constrainDict.split('\\n')\n",
    "\n",
    "def constraining_score(text):\n",
    "    constrainWordnum =0\n",
    "    rawToken = tokenizer(text)\n",
    "    for word in rawToken:\n",
    "        if word in constrainDictionary:\n",
    "            constrainWordnum +=1\n",
    "    sumConstrainScore = constrainWordnum \n",
    "    \n",
    "    return sumConstrainScore\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def positive_word_prop(positiveScore,wordcount):\n",
    "    positive_word_proportion = 0\n",
    "    if wordcount !=0:\n",
    "        positive_word_proportion = positiveScore / wordcount\n",
    "        \n",
    "    return positive_word_proportion\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def negative_word_prop(negativeScore,wordcount):\n",
    "    negative_word_proportion = 0\n",
    "    if wordcount !=0:\n",
    "        negative_word_proportion = negativeScore / wordcount\n",
    "        \n",
    "    return negative_word_proportion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def uncertain_word_prop(uncertainScore,wordcount):\n",
    "    uncertain_word_proportion = 0\n",
    "    if wordcount !=0:\n",
    "        uncertain_word_proportion = uncertainScore / wordcount\n",
    "        \n",
    "    return uncertain_word_proportion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def constraining_word_prop(constrainingScore,wordcount):\n",
    "    constraining_word_proportion = 0\n",
    "    if wordcount !=0:\n",
    "        constraining_word_proportion = constrainingScore / wordcount\n",
    "        \n",
    "    return constraining_word_proportion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def constrain_word_whole(mdaText,qqdmrText,rfText):\n",
    "    wholeDoc = mdaText + qqdmrText + rfText\n",
    "    constrainWordnumWhole =0\n",
    "    rawToken = tokenizer(wholeDoc)\n",
    "    for word in rawToken:\n",
    "        if word in constrainDictionary:\n",
    "            constrainWordnumWhole +=1\n",
    "    sumConstrainScoreWhole = constrainWordnumWhole \n",
    "    \n",
    "    return sumConstrainScoreWhole"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[WinError 3] The system cannot find the path specified: 'C:\\\\Users\\\\Jinu Augustine\\\\Documents\\\\BlackCoffer\\\\test'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-24-583cc0922680>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0minputDirectory\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34mr'C:\\Users\\Jinu Augustine\\Documents\\BlackCoffer\\test'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mmasterFile\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34mr'C:\\Users\\Jinu Augustine\\Documents\\BlackCoffer\\cik_list.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mdataList\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrawdata_extract\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputDirectory\u001b[0m \u001b[1;33m,\u001b[0m \u001b[0mmasterFile\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataList\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-4-e06c7f8638aa>\u001b[0m in \u001b[0;36mrawdata_extract\u001b[1;34m(path, cikListFile)\u001b[0m\n\u001b[0;32m     14\u001b[0m         \u001b[0mform\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrow\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'FORM'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m         \u001b[0msecfname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mrow\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'SECFNAME'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 16\u001b[1;33m         \u001b[1;32mfor\u001b[0m \u001b[0mfileName\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     17\u001b[0m             \u001b[0mfilenameopen\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfileName\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m             \u001b[0mdirFileName\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfilenameopen\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'\\\\'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 3] The system cannot find the path specified: 'C:\\\\Users\\\\Jinu Augustine\\\\Documents\\\\BlackCoffer\\\\test'"
     ]
    }
   ],
   "source": [
    "inputDirectory = (r'C:\\Users\\Jinu Augustine\\Documents\\BlackCoffer\\test')\n",
    "masterFile = (r'C:\\Users\\Jinu Augustine\\Documents\\BlackCoffer\\cik_list.csv')\n",
    "dataList = rawdata_extract(inputDirectory , masterFile)\n",
    "df = pd.DataFrame(dataList)\n",
    "\n",
    "df['mda_positive_score'] = df.mda_extract.apply(positive_score)\n",
    "df['mda_negative_score'] = df.mda_extract.apply(negative_word)\n",
    "df['mda_polarity_score'] = np.vectorize(polarity_score)(df['mda_positive_score'],df['mda_negative_score'])\n",
    "df['mda_average_sentence_length'] = df.mda_extract.apply(average_sentence_length)\n",
    "df['mda_percentage_of_complex_words'] = df.mda_extract.apply(percentage_complex_word)\n",
    "df['mda_fog_index'] = np.vectorize(fog_index)(df['mda_average_sentence_length'],df['mda_percentage_of_complex_words'])\n",
    "df['mda_complex_word_count']= df.mda_extract.apply(complex_word_count)\n",
    "df['mda_word_count'] = df.mda_extract.apply(total_word_count)\n",
    "df['mda_uncertainty_score']=df.mda_extract.apply(uncertainty_score)\n",
    "df['mda_constraining_score'] = df.mda_extract.apply(constraining_score)\n",
    "df['mda_positive_word_proportion'] = np.vectorize(positive_word_prop)(df['mda_positive_score'],df['mda_word_count'])\n",
    "df['mda_negative_word_proportion'] = np.vectorize(negative_word_prop)(df['mda_negative_score'],df['mda_word_count'])\n",
    "df['mda_uncertainty_word_proportion'] = np.vectorize(uncertain_word_prop)(df['mda_uncertainty_score'],df['mda_word_count'])\n",
    "df['mda_constraining_word_proportion'] = np.vectorize(constraining_word_prop)(df['mda_constraining_score'],df['mda_word_count'])\n",
    "\n",
    "df['qqdmr_positive_score'] = df.qqd_extract.apply(positive_score)\n",
    "df['qqdmr_negative_score'] = df.qqd_extract.apply(negative_word)\n",
    "df['qqdmr_polarity_score'] = np.vectorize(polarity_score)(df['qqdmr_positive_score'],df['qqdmr_negative_score'])\n",
    "df['qqdmr_average_sentence_length'] = df.qqd_extract.apply(average_sentence_length)\n",
    "df['qqdmr_percentage_of_complex_words'] = df.qqd_extract.apply(percentage_complex_word)\n",
    "df['qqdmr_fog_index'] = np.vectorize(fog_index)(df['qqdmr_average_sentence_length'],df['qqdmr_percentage_of_complex_words'])\n",
    "df['qqdmr_complex_word_count']= df.qqd_extract.apply(complex_word_count)\n",
    "df['qqdmr_word_count'] = df.qqd_extract.apply(total_word_count)\n",
    "df['qqdmr_uncertainty_score']=df.qqd_extract.apply(uncertainty_score)\n",
    "df['qqdmr_constraining_score'] = df.qqd_extract.apply(constraining_score)\n",
    "df['qqdmr_positive_word_proportion'] = np.vectorize(positive_word_prop)(df['qqdmr_positive_score'],df['qqdmr_word_count'])\n",
    "df['qqdmr_negative_word_proportion'] = np.vectorize(negative_word_prop)(df['qqdmr_negative_score'],df['qqdmr_word_count'])\n",
    "df['qqdmr_uncertainty_word_proportion'] = np.vectorize(uncertain_word_prop)(df['qqdmr_uncertainty_score'],df['qqdmr_word_count'])\n",
    "df['qqdmr_constraining_word_proportion'] = np.vectorize(constraining_word_prop)(df['qqdmr_constraining_score'],df['qqdmr_word_count'])\n",
    "\n",
    "df['rf_positive_score'] = df.riskfactor_extract.apply(positive_score)\n",
    "df['rf_negative_score'] = df.riskfactor_extract.apply(negative_word)\n",
    "df['rf_polarity_score'] = np.vectorize(polarity_score)(df['rf_positive_score'],df['rf_negative_score'])\n",
    "df['rf_average_sentence_length'] = df.riskfactor_extract.apply(average_sentence_length)\n",
    "df['rf_percentage_of_complex_words'] = df.riskfactor_extract.apply(percentage_complex_word)\n",
    "df['rf_fog_index'] = np.vectorize(fog_index)(df['rf_average_sentence_length'],df['rf_percentage_of_complex_words'])\n",
    "df['rf_complex_word_count']= df.riskfactor_extract.apply(complex_word_count)\n",
    "df['rf_word_count'] = df.riskfactor_extract.apply(total_word_count)\n",
    "df['rf_uncertainty_score']=df.riskfactor_extract.apply(uncertainty_score)\n",
    "df['rf_constraining_score'] = df.riskfactor_extract.apply(constraining_score)\n",
    "df['rf_positive_word_proportion'] = np.vectorize(positive_word_prop)(df['rf_positive_score'],df['rf_word_count'])\n",
    "df['rf_negative_word_proportion'] = np.vectorize(negative_word_prop)(df['rf_negative_score'],df['rf_word_count'])\n",
    "df['rf_uncertainty_word_proportion'] = np.vectorize(uncertain_word_prop)(df['rf_uncertainty_score'],df['rf_word_count'])\n",
    "df['rf_constraining_word_proportion'] = np.vectorize(constraining_word_prop)(df['rf_constraining_score'],df['rf_word_count'])\n",
    "\n",
    "df['constraining_words_whole_report'] = np.vectorize(constrain_word_whole)(df['mda_extract'],df['qqd_extract'],df['riskfactor_extract'])\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputTextCol = ['mda_extract','qqd_extract','riskfactor_extract']\n",
    "finalOutput = df.drop(inputTextCol,1)\n",
    "\n",
    "finalOutput.head(150)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "finalOutput.to_excel(\"output.xlsx\",\n",
    "             sheet_name='Sheet_name_1')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}