data_extraction_llms.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip3 install --upgrade --quiet langchain langchain-community langchain-openai chromadb \n",
    "!pip3 install --upgrade --quiet pypdf pandas streamlit python-dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Langchain modules\n",
    "from langchain.document_loaders import PyPDFLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain_openai import OpenAIEmbeddings, ChatOpenAI\n",
    "from langchain.vectorstores import Chroma\n",
    "from langchain_core.runnables import RunnablePassthrough\n",
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "from langchain_core.pydantic_v1 import BaseModel, Field\n",
    "\n",
    "# Other modules and packages\n",
    "import os\n",
    "import tempfile\n",
    "import streamlit as st  \n",
    "import pandas as pd\n",
    "from dotenv import load_dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_dotenv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "OPENAI_API_KEY = os.environ.get(\"OPENAI_API_KEY\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define our LLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AIMessage(content='Why was the cat sitting on the computer?\\n\\nBecause it wanted to keep an eye on the mouse!', additional_kwargs={'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 20, 'prompt_tokens': 13, 'total_tokens': 33, 'completion_tokens_details': {'reasoning_tokens': 0}}, 'model_name': 'gpt-4o-mini-2024-07-18', 'system_fingerprint': 'fp_1bb46167f9', 'finish_reason': 'stop', 'logprobs': None}, id='run-c3da8491-2b15-4c84-b5e0-5788da81a27f-0', usage_metadata={'input_tokens': 13, 'output_tokens': 20, 'total_tokens': 33})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm = ChatOpenAI(model=\"gpt-4o-mini\", api_key=OPENAI_API_KEY)\n",
    "llm.invoke(\"Tell me a joke about cats\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process PDF document"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load PDF document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Ignoring wrong pointing object 18 0 (offset 0)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[Document(metadata={'source': 'data/Oppenheimer-2006-Applied_Cognitive_Psychology.pdf', 'page': 0}, page_content='APPLIED COGNITIVE PSYCHOLOGYAppl. Cognit. Psychol.20: 139–156 (2006)Published online 31 October 2005 in Wiley InterScience(www.interscience.wiley.com) DOI: 10.1002/acp.1178Consequences of Erudite Vernacular Utilized Irrespectiveof Necessity: Problems with Using Long Words NeedlesslyDANIEL M. OPPENHEIMER*Princeton University, USASUMMARYMost texts on writing style encourage authors to avoid overly-complex words. However, a majorityof undergraduates admit to deliberately increasing the complexity of their vocabulary so as to givethe impression of intelligence. This paper explores the extent to which this strategy is effective.Experiments 1–3 manipulate complexity of texts and ﬁnd a negative relationship between complex-ity and judged intelligence. This relationship held regardless of the quality of the original essay, andirrespective of the participants’ prior expectations of essay quality. The negative impact ofcomplexity was mediated by processing ﬂuency. Experiment 4 directly manipulated ﬂuency andfound that texts in hard to read fonts are judged to come from less intelligent authors. Experiment 5investigated discounting of ﬂuency. When obvious causes for low ﬂuency exist that are not relevantto the judgement at hand, people reduce their reliance on ﬂuency as a cue; in fact, in an effort not tobe inﬂuenced by the irrelevant source of ﬂuency, they over-compensate and are biased in the oppositedirection. Implications and applications are discussed. Copyright#2005 John Wiley & Sons, Ltd.When it comes to writing, most experts agree that clarity, simplicity and parsimony areideals that authors should strive for. In their classic manual of style, Strunk and White(1979) encourage authors to ‘omit needless words.’ Daryl Bem’s (1995) guidelines forsubmission toPsychological Bulletinadvise, ‘the ﬁrst step towards clarity is writingsimply.’ Even the APA publication manual (1996) recommends, ‘direct, declarativesentences with simple common words are usually best.’However, most of us can likely recall having read papers, either by colleagues orstudents, in which the author appears to be deliberately using overly complex words.Experience suggests that the experts’ advice contrasts with prevailing wisdom on how tosound more intelligent as a writer. In fact, when 110 Stanford undergraduates were polledabout their writing habits, most of them admitted that they had made their writing morecomplex in order to appear smarter. For example, when asked, ‘Have you ever changed thewords in an academic essay to make the essay sound more valid or intelligent by usingcomplicated language?’ 86.4% of the sample admitted to having done so. Nearly two-thirds answered yes to the question, ‘When you write an essay, do you turn to the thesaurusto choose words that are more complex to give the impression that the content is morevalid or intelligent?’Copyright#2005 John Wiley & Sons, Ltd.*Correspondence to: D. M. Oppenheimer, Department of Psychology, Princeton University, Green Hall Room2-S-8, Princeton, NJ 08540, USA. E-mail: doppenhe@princeton.edu'),\n",
       " Document(metadata={'source': 'data/Oppenheimer-2006-Applied_Cognitive_Psychology.pdf', 'page': 1}, page_content='There are many plausible reasons that the use of million-dollar words would leadreaders to believe that an author is smart. Intelligence and large vocabularies are positivelycorrelated (Spearman, 1904). Therefore, by displaying a large vocabulary, one may beproviding cues that he or she is intelligent as well. Secondly, writers are assumed to beconforming to the Gricean maxim of manner, ‘avoid obscurity of expression’ (Grice,1975). If authors are believed to be writing as simply as possible, but a text is nonethelesscomplex, a reader might believe that the ideas expressed in that text are also complex,defying all attempts to simplify the language. Further, individuals forced to strugglethrough a complex text might experience dissonance if they believe that the ideas beingconveyed are simple (Festinger, 1957). Thus, individuals might be motivated to perceive adifﬁcult text as being more worthwhile, thereby justifying the effort of processing.Indeed, there is some evidence that complex vocabulary can be indicative of a moreintelligent author. For example, Pennebaker and King (1999) have shown that thepercentage of long words used in class assignments positively correlates with SATscores and exam grades on both multiple choice and essay tests. However it is difﬁcultto draw conclusions about the effectivenessof a strategy of complexity from this data.The study did not look at how readers of the texts containing the long words perceivedthe authors’ intelligence. Thus, it is possible that although students using complexvocabularies are objectively very knowledgeable, they might nonetheless be perceivedas being less so.Why might we believe that the experts might be correct in recommending simplicity inwriting? One theory that predicts the effectiveness of straightforward writing is that ofprocessing ﬂuency. Simpler writing is easier to process, and studies have demonstratedthat processing ﬂuency is associated with a variety of positive dimensions. Fluency leadsto higher judgements of truth (Reber & Schwarz, 1999), conﬁdence (Norwick & Epley,2002), frequency (Tversky & Kahneman, 1973), fame (Jacoby, Kelley, Brown, &Jasechko, 1989), and even liking (Reber, Winkielman, & Schwarz, 1998). Furthermore,the effects of ﬂuency are strongest when the ﬂuency is discrepant—when the amount ofexperienced ﬂuency is surprising (Whittlesea & Williams, 2001a, 2001b). As such, itwould not be surprising if the lower ﬂuency of overly complex texts caused readers to havenegative evaluations of those texts and the associated authors, especially if the complexitywas unnecessary and thus surprising readers with the relative disﬂuency of the text.Both the experts and prevailing wisdom present plausible views, but which (if either) iscorrect? The present paper provides an empirical investigation of the strategy of complex-ity, and ﬁnds such a strategy to be unsuccessful. Five studies demonstrate that the loss ofﬂuency due to needless complexity in a text negatively impacts raters’ assessments of thetext’s authors.EXPERIMENT 1Experiment 1 aimed to answer several simple questions. First, does increasing thecomplexity of text succeed in making the author appear more intelligent? Second, towhat extent does the success of this strategy depend on the quality of the original, simplerwriting? Finally, if the strategy is unsuccessful, is the failure of the strategy due to loss ofﬂuency? To answer these questions, graduate school admission essays were made morecomplex by substituting some of the original words with their longest applicable thesaurusentries.140D. M. Oppenheimer\\nCopyright#2005 John Wiley & Sons, Ltd. Appl. Cognit. Psychol. 20: 139–156 (2006)'),\n",
       " Document(metadata={'source': 'data/Oppenheimer-2006-Applied_Cognitive_Psychology.pdf', 'page': 2}, page_content='While word length is not perfectly interchangeable with sentence complexity—forexample, complexity can come from grammatical structure or infrequent words aswell—it is a useful proxy. Using length as a manipulation of complexity allows for asimple, easily replicable word replacement algorithm. By keeping content constant andvarying the complexity of vocabulary, it was possible to investigate the effectiveness ofcomplexity.Participants and procedureSeventy-one Stanford University undergraduates participated to fulﬁl part of a courserequirement. The survey was included in a packet of unrelated one-page questionnaires.Packets were distributed in class, and participants were given a week to complete the entirepacket.Stimuli and designSix personal statements for admissions to graduate studies in English Literature weredownloaded from writing improvement websites. The essays varied greatly both in contentand quality of writing. Logical excerpts ranging from 138 to 253 words in length were thentaken from each essay. A ‘highly complex’ version of each excerpt was prepared byreplacing every noun, verb and adjective with its longest entry in the Microsoft Word 2000thesaurus. Words that were longer than any thesaurus entry, were not listed in thethesaurus, or for which there was no entry with the same linguistic sense were notreplaced. If two entries were of the same length, the replacement was chosen alphabe-tically. When necessary, minor modiﬁcations were made to the essay to maintain thegrammatical structure of a sentence (e.g. replacing ‘an’ with ‘a’ for replacement wordsbeginning with consonants). A ‘moderately complex’ version of each excerpt was createdusing the same algorithm as above, except replacing only every third applicable word.Examples of the stimuli can be found in the appendix.Each participant received only one excerpt. Participants were informed that the excerptcame from a personal statement for graduate study in the Stanford English department.They were instructed to read the passage, decide whether or not to accept the applicant,and rate their conﬁdence in their decision on a 7-point scale.1They were then asked howdifﬁcult the passage was to understand, also on a seven-point scale.ResultsThe data of one participant was discarded due to an illegible answer. Analysis of themanipulation check showed that more complex texts were more difﬁcult to read. (x¼2.9,4.0 and 4.3 for simple, moderately complex and highly complex, respectively). Thesedifferences were reliable,F(2, 68)¼4.46,p<0.05, Cohen’sf¼0.18. For other analyses,acceptance ratings (þ1 for accept,/C01 for reject) were multiplied by conﬁdence ratings tocreate a/C07 to 7 scale of admission conﬁdence. Level of complexity had a reliableinﬂuence on admission conﬁdence ratings,F(2, 70)¼2.46,p<0.05, Cohen’sf¼0.12.1With the exception of the dichotomous admissions decision, all dependent measures reported in this paper areseven point scales ranging from 1¼‘not at all’ to 7¼‘very’.Problems with long words141\\nCopyright#2005 John Wiley & Sons, Ltd. Appl. Cognit. Psychol. 20: 139–156 (2006)')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loader = PyPDFLoader(\"data/Oppenheimer-2006-Applied_Cognitive_Psychology.pdf\")\n",
    "pages = loader.load()\n",
    "pages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1500,\n",
    "                                            chunk_overlap=200,\n",
    "                                            length_function=len,\n",
    "                                            separators=[\"\\n\\n\", \"\\n\", \" \"])\n",
    "chunks = text_splitter.split_documents(pages)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_embedding_function():\n",
    "    embeddings = OpenAIEmbeddings(\n",
    "        model=\"text-embedding-ada-002\", openai_api_key=OPENAI_API_KEY\n",
    "    )\n",
    "    return embeddings\n",
    "\n",
    "embedding_function = get_embedding_function()\n",
    "test_vector = embedding_function.embed_query(\"cat\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.17440875566198188}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.evaluation import load_evaluator\n",
    "\n",
    "evaluator = load_evaluator(evaluator=\"embedding_distance\", \n",
    "                            embeddings=embedding_function)\n",
    "\n",
    "evaluator.evaluate_strings(prediction=\"Amsterdam\", reference=\"coffeeshop\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score': 0.22417909850229667}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.evaluate_strings(prediction=\"Paris\", reference=\"coffeeshop\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create vector database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import uuid\n",
    "\n",
    "def create_vectorstore(chunks, embedding_function, vectorstore_path):\n",
    "\n",
    "    # Create a list of unique ids for each document based on the content\n",
    "    ids = [str(uuid.uuid5(uuid.NAMESPACE_DNS, doc.page_content)) for doc in chunks]\n",
    "    \n",
    "    # Ensure that only unique docs with unique ids are kept\n",
    "    unique_ids = set()\n",
    "    unique_chunks = []\n",
    "    \n",
    "    unique_chunks = [] \n",
    "    for chunk, id in zip(chunks, ids):     \n",
    "        if id not in unique_ids:       \n",
    "            unique_ids.add(id)\n",
    "            unique_chunks.append(chunk) \n",
    "\n",
    "    # Create a new Chroma database from the documents\n",
    "    vectorstore = Chroma.from_documents(documents=unique_chunks, \n",
    "                                        ids=list(unique_ids),\n",
    "                                        embedding=embedding_function, \n",
    "                                        persist_directory = vectorstore_path)\n",
    "\n",
    "    vectorstore.persist()\n",
    "    \n",
    "    return vectorstore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create vectorstore\n",
    "vectorstore = create_vectorstore(chunks=chunks, \n",
    "                                 embedding_function=embedding_function, \n",
    "                                 vectorstore_path=\"vectorstore_test\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Query for relevant data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load vectorstore\n",
    "vectorstore = Chroma(persist_directory=\"vectorstore_chroma\", embedding_function=embedding_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(metadata={'page': 1, 'source': 'data/1995-watanabe.pdf'}, page_content='Am. J. Physiol. 250, G85~91. \\nWatanabe S. and Dawes C. (1988) The effects of different \\nfoods and concentrations of citric acid on the flow rate of \\nwhole saliva in man. Archs oral Biol. 33, 1-5. \\nWatanabe S. and Dawes C. (1990) Salivary flow rates and \\nsalivary film thickness in five-year-old children. J. dent. \\nRes. 69, 1150-1153. \\nWatanabe S. (1992) Salivary clearance from different re- \\ngions of the mouth in children. Caries Res. 26, 423-427.'),\n",
       " Document(metadata={'page': 1, 'source': 'data/1995-watanabe.pdf'}, page_content='grants-in-aid from the Ministry of Education, Science, and \\nCulture of Japan, Grants 02807189, 05671719 (to S.W.), and \\nby a Grant from the Research Foundation of the Health \\nSciences University of Hokkaido (to S.W.). \\nREFERENCES \\nBecks H. and Wainwright W. W. (1943) Human saliva XIII. \\nRate of flow of resting saliva of healthy individuals. \\nJ. dent. Res. 22, 391-396. \\nDawes C. (1987) Physiological factors affecting salivary flow \\nrate, oral sugar clearance, and the sensation of dry mouth \\nin man. J. dent. Res. 66, 648~53. \\nDawes C. and Watanabe S. (1987) The effect of taste \\nadaptation on salivary flow rate and salivary sugar clear- \\nance. J. dent. Res. 66, 740-744. \\nDuncan K. H., Bacon J. A. and Weinsier R. L. (1983) The \\neffects of high and low energy density diets on satiety, \\nenergy intake, and eating time of obese and nonobese \\nsubjects. Am. J. clin. Nutr. 37, 763-767. \\nHeintze U., Birkhed D. and Bj6rn H. (1983) Secretion rate \\nand buffer effect of resting and stimulated whole saliva as \\na function of age and sex. Swed. dent. J. 7, 227-238. \\nLichter I. and Muir R. C. (1975) The pattern of swallowing \\nduring sleep. Electroenceph. clin. Neurophysiol. 38, \\n427-432. \\nRichardson C. T. and Feldman M. (1986) Salivary response \\nto food in humans and its effect on gastric acid secretion. \\nAm. J. Physiol. 250, G85~91. \\nWatanabe S. and Dawes C. (1988) The effects of different \\nfoods and concentrations of citric acid on the flow rate of'),\n",
       " Document(metadata={'page': 0, 'source': 'data/1995-watanabe.pdf'}, page_content='foods was randomized for all participants. The dry \\nweights of the food-saliva mixture for rice, apple and \\ncookie were subsequently obtained by freeze-drying \\n(Freeze dryer VD-15, Taitec, Japan). The percentage \\ndry weights of the same batches of those foods \\n(unchewed) were also obtained, which allowed calcu- \\nlation of the percentage of food either inadvertently \\nswallowed or retained in the mouth and expectorated, \\nby the formula described by Watanabe and Dawes \\n(1988). \\nThere were no significant differences due to gender \\nin salivary flow rate and times spent awake and eating \\nand thus the results for boys and girls were pooled. \\nTable 1 shows the mean times spent eating (breakfast, \\nlunch, dinner and snacks), awake but not eating, and \\nsleeping. There is surprisingly little information about \\nthe length of time spent eating each day, particularly \\nwhen food is self-selected. From a study of 10 obese \\nand 10 non-obese persons consuming two set diets, \\nDuncan, Bacon and Weinsier (1983) reported that \\ntotal eating time per day was independent of the \\n781'),\n",
       " Document(metadata={'page': 0, 'source': 'data/1995-watanabe.pdf'}, page_content='At the beginning of the main study, unstimulated \\nwhole saliva was collected. The children were seated, \\nwith head down, anti saliva was collected for 5 min by \\nbeing allowed to drip off the lower lip into a weighed \\ncontainer. They swallowed immediately before the \\ncollection, and at the end forcibly spat out any saliva \\nremaining in the mauth into the container, and the \\nincrease in weight was determined (Dawes, 1987). \\nThe children were watched closely so that swallowing \\ndid not occur during the 5-min collection period. The \\nsix representative foods were steamed rice, sausage, mashed potato, cookie, apple and pickled radish. \\nDuring the main study, the amount of a given food \\nwas weighed to the nearest 10 mg on a top-loading \\nbalance (FX3200, A & D, Japan). The participants \\ntook their usual bite-size portions and chewed until \\nsuch time as they would normally have swallowed. \\nHowever, instead of swallowing, they spat out the \\nfood bolus into a previously weighed container. The \\ntime required for all the food to be masticated and \\nspat out was recorded. The volume of saliva secreted \\nin that time was determined by subtracting the initial \\nweight of the food from that of the food-saliva \\nmixture. \\nA rest period of 4 or 5 min was allowed between \\neach successive food. The order of presentation of the \\nfoods was randomized for all participants. The dry \\nweights of the food-saliva mixture for rice, apple and \\ncookie were subsequently obtained by freeze-drying')]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create retriever and get relevant chunks\n",
    "retriever = vectorstore.as_retriever(search_type=\"similarity\")\n",
    "relevant_chunks = retriever.invoke(\"What is the title of the paper?\")\n",
    "relevant_chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prompt template\n",
    "PROMPT_TEMPLATE = \"\"\"\n",
    "You are an assistant for question-answering tasks.\n",
    "Use the following pieces of retrieved context to answer\n",
    "the question. If you don't know the answer, say that you\n",
    "don't know. DON'T MAKE UP ANYTHING.\n",
    "\n",
    "{context}\n",
    "\n",
    "---\n",
    "\n",
    "Answer the question based on the above context: {question}\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Generate responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Human: \n",
      "You are an assistant for question-answering tasks.\n",
      "Use the following pieces of retrieved context to answer\n",
      "the question. If you don't know the answer, say that you\n",
      "don't know. DON'T MAKE UP ANYTHING.\n",
      "\n",
      "Copyright#2005 John Wiley & Sons, Ltd. Appl. Cognit. Psychol. 20: 139–156 (2006)\n",
      "\n",
      "---\n",
      "\n",
      "language?’ 86.4% of the sample admitted to having done so. Nearly two-thirds answered yes to the question, ‘When you write an essay, do you turn to the thesaurusto choose words that are more complex to give the impression that the content is morevalid or intelligent?’Copyright#2005 John Wiley & Sons, Ltd.*Correspondence to: D. M. Oppenheimer, Department of Psychology, Princeton University, Green Hall Room2-S-8, Princeton, NJ 08540, USA. E-mail: doppenhe@princeton.edu\n",
      "\n",
      "---\n",
      "\n",
      "APPLIED COGNITIVE PSYCHOLOGYAppl. Cognit. Psychol.20: 139–156 (2006)Published online 31 October 2005 in Wiley InterScience(www.interscience.wiley.com) DOI: 10.1002/acp.1178Consequences of Erudite Vernacular Utilized Irrespectiveof Necessity: Problems with Using Long Words NeedlesslyDANIEL M. OPPENHEIMER*Princeton University, USASUMMARYMost texts on writing style encourage authors to avoid overly-complex words. However, a majorityof undergraduates admit to deliberately increasing the complexity of their vocabulary so as to givethe impression of intelligence. This paper explores the extent to which this strategy is effective.Experiments 1–3 manipulate complexity of texts and ﬁnd a negative relationship between complex-ity and judged intelligence. This relationship held regardless of the quality of the original essay, andirrespective of the participants’ prior expectations of essay quality. The negative impact ofcomplexity was mediated by processing ﬂuency. Experiment 4 directly manipulated ﬂuency andfound that texts in hard to read fonts are judged to come from less intelligent authors. Experiment 5investigated discounting of ﬂuency. When obvious causes for low ﬂuency exist that are not relevantto the judgement at hand, people reduce their reliance on ﬂuency as a cue; in fact, in an effort not tobe inﬂuenced by the irrelevant source of ﬂuency, they over-compensate and are biased in the oppositedirection. Implications and applications are discussed. Copyright#2005 John Wiley &\n",
      "\n",
      "---\n",
      "\n",
      "surprising readers with the relative disﬂuency of the text.Both the experts and prevailing wisdom present plausible views, but which (if either) iscorrect? The present paper provides an empirical investigation of the strategy of complex-ity, and ﬁnds such a strategy to be unsuccessful. Five studies demonstrate that the loss ofﬂuency due to needless complexity in a text negatively impacts raters’ assessments of thetext’s authors.EXPERIMENT 1Experiment 1 aimed to answer several simple questions. First, does increasing thecomplexity of text succeed in making the author appear more intelligent? Second, towhat extent does the success of this strategy depend on the quality of the original, simplerwriting? Finally, if the strategy is unsuccessful, is the failure of the strategy due to loss ofﬂuency? To answer these questions, graduate school admission essays were made morecomplex by substituting some of the original words with their longest applicable thesaurusentries.140D. M. Oppenheimer\n",
      "\n",
      "---\n",
      "\n",
      "Answer the question based on the above context: What is the title of the paper?\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Concatenate context text\n",
    "context_text = \"\\n\\n---\\n\\n\".join([doc.page_content for doc in relevant_chunks])\n",
    "\n",
    "# Create prompt\n",
    "prompt_template = ChatPromptTemplate.from_template(PROMPT_TEMPLATE)\n",
    "prompt = prompt_template.format(context=context_text, \n",
    "                                question=\"What is the title of the paper?\")\n",
    "print(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AIMessage(content='The title of the paper is \"Consequences of Erudite Vernacular Utilized Irrespective of Necessity: Problems with Using Long Words Needlessly.\"', additional_kwargs={'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 32, 'prompt_tokens': 773, 'total_tokens': 805}, 'model_name': 'gpt-4o-mini-2024-07-18', 'system_fingerprint': 'fp_483d39d857', 'finish_reason': 'stop', 'logprobs': None}, id='run-e2017e06-5717-4a90-a7dc-47634ed2728f-0', usage_metadata={'input_tokens': 773, 'output_tokens': 32, 'total_tokens': 805})"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm.invoke(prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using Langchain Expression Language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AIMessage(content='The title of the paper is \"Consequences of Erudite Vernacular Utilized Irrespective of Necessity: Problems with Using Long Words Needlessly.\"', additional_kwargs={'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 32, 'prompt_tokens': 767, 'total_tokens': 799}, 'model_name': 'gpt-4o-mini-2024-07-18', 'system_fingerprint': 'fp_483d39d857', 'finish_reason': 'stop', 'logprobs': None}, id='run-b478d845-c13f-4ee4-868d-a199f8534a9f-0', usage_metadata={'input_tokens': 767, 'output_tokens': 32, 'total_tokens': 799})"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def format_docs(docs):\n",
    "    return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
    "\n",
    "rag_chain = (\n",
    "            {\"context\": retriever | format_docs, \"question\": RunnablePassthrough()}\n",
    "            | prompt_template\n",
    "            | llm\n",
    "        )\n",
    "rag_chain.invoke(\"What's the title of this paper?\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate structured responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AnswerWithSources(BaseModel):\n",
    "    \"\"\"An answer to the question, with sources and reasoning.\"\"\"\n",
    "    answer: str = Field(description=\"Answer to question\")\n",
    "    sources: str = Field(description=\"Full direct text chunk from the context used to answer the question\")\n",
    "    reasoning: str = Field(description=\"Explain the reasoning of the answer based on the sources\")\n",
    "    \n",
    "class ExtractedInfo(BaseModel):\n",
    "    \"\"\"Extracted information about the research article\"\"\"\n",
    "    paper_title: AnswerWithSources\n",
    "    paper_summary: AnswerWithSources\n",
    "    publication_year: AnswerWithSources\n",
    "    paper_authors: AnswerWithSources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtractedInfo(paper_title=AnswerWithSources(answer='Consequences of Erudite Vernacular Utilized Irrespective of Necessity: Problems with Using Long Words Needlessly', sources='Consequences of Erudite Vernacular Utilized Irrespective of Necessity: Problems with Using Long Words Needlessly', reasoning='The title is explicitly mentioned in the context provided.'), paper_summary=AnswerWithSources(answer='Most texts on writing style encourage authors to avoid overly-complex words. However, a majority of undergraduates admit to deliberately increasing the complexity of their vocabulary so as to give the impression of intelligence. This paper explores the extent to which this strategy is effective. Experiments 1–3 manipulate complexity of texts and find a negative relationship between complexity and judged intelligence.', sources='Most texts on writing style encourage authors to avoid overly-complex words. However, a majority of undergraduates admit to deliberately increasing the complexity of their vocabulary so as to give the impression of intelligence. This paper explores the extent to which this strategy is effective. Experiments 1–3 manipulate complexity of texts and find a negative relationship between complexity and judged intelligence.', reasoning='The summary captures the main findings and focus of the research paper as described in the provided text.'), publication_year=AnswerWithSources(answer='2006', sources='APPLIED COGNITIVE PSYCHOLOGY Appl. Cognit. Psychol. 20: 139–156 (2006)', reasoning='The publication year is indicated in the citation at the beginning of the context.'), paper_authors=AnswerWithSources(answer='Daniel M. Oppenheimer', sources='DANIEL M. OPPENHEIMER*Princeton University, USA', reasoning=\"The author's name is mentioned along with their affiliation in the provided context.\"))"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rag_chain = (\n",
    "            {\"context\": retriever | format_docs, \"question\": RunnablePassthrough()}\n",
    "            | prompt_template\n",
    "            | llm.with_structured_output(ExtractedInfo, strict=True)\n",
    "        )\n",
    "\n",
    "rag_chain.invoke(\"Give me the title, summary, publication date, authors of the research paper.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transform response into a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paper_title</th>\n",
       "      <th>paper_summary</th>\n",
       "      <th>publication_year</th>\n",
       "      <th>paper_authors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>answer</th>\n",
       "      <td>Consequences of Erudite Vernacular Utilized Ir...</td>\n",
       "      <td>The paper explores the negative relationship b...</td>\n",
       "      <td>2006</td>\n",
       "      <td>Daniel M. Oppenheimer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source</th>\n",
       "      <td>Copyright#2005 John Wiley &amp; Sons, Ltd. Appl. C...</td>\n",
       "      <td>Most texts on writing style encourage authors ...</td>\n",
       "      <td>Appl. Cognit. Psychol. 20: 139–156 (2006)</td>\n",
       "      <td>Correspondence to: D. M. Oppenheimer, Departme...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reasoning</th>\n",
       "      <td>The title is explicitly mentioned at the begin...</td>\n",
       "      <td>The summary is derived from the overall conten...</td>\n",
       "      <td>The publication year is indicated in the citat...</td>\n",
       "      <td>The author’s name is provided in the correspon...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 paper_title  \\\n",
       "answer     Consequences of Erudite Vernacular Utilized Ir...   \n",
       "source     Copyright#2005 John Wiley & Sons, Ltd. Appl. C...   \n",
       "reasoning  The title is explicitly mentioned at the begin...   \n",
       "\n",
       "                                               paper_summary  \\\n",
       "answer     The paper explores the negative relationship b...   \n",
       "source     Most texts on writing style encourage authors ...   \n",
       "reasoning  The summary is derived from the overall conten...   \n",
       "\n",
       "                                            publication_year  \\\n",
       "answer                                                  2006   \n",
       "source             Appl. Cognit. Psychol. 20: 139–156 (2006)   \n",
       "reasoning  The publication year is indicated in the citat...   \n",
       "\n",
       "                                               paper_authors  \n",
       "answer                                 Daniel M. Oppenheimer  \n",
       "source     Correspondence to: D. M. Oppenheimer, Departme...  \n",
       "reasoning  The author’s name is provided in the correspon...  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "structured_response = rag_chain.invoke(\"Give me the title, summary, publication date, authors of the research paper.\")\n",
    "df = pd.DataFrame([structured_response.dict()])\n",
    "\n",
    "# Transforming into a table with two rows: 'answer' and 'source'\n",
    "answer_row = []\n",
    "source_row = []\n",
    "reasoning_row = []\n",
    "\n",
    "for col in df.columns:\n",
    "    answer_row.append(df[col][0]['answer'])\n",
    "    source_row.append(df[col][0]['sources'])\n",
    "    reasoning_row.append(df[col][0]['reasoning'])\n",
    "\n",
    "# Create new dataframe with two rows: 'answer' and 'source'\n",
    "structured_response_df = pd.DataFrame([answer_row, source_row, reasoning_row], columns=df.columns, index=['answer', 'source', 'reasoning'])\n",
    "structured_response_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}