cleaning_enem_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ENEM - Data Cleaning and Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import libraries\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import Dataset\n",
    "dataEnem = pd.read_csv (\"microdados_enem_2019/DADOS/MICRODADOS_ENEM_2019.csv\", sep=';', encoding='ISO-8859-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['NU_INSCRICAO', 'NU_ANO', 'CO_MUNICIPIO_RESIDENCIA',\n",
       "       'NO_MUNICIPIO_RESIDENCIA', 'CO_UF_RESIDENCIA', 'SG_UF_RESIDENCIA',\n",
       "       'NU_IDADE', 'TP_SEXO', 'TP_ESTADO_CIVIL', 'TP_COR_RACA',\n",
       "       'TP_NACIONALIDADE', 'CO_MUNICIPIO_NASCIMENTO',\n",
       "       'NO_MUNICIPIO_NASCIMENTO', 'CO_UF_NASCIMENTO', 'SG_UF_NASCIMENTO',\n",
       "       'TP_ST_CONCLUSAO', 'TP_ANO_CONCLUIU', 'TP_ESCOLA', 'TP_ENSINO',\n",
       "       'IN_TREINEIRO', 'CO_ESCOLA', 'CO_MUNICIPIO_ESC',\n",
       "       'NO_MUNICIPIO_ESC', 'CO_UF_ESC', 'SG_UF_ESC',\n",
       "       'TP_DEPENDENCIA_ADM_ESC', 'TP_LOCALIZACAO_ESC', 'TP_SIT_FUNC_ESC',\n",
       "       'IN_BAIXA_VISAO', 'IN_CEGUEIRA', 'IN_SURDEZ',\n",
       "       'IN_DEFICIENCIA_AUDITIVA', 'IN_SURDO_CEGUEIRA',\n",
       "       'IN_DEFICIENCIA_FISICA', 'IN_DEFICIENCIA_MENTAL',\n",
       "       'IN_DEFICIT_ATENCAO', 'IN_DISLEXIA', 'IN_DISCALCULIA',\n",
       "       'IN_AUTISMO', 'IN_VISAO_MONOCULAR', 'IN_OUTRA_DEF', 'IN_GESTANTE',\n",
       "       'IN_LACTANTE', 'IN_IDOSO', 'IN_ESTUDA_CLASSE_HOSPITALAR',\n",
       "       'IN_SEM_RECURSO', 'IN_BRAILLE', 'IN_AMPLIADA_24', 'IN_AMPLIADA_18',\n",
       "       'IN_LEDOR', 'IN_ACESSO', 'IN_TRANSCRICAO', 'IN_LIBRAS',\n",
       "       'IN_TEMPO_ADICIONAL', 'IN_LEITURA_LABIAL', 'IN_MESA_CADEIRA_RODAS',\n",
       "       'IN_MESA_CADEIRA_SEPARADA', 'IN_APOIO_PERNA', 'IN_GUIA_INTERPRETE',\n",
       "       'IN_COMPUTADOR', 'IN_CADEIRA_ESPECIAL', 'IN_CADEIRA_CANHOTO',\n",
       "       'IN_CADEIRA_ACOLCHOADA', 'IN_PROVA_DEITADO', 'IN_MOBILIARIO_OBESO',\n",
       "       'IN_LAMINA_OVERLAY', 'IN_PROTETOR_AURICULAR', 'IN_MEDIDOR_GLICOSE',\n",
       "       'IN_MAQUINA_BRAILE', 'IN_SOROBAN', 'IN_MARCA_PASSO', 'IN_SONDA',\n",
       "       'IN_MEDICAMENTOS', 'IN_SALA_INDIVIDUAL', 'IN_SALA_ESPECIAL',\n",
       "       'IN_SALA_ACOMPANHANTE', 'IN_MOBILIARIO_ESPECIFICO',\n",
       "       'IN_MATERIAL_ESPECIFICO', 'IN_NOME_SOCIAL', 'CO_MUNICIPIO_PROVA',\n",
       "       'NO_MUNICIPIO_PROVA', 'CO_UF_PROVA', 'SG_UF_PROVA',\n",
       "       'TP_PRESENCA_CN', 'TP_PRESENCA_CH', 'TP_PRESENCA_LC',\n",
       "       'TP_PRESENCA_MT', 'CO_PROVA_CN', 'CO_PROVA_CH', 'CO_PROVA_LC',\n",
       "       'CO_PROVA_MT', 'NU_NOTA_CN', 'NU_NOTA_CH', 'NU_NOTA_LC',\n",
       "       'NU_NOTA_MT', 'TX_RESPOSTAS_CN', 'TX_RESPOSTAS_CH',\n",
       "       'TX_RESPOSTAS_LC', 'TX_RESPOSTAS_MT', 'TP_LINGUA',\n",
       "       'TX_GABARITO_CN', 'TX_GABARITO_CH', 'TX_GABARITO_LC',\n",
       "       'TX_GABARITO_MT', 'TP_STATUS_REDACAO', 'NU_NOTA_COMP1',\n",
       "       'NU_NOTA_COMP2', 'NU_NOTA_COMP3', 'NU_NOTA_COMP4', 'NU_NOTA_COMP5',\n",
       "       'NU_NOTA_REDACAO', 'Q001', 'Q002', 'Q003', 'Q004', 'Q005', 'Q006',\n",
       "       'Q007', 'Q008', 'Q009', 'Q010', 'Q011', 'Q012', 'Q013', 'Q014',\n",
       "       'Q015', 'Q016', 'Q017', 'Q018', 'Q019', 'Q020', 'Q021', 'Q022',\n",
       "       'Q023', 'Q024', 'Q025'], dtype=object)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataEnem.columns.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The contextual biases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Select unwanted columns\n",
    "selectedColumns = ['NU_INSCRICAO', 'TP_NACIONALIDADE', 'TP_ST_CONCLUSAO', 'TP_ESCOLA',\n",
    "       'TP_DEPENDENCIA_ADM_ESC', 'TP_SIT_FUNC_ESC',\n",
    "       'TP_PRESENCA_CN', 'TP_PRESENCA_CH', 'TP_PRESENCA_LC',\n",
    "       'TP_PRESENCA_MT', 'NU_NOTA_CN', 'NU_NOTA_CH', 'NU_NOTA_LC',\n",
    "       'NU_NOTA_MT', 'NU_NOTA_REDACAO', 'Q001', 'Q002', 'Q003', 'Q004', 'Q005', 'Q006',\n",
    "       'Q007', 'Q008', 'Q009', 'Q010', 'Q011', 'Q012', 'Q013', 'Q014',\n",
    "       'Q015', 'Q016', 'Q017', 'Q018', 'Q019', 'Q020', 'Q021', 'Q022',\n",
    "       'Q023', 'Q024', 'Q025']\n",
    "\n",
    "#Filter unwanted columns\n",
    "filteredDataEnem = dataEnem.filter(items=selectedColumns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Filter brazilian only\n",
    "brDataEnem = filteredDataEnem[filteredDataEnem['TP_NACIONALIDADE'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Filter only people who is in the last year of high school\n",
    "brDataEnem = brDataEnem[brDataEnem['TP_ST_CONCLUSAO'] == 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Filter only people from schools that are open\n",
    "brDataEnem = brDataEnem[brDataEnem['TP_SIT_FUNC_ESC'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Filter only people that did not miss any test\n",
    "brDataEnem = brDataEnem[(brDataEnem['TP_PRESENCA_CN'] == 1) & (brDataEnem['TP_PRESENCA_LC'] == 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "brDataEnem.to_csv('brDataEnem.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Doubts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Since we have a well-defined context and purpose, it make sense to define the most important features to predict our result, but does it make more sense to use automated methods like for example Stepwise Selection or LASSO regression?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. Since our learning model was not fully successful, should we have used another classification model, or do you think the problem is associated with the size of the training dataset? Another possibility that came to mind is related to the large number of features (Apr 25) and the number of possible classifications, success and failure seems very binary."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. For the next and final phase we want to predict students scores based on their socioeconomic data, that put us on a regression kind of problem. How do you think it would be better for us to work? Should we focous on the Gradient Descent model?"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}