Rodrigo_campanha_natal_2021.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a1b3e5e2",
   "metadata": {},
   "source": [
    "Importando as bibliotecas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f9017f6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import chardet\n",
    "import unidecode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "620c8a29",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Identificar o encoding do arquivo\n",
    "\n",
    "with open('natal2021.csv', 'rb') as f:\n",
    "    result = chardet.detect(f.read())\n",
    "encoding = result['encoding']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "af1c9977",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. Processar o arquivo de 1000 em 1000 linhas\n",
    "\n",
    "chunk_size = 1000\n",
    "chunks = pd.read_csv('natal2021.csv', chunksize=chunk_size, encoding=encoding)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "eadd77c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. Remover os espaços das colunas\n",
    "\n",
    "def remove_spaces(df):\n",
    "    for col in df.columns:\n",
    "        if df[col].dtype == 'object':\n",
    "            df[col] = df[col].str.strip()\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8f096c70",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4. Criar uma coluna CITY_ASCII no arquivo\n",
    "\n",
    "def create_ascii_col(df):\n",
    "    df['CITY_ASCII'] = df['CITY'].apply(lambda x: unidecode.unidecode(x).upper())\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "409e3fe5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5. Remover os caracteres não numéricos da coluna PHONE, os - e os ()\n",
    "\n",
    "def remove_non_numeric(df):\n",
    "    df['PHONE'] = df['PHONE'].str.replace(r'[() -]+', '', regex=True)\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f12cb353",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing chunk 1\n",
      "Processing chunk 2\n",
      "Processing chunk 3\n",
      "Processing chunk 4\n",
      "Processing chunk 5\n",
      "Processing chunk 6\n",
      "Processing chunk 7\n",
      "Processing chunk 8\n",
      "Processing chunk 9\n",
      "Processing chunk 10\n",
      "Processing chunk 11\n",
      "Processing chunk 12\n",
      "Processing chunk 13\n",
      "Processing chunk 14\n",
      "Processing chunk 15\n",
      "Processing chunk 16\n",
      "Processing chunk 17\n",
      "Processing chunk 18\n",
      "Processing chunk 19\n",
      "Processing chunk 20\n",
      "Processing chunk 21\n",
      "Processing chunk 22\n",
      "Processing chunk 23\n",
      "Processing chunk 24\n",
      "Processing chunk 25\n",
      "Processing chunk 26\n",
      "Processing chunk 27\n",
      "Processing chunk 28\n",
      "Processing chunk 29\n",
      "Processing chunk 30\n",
      "Processing chunk 31\n",
      "Processing chunk 32\n",
      "Processing chunk 33\n",
      "Processing chunk 34\n",
      "Processing chunk 35\n",
      "Processing chunk 36\n",
      "Processing chunk 37\n",
      "Processing chunk 38\n",
      "Processing chunk 39\n",
      "Processing chunk 40\n",
      "Processing chunk 41\n",
      "Processing chunk 42\n",
      "Processing chunk 43\n",
      "Processing chunk 44\n",
      "Processing chunk 45\n",
      "Processing chunk 46\n",
      "Processing chunk 47\n",
      "Processing chunk 48\n",
      "Processing chunk 49\n",
      "Processing chunk 50\n",
      "Processing chunk 51\n",
      "Processing chunk 52\n",
      "Processing chunk 53\n",
      "Processing chunk 54\n",
      "Processing chunk 55\n",
      "Processing chunk 56\n",
      "Processing chunk 57\n",
      "Processing chunk 58\n",
      "Processing chunk 59\n",
      "Processing chunk 60\n",
      "Processing chunk 61\n",
      "Processing chunk 62\n",
      "Processing chunk 63\n",
      "Processing chunk 64\n",
      "Processing chunk 65\n",
      "Processing chunk 66\n",
      "Processing chunk 67\n",
      "Processing chunk 68\n",
      "Processing chunk 69\n",
      "Processing chunk 70\n",
      "Processing chunk 71\n",
      "Processing chunk 72\n",
      "Processing chunk 73\n",
      "Processing chunk 74\n",
      "Processing chunk 75\n",
      "Processing chunk 76\n",
      "Processing chunk 77\n",
      "Processing chunk 78\n",
      "Processing chunk 79\n",
      "Processing chunk 80\n",
      "Processing chunk 81\n",
      "Processing chunk 82\n",
      "Processing chunk 83\n",
      "Processing chunk 84\n",
      "Processing chunk 85\n",
      "Processing chunk 86\n",
      "Processing chunk 87\n",
      "Processing chunk 88\n",
      "Processing chunk 89\n",
      "Processing chunk 90\n",
      "Processing chunk 91\n",
      "Processing chunk 92\n",
      "Processing chunk 93\n",
      "Processing chunk 94\n",
      "Processing chunk 95\n",
      "Processing chunk 96\n",
      "Processing chunk 97\n",
      "Processing chunk 98\n",
      "Processing chunk 99\n",
      "Processing chunk 100\n",
      "Processing complete!\n"
     ]
    }
   ],
   "source": [
    "#6. Aplicar as funções a cada chunk e salvar em um novo arquivo CSV\n",
    "\n",
    "with open('arquivo_tratado_natal2021.csv', 'w', encoding='utf-8', newline='') as f:\n",
    "    for i, chunk in enumerate(chunks):\n",
    "        print(f'Processing chunk {i+1}')\n",
    "        chunk = remove_spaces(chunk)\n",
    "        chunk = create_ascii_col(chunk)\n",
    "        chunk = remove_non_numeric(chunk)\n",
    "        chunk.to_csv(f, header=(i == 0), index=True)\n",
    "print('Processing complete!')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c2962a37",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv ('arquivo_tratado_natal2021.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1db04f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>CUSTOMER_ID</th>\n",
       "      <th>CITY</th>\n",
       "      <th>PHONE</th>\n",
       "      <th>CITY_ASCII</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>aaf2e957-7842-4cca-854d-a29f1aad6dfd</td>\n",
       "      <td>São Paulo</td>\n",
       "      <td>51986338462</td>\n",
       "      <td>SAO PAULO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>26cb530b-a07f-41a7-b584-3fed3ab72385</td>\n",
       "      <td>São Paulo</td>\n",
       "      <td>51969771473</td>\n",
       "      <td>SAO PAULO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>c68ae612-d074-4af1-9f11-da5588d218e1</td>\n",
       "      <td>Florianópolis</td>\n",
       "      <td>51900166886</td>\n",
       "      <td>FLORIANOPOLIS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>b90f9e45-4faa-414d-b1a1-413783c04246</td>\n",
       "      <td>Porto Alegre</td>\n",
       "      <td>51927096260</td>\n",
       "      <td>PORTO ALEGRE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5dfe83e1-61c7-442e-8a9f-f30a5c649bae</td>\n",
       "      <td>São Paulo</td>\n",
       "      <td>51924852515</td>\n",
       "      <td>SAO PAULO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>99995</td>\n",
       "      <td>5f4997e2-5e4c-46c2-ab59-7fd6e6469689</td>\n",
       "      <td>São Paulo</td>\n",
       "      <td>51927822710</td>\n",
       "      <td>SAO PAULO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>99996</td>\n",
       "      <td>a161c52f-6986-4808-8eb2-19f5b21301b7</td>\n",
       "      <td>Florianópolis</td>\n",
       "      <td>51965547529</td>\n",
       "      <td>FLORIANOPOLIS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>99997</td>\n",
       "      <td>c8fdc9fb-fcdb-4233-80b0-de6bd06724da</td>\n",
       "      <td>São Paulo</td>\n",
       "      <td>51967013112</td>\n",
       "      <td>SAO PAULO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>99998</td>\n",
       "      <td>925140de-7132-4625-93b5-bf950d84cca0</td>\n",
       "      <td>São Paulo</td>\n",
       "      <td>51945979395</td>\n",
       "      <td>SAO PAULO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>99999</td>\n",
       "      <td>889a774f-8f8d-4647-8385-b19f0ee814b1</td>\n",
       "      <td>Florianópolis</td>\n",
       "      <td>51979713333</td>\n",
       "      <td>FLORIANOPOLIS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0                           CUSTOMER_ID           CITY  \\\n",
       "0               0  aaf2e957-7842-4cca-854d-a29f1aad6dfd      São Paulo   \n",
       "1               1  26cb530b-a07f-41a7-b584-3fed3ab72385      São Paulo   \n",
       "2               2  c68ae612-d074-4af1-9f11-da5588d218e1  Florianópolis   \n",
       "3               3  b90f9e45-4faa-414d-b1a1-413783c04246   Porto Alegre   \n",
       "4               4  5dfe83e1-61c7-442e-8a9f-f30a5c649bae      São Paulo   \n",
       "...           ...                                   ...            ...   \n",
       "99995       99995  5f4997e2-5e4c-46c2-ab59-7fd6e6469689      São Paulo   \n",
       "99996       99996  a161c52f-6986-4808-8eb2-19f5b21301b7  Florianópolis   \n",
       "99997       99997  c8fdc9fb-fcdb-4233-80b0-de6bd06724da      São Paulo   \n",
       "99998       99998  925140de-7132-4625-93b5-bf950d84cca0      São Paulo   \n",
       "99999       99999  889a774f-8f8d-4647-8385-b19f0ee814b1  Florianópolis   \n",
       "\n",
       "             PHONE     CITY_ASCII  \n",
       "0      51986338462      SAO PAULO  \n",
       "1      51969771473      SAO PAULO  \n",
       "2      51900166886  FLORIANOPOLIS  \n",
       "3      51927096260   PORTO ALEGRE  \n",
       "4      51924852515      SAO PAULO  \n",
       "...            ...            ...  \n",
       "99995  51927822710      SAO PAULO  \n",
       "99996  51965547529  FLORIANOPOLIS  \n",
       "99997  51967013112      SAO PAULO  \n",
       "99998  51945979395      SAO PAULO  \n",
       "99999  51979713333  FLORIANOPOLIS  \n",
       "\n",
       "[100000 rows x 5 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9df0e5f5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}