Rodrigo_campanha_natal_2021_V2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "0e3ba44d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importando as bibliotecas\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from unidecode import unidecode\n",
    "import re\n",
    "import chardet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "1d7cb209",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remover os espaços em branco e converter em letras maiúsculas; \n",
    "# Usar a biblioteca unidecode para tratar as colunas com acentos e caracteres especiais.\n",
    "\n",
    "def clean_city(city):\n",
    "    cleaned_city = city.strip().upper()\n",
    "    return unidecode(cleaned_city)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "083de391",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remover os caracteres não numéricos\n",
    "\n",
    "def clean_phone(phone):\n",
    "    return re.sub(r'\\D', '', phone)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "dcfb518a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ler, tratar e processar os dados do arquivo CSV em chunks.\n",
    "\n",
    "def process_chunk(chunk):\n",
    "    chunk['CITY'] = chunk['CITY'].apply(lambda x: x.strip())\n",
    "    chunk['CITY_ASCII'] = chunk['CITY'].apply(clean_city)\n",
    "    chunk['PHONE'] = chunk['PHONE'].apply(clean_phone)\n",
    "    return chunk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "2904918a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Arquivo de entrada e de saída.\n",
    "\n",
    "input_file = 'natal2021.csv'\n",
    "output_file = 'arquivo_tratado_natal2021_V2.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "8cc63b67",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Identificar o encoding do arquivo\n",
    "\n",
    "with open(input_file, 'rb') as file:\n",
    "    result = chardet.detect(file.read())\n",
    "encoding = result['encoding']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "ee4ecdd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ler e processar os pedaços (chunks de 1000 linhas) do arquivo CSV.\n",
    "\n",
    "chunksize = 1000\n",
    "chunks = []\n",
    "\n",
    "for chunk in pd.read_csv(input_file, encoding=encoding, chunksize=chunksize):\n",
    "    cleaned_chunk = process_chunk(chunk)\n",
    "    chunks.append(cleaned_chunk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "e600cb93",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concatenar os chunks e salvar o arquivo .csv com a nova coluna em utf-8-sig com BOM\n",
    "\n",
    "cleaned_data = pd.concat(chunks, axis=0)\n",
    "cleaned_data.to_csv(output_file, index=False, encoding='utf-8-sig')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "f8c4e5c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ler o arquivo CSV e criar um dataframe.\n",
    "\n",
    "df = pd.read_csv ('arquivo_tratado_natal2021_V2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "cf1ce941",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CUSTOMER_ID</th>\n",
       "      <th>CITY</th>\n",
       "      <th>PHONE</th>\n",
       "      <th>CITY_ASCII</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aaf2e957-7842-4cca-854d-a29f1aad6dfd</td>\n",
       "      <td>São Paulo</td>\n",
       "      <td>51986338462</td>\n",
       "      <td>SAO PAULO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>26cb530b-a07f-41a7-b584-3fed3ab72385</td>\n",
       "      <td>São Paulo</td>\n",
       "      <td>51969771473</td>\n",
       "      <td>SAO PAULO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c68ae612-d074-4af1-9f11-da5588d218e1</td>\n",
       "      <td>Florianópolis</td>\n",
       "      <td>51900166886</td>\n",
       "      <td>FLORIANOPOLIS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b90f9e45-4faa-414d-b1a1-413783c04246</td>\n",
       "      <td>Porto Alegre</td>\n",
       "      <td>51927096260</td>\n",
       "      <td>PORTO ALEGRE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5dfe83e1-61c7-442e-8a9f-f30a5c649bae</td>\n",
       "      <td>São Paulo</td>\n",
       "      <td>51924852515</td>\n",
       "      <td>SAO PAULO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>5f4997e2-5e4c-46c2-ab59-7fd6e6469689</td>\n",
       "      <td>São Paulo</td>\n",
       "      <td>51927822710</td>\n",
       "      <td>SAO PAULO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>a161c52f-6986-4808-8eb2-19f5b21301b7</td>\n",
       "      <td>Florianópolis</td>\n",
       "      <td>51965547529</td>\n",
       "      <td>FLORIANOPOLIS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>c8fdc9fb-fcdb-4233-80b0-de6bd06724da</td>\n",
       "      <td>São Paulo</td>\n",
       "      <td>51967013112</td>\n",
       "      <td>SAO PAULO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>925140de-7132-4625-93b5-bf950d84cca0</td>\n",
       "      <td>São Paulo</td>\n",
       "      <td>51945979395</td>\n",
       "      <td>SAO PAULO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>889a774f-8f8d-4647-8385-b19f0ee814b1</td>\n",
       "      <td>Florianópolis</td>\n",
       "      <td>51979713333</td>\n",
       "      <td>FLORIANOPOLIS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  CUSTOMER_ID           CITY        PHONE  \\\n",
       "0        aaf2e957-7842-4cca-854d-a29f1aad6dfd      São Paulo  51986338462   \n",
       "1      26cb530b-a07f-41a7-b584-3fed3ab72385        São Paulo  51969771473   \n",
       "2        c68ae612-d074-4af1-9f11-da5588d218e1  Florianópolis  51900166886   \n",
       "3        b90f9e45-4faa-414d-b1a1-413783c04246   Porto Alegre  51927096260   \n",
       "4       5dfe83e1-61c7-442e-8a9f-f30a5c649bae       São Paulo  51924852515   \n",
       "...                                       ...            ...          ...   \n",
       "99995    5f4997e2-5e4c-46c2-ab59-7fd6e6469689      São Paulo  51927822710   \n",
       "99996    a161c52f-6986-4808-8eb2-19f5b21301b7  Florianópolis  51965547529   \n",
       "99997    c8fdc9fb-fcdb-4233-80b0-de6bd06724da      São Paulo  51967013112   \n",
       "99998    925140de-7132-4625-93b5-bf950d84cca0      São Paulo  51945979395   \n",
       "99999    889a774f-8f8d-4647-8385-b19f0ee814b1  Florianópolis  51979713333   \n",
       "\n",
       "          CITY_ASCII  \n",
       "0          SAO PAULO  \n",
       "1          SAO PAULO  \n",
       "2      FLORIANOPOLIS  \n",
       "3       PORTO ALEGRE  \n",
       "4          SAO PAULO  \n",
       "...              ...  \n",
       "99995      SAO PAULO  \n",
       "99996  FLORIANOPOLIS  \n",
       "99997      SAO PAULO  \n",
       "99998      SAO PAULO  \n",
       "99999  FLORIANOPOLIS  \n",
       "\n",
       "[100000 rows x 4 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Visualizar o dataframe.\n",
    "\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}