alibaba_crawler_final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import selenium\n",
    "from selenium import webdriver\n",
    "import time\n",
    "import pandas as pd\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract Links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_links():\n",
    "    # To extract links from given list of category urls and store in csv\n",
    "    driver = webdriver.Chrome('/home/atul/scraping_driver/chromedriver')\n",
    "    driver.maximize_window()\n",
    "    time.sleep(5)\n",
    "    comp_categ_list = []\n",
    "    comp_list_link = []\n",
    "    hyperlinks = ['https://www.alibaba.com/Medical-Devices_p100002906?spm=a27aq.13923566.9010494120.12.1d7231a8rQzV1u','https://www.alibaba.com/Medical-Consumables_p127866004?spm=a27aq.13923566.9010494120.13.1d7231a8rQzV1u','https://www.alibaba.com/Health-Care-Products_p100002908?spm=a27aq.13923566.9010494120.16.1d7231a8rQzV1u']\n",
    "    i=0\n",
    "    for link in hyperlinks:\n",
    "        print(link)\n",
    "        if i == 0:\n",
    "            categ_str = 'Medical Devices'\n",
    "        elif i == 1:\n",
    "            categ_str = 'Medical Consumables'\n",
    "        elif i == 2:\n",
    "            categ_str = 'Health Care Products'\n",
    "        driver.get(link)\n",
    "        time.sleep(10)\n",
    "        SCROLL_PAUSE_TIME = 0.7\n",
    "\n",
    "        # Get scroll height\n",
    "        last_height = driver.execute_script(\"return document.body.scrollHeight\")+100\n",
    "\n",
    "        while True:\n",
    "            # Scroll down to bottom\n",
    "            driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "\n",
    "            # Wait to load page\n",
    "            time.sleep(SCROLL_PAUSE_TIME)\n",
    "\n",
    "            # Calculate new scroll height and compare with last scroll height\n",
    "            new_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "            if new_height == last_height:\n",
    "                break\n",
    "            last_height = new_height\n",
    "        time.sleep(5)\n",
    "        elements = driver.find_elements_by_class_name('product-detail')\n",
    "        print(len(elements))\n",
    "        links = set()\n",
    "        for element in elements:\n",
    "            product_link = element.get_attribute('href')\n",
    "            if product_link not in links:\n",
    "                links.add(product_link)\n",
    "            else:\n",
    "                continue\n",
    "        list_links = list(links)\n",
    "        print(\"Category\",i,len(list_links))\n",
    "        comp_list_link.extend(list_links)\n",
    "        comp_categ_list.extend([categ_str]*len(list_links))\n",
    "        i+=1\n",
    "    driver.quit()\n",
    "    df = pd.DataFrame(zip(comp_categ_list,comp_list_link),columns = ['category','product_urls'])\n",
    "    print(df)\n",
    "    df.to_csv('alibaba_product_urls1.csv',index = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_links()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('alibaba_product_urls1.csv') #update path of product_urls.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_company_overview(driver):\n",
    "    keys = [i.text for i in driver.find_elements_by_css_selector('.field-title')]\n",
    "    values = [i.get_attribute('title') if i.get_attribute('title')!='' else i.text for i in driver.find_elements_by_css_selector('.content-value') ]\n",
    "    return dict(zip(keys,values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_production_capacity(driver):\n",
    "    prod_cap_dic = {}\n",
    "    prod_cap_dic['factory_info'] = {}\n",
    "    keys = [i.text for i in driver.find_elements_by_css_selector('.icbu-shop-table-col-item .title div')]\n",
    "    values = [i.text for i in driver.find_elements_by_css_selector('.icbu-shop-table-col-item .content span')]\n",
    "    prod_cap_dic['factory_info'] = dict(zip(keys,values))\n",
    "    return prod_cap_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rd_capacity(driver):\n",
    "    rd_capacity = {}\n",
    "    rd_capacity['certification'] = []\n",
    "    keys = ['Picture','Certification Name','Issued By','Business Scope','Available Date','Verified']\n",
    "    li = driver.find_elements_by_css_selector('.icbu-pc-cpRDCapacity .next-table-cell .next-table-cell-wrapper')\n",
    "    val = []\n",
    "    for element in li:\n",
    "        if element.text=='':\n",
    "            try:\n",
    "                new_val = element.find_element_by_tag_name('img').get_attribute('src')\n",
    "            except:\n",
    "                new_val = element.text\n",
    "        else:\n",
    "            new_val = element.text\n",
    "        val.append(new_val)\n",
    "    i=0\n",
    "    while i< len(val):\n",
    "        d = dict(zip(keys,val[i:i+6]))\n",
    "        i+=6\n",
    "        rd_capacity['certification'].append(d)\n",
    "    #rd_capacity['no_of_people'] = [i.text for i in driver.find_elements_by_css_selector('.icbu-pc-cpRDCapacity .infoList-mod-field~ .infoList-mod-field+ .infoList-mod-field .content div')][-1]\n",
    "    return rd_capacity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_trade_capacity(driver):\n",
    "    tr_capacity = {}\n",
    "    tr_capacity['main_markets and products'] = []\n",
    "    keys = ['Main Markets', 'Total Revenue(%)','Main Product(s)','Verified']\n",
    "    li = [i.text for i in driver.find_elements_by_css_selector('.icbu-pc-cpTradeCapability .next-table-cell .next-table-cell-wrapper')]\n",
    "    i=0\n",
    "    while i< len(li):\n",
    "        d = dict(zip(keys,li[i:i+4]))\n",
    "        i+=4\n",
    "        tr_capacity['main_markets and products'].append(d)\n",
    "    return tr_capacity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_product_details(browser):\n",
    "    product_details = {}\n",
    "    try:\n",
    "        title = browser.find_element_by_class_name(\"ma-title\")\n",
    "        product_details[\"title\"] = title.text\n",
    "        print(\"-------1-------\")\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "\n",
    "    try:\n",
    "        icons = browser.find_element_by_class_name(\"util-clearfix\")\n",
    "        icon_list = icons.find_elements_by_css_selector(\"img\")\n",
    "        images_list = []\n",
    "        for icon in icon_list:\n",
    "#                 icon.click()\n",
    "            images_list.append(str(icon.get_attribute(\"src\")).replace(\"_50x50.jpg\",\"\"))\n",
    "        product_details[\"product_images\"] = images_list\n",
    "        print(\"-------2-------\")\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "\n",
    "#     details = browser.find_element_by_class_name(\"module-productPackagingAndQuickDetail\")\n",
    "#     print(details.text)\n",
    "\n",
    "    overview = browser.find_element_by_class_name(\"do-overview\")\n",
    "    entries = overview.find_elements_by_class_name(\"do-entry\")\n",
    "    overview_dict = {}\n",
    "    for entry in entries:\n",
    "        title = entry.find_element_by_class_name(\"do-entry-title\")\n",
    "        overview_list = entry.find_element_by_class_name(\"do-entry-list\")\n",
    "        entry_items = overview_list.find_elements_by_tag_name(\"dl\")\n",
    "        item_dict = {}\n",
    "        for item in entry_items:\n",
    "            try:\n",
    "                item_list = str(item.text).splitlines()\n",
    "                item_dict[item_list[0]] = item_list[1]\n",
    "            except Exception as e:\n",
    "                print(\"List\",e)\n",
    "                continue\n",
    "            print(\"-------3-------\")\n",
    "        overview_dict[title.text] = item_dict\n",
    "    product_details[\"overview\"] = overview_dict\n",
    " ###############################################################################################################   \n",
    "    \n",
    "    product_desc = {}\n",
    "    try:\n",
    "        child_eles = browser.find_elements_by_xpath(\"//*[@id='J-rich-text-description']/div\")\n",
    "        for child_div in child_eles:\n",
    "            data_section = child_div.get_attribute(\"data-section-title\")\n",
    "            print(data_section)\n",
    "            if data_section ==\"Product Description\":\n",
    "                print(data_section)\n",
    "                try:\n",
    "                    tables = child_div.find_elements_by_tag_name('table')[1]\n",
    "                    table1_dict = {}\n",
    "                    tr_list = tables.find_elements_by_tag_name('tr')\n",
    "                    for tr in tr_list:\n",
    "                        td = tr.find_elements_by_tag_name('td')\n",
    "                        table1_dict[td[0].text] = td[1].text\n",
    "                        print(\"-------4-------\",table1_dict)\n",
    "                        product_desc = table1_dict\n",
    "                except Exception as e:\n",
    "                    # Not a table, handling for image\n",
    "                    print(e)\n",
    "                # For images:\n",
    "                \n",
    "            elif data_section==\"User Manual\":\n",
    "                manual_img = []\n",
    "                try:\n",
    "                    user_manual = child_div\n",
    "                    user_manual_img = user_manual.find_element_by_tag_name('img')\n",
    "                    manual_img.append(user_manual_img.get_attribute('data-src'))\n",
    "                    print(\"\")\n",
    "                    print(\"-------7-------\")\n",
    "                except Exception as e:\n",
    "                    print(\"User Manual:\",e)\n",
    "            elif data_section==\"Company Overview\":\n",
    "                try:\n",
    "                    company_overview = child_div\n",
    "                    company_overview_imgs = company_overview.find_elements_by_tag_name('img')\n",
    "                    for img in company_overview_imgs:\n",
    "                        manual_img.append(img.get_attribute('data-src'))\n",
    "                        print(\"-------8-------\")\n",
    "                    product_details[\"user_manual\"] = manual_img\n",
    "                except Exception as e:\n",
    "                    print(\"Company Overview:\",e)\n",
    "            elif data_section==\"Our Partners\":                \n",
    "                try:\n",
    "                    partner = child_div\n",
    "                    partner_imgs = partner.find_elements_by_tag_name('img')\n",
    "                    for img in partner_imgs:\n",
    "                        product_details[\"partner\"] = img.get_attribute('data-src')\n",
    "                        print(\"-------9-------\")\n",
    "                except:\n",
    "                    print(\"Our Partners:\",e)\n",
    "            elif data_section==\"Our Services\":                \n",
    "                try:\n",
    "                    our_service = {}\n",
    "                    services = child_div\n",
    "\n",
    "                    service_table = services.find_elements_by_tag_name(\"table\")[1]\n",
    "                    tr_list = service_table.find_elements_by_tag_name('tr')\n",
    "                    service_dict = {}\n",
    "                    for tr in tr_list:\n",
    "                        td = tr.find_elements_by_tag_name('td')\n",
    "                        service_dict[td[0].text] = td[1].text\n",
    "                    our_service[\"description\"] = service_dict\n",
    "\n",
    "                    service_img = services.find_element_by_tag_name('img')\n",
    "                    our_service[\"images\"] = service_img.get_attribute('data-src')\n",
    "\n",
    "                    product_details[\"our_services\"] = our_service\n",
    "                except Exception as e:\n",
    "                    print(\"Our services:\",e)\n",
    "            elif data_section==\"Certifications\":\n",
    "                try:\n",
    "                    certificate = child_div\n",
    "                    certificate_img = certificate.find_element_by_tag_name('img')\n",
    "                    product_details[\"certificate\"] = certificate_img.get_attribute('data-src')\n",
    "                except Exception as e:\n",
    "                    print(\"certificate:\",e)\n",
    "            elif data_section==\"Packing & Delivery\":                \n",
    "                try:\n",
    "                    pack_delv_dict = {}\n",
    "                    pack_deliv = child_div\n",
    "                    pack_deliv_imgs = pack_deliv.find_elements_by_tag_name('img')\n",
    "                    pack_delv_dict[\"packing\"] = pack_deliv_imgs[0].get_attribute('data-src')\n",
    "                #         print('---paking----',packing)\n",
    "\n",
    "                    pack_delv_dict[\"delivery\"] = pack_deliv_imgs[1].get_attribute('data-src')\n",
    "                #         print('--delivery---',delivery)\n",
    "                    delivery_table = pack_deliv.find_elements_by_tag_name(\"table\")[1]\n",
    "                    tr_list = delivery_table.find_elements_by_tag_name('tr')\n",
    "                    delivery_dict = {}\n",
    "                    for tr in tr_list:\n",
    "                        td = tr.find_elements_by_tag_name('td')\n",
    "                        delivery_dict[td[0].text] = td[1].text\n",
    "                    pack_delv_dict[\"description\"] = delivery_dict\n",
    "                    product_details[\"packing_delivery\"] = pack_delv_dict\n",
    "                except Exception as e:\n",
    "                    print(\"packaging:\",e)\n",
    "            elif data_section==\"FAQ\":\n",
    "                try:\n",
    "                    faqs = child_div\n",
    "                #     print(faqs.text)\n",
    "                    faq_list = faqs.find_elements_by_tag_name('span')\n",
    "                    #if len(faq_list)==1:\n",
    "                        \n",
    "                    faq_dict = {}\n",
    "                    for faq in faq_list:\n",
    "                        if \":\" in faq.text:\n",
    "                            faq_qa = str(faq.text).split(\":\")\n",
    "                            faq_dict[faq_qa[0]] = faq_qa[1]\n",
    "                    product_details[\"FAQ\"] = faq_dict\n",
    "                except Exception as e:\n",
    "                    print(\"faq:\",e)\n",
    "        try:\n",
    "            product_details['image_content']={}\n",
    "            dic = {}\n",
    "            for p in child_eles:\n",
    "                for ele in p.find_elements_by_tag_name('img'):\n",
    "                    src = ele.get_attribute('data-src')\n",
    "                    if src:\n",
    "                        dic[p.get_attribute('data-section-title')]='https:'+src\n",
    "                product_details['image_content']=dic\n",
    "        except Exception as e:\n",
    "            print(\"image content\",e)\n",
    "        return product_details\n",
    "    except:\n",
    "        print(\"Product desc not found !\")  \n",
    "        return product_details                  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_product_company_details(driver):\n",
    "    dic = {}\n",
    "    dic['product_details']  = {}\n",
    "    dic['product_details'] = get_product_details(driver)\n",
    "    # company\n",
    "    try:\n",
    "        driver.find_element_by_css_selector('.active+ .next-tabs-nav-appear-active .tab-name').click()\n",
    "        dic['company_profile'] = {}\n",
    "        dic['company_profile']['company_overview'] = dict()\n",
    "        dic['company_profile']['production_capacity'] = dict()\n",
    "        dic['company_profile']['r&d_capacity'] = dict()\n",
    "        dic['company_profile']['trade_capacity'] = dict()\n",
    "        dic['company_profile']['company_overview'] = get_company_overview(driver)\n",
    "        dic['company_profile']['production_capacity'] = get_production_capacity(driver)\n",
    "        try:\n",
    "            driver.find_element_by_css_selector('.icbu-pc-cpRDCapacity .view-more').click()\n",
    "            dic['company_profile']['r&d_capacity'] = get_rd_capacity(driver)\n",
    "        except Exception as e:\n",
    "        #J-ls-grid-desc > div.tab-body > div.tab-body-pane.ls-icon.ls-company.show > div.alisite > div:nth-child(5) > div > div > div > div.mod-content > div > div.mod-view-more > div\n",
    "            print(\"R&D\",e)\n",
    "        try:\n",
    "            driver.find_element_by_css_selector('.icbu-pc-cpTradeCapability .view-more').click()\n",
    "            dic['company_profile']['trade_capacity'] = get_trade_capacity(driver)\n",
    "        except Exception as e:\n",
    "            print(\"Trade\",e)\n",
    "\n",
    "    except Exception as e:\n",
    "        print(\"Company\",e)\n",
    "    return dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_json_for_url(url):\n",
    "    # To extract content from a product url\n",
    "    driver = webdriver.Chrome('/home/atul/scraping_driver/chromedriver')\n",
    "    driver.maximize_window()\n",
    "    time.sleep(5)\n",
    "    driver.get(url)\n",
    "    time.sleep(5)\n",
    "    product_name = driver.find_element_by_class_name('ma-title').get_attribute('title')\n",
    "    SCROLL_PAUSE_TIME = 0.7\n",
    "\n",
    "    # Get scroll height\n",
    "    last_height = driver.execute_script(\"return document.body.scrollHeight\")+100\n",
    "\n",
    "    while True:\n",
    "        # Scroll down to bottom\n",
    "        driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "\n",
    "        # Wait to load page\n",
    "        time.sleep(SCROLL_PAUSE_TIME)\n",
    "\n",
    "        # Calculate new scroll height and compare with last scroll height\n",
    "        new_height = driver.execute_script(\"return document.body.scrollHeight\")\n",
    "        if new_height == last_height:\n",
    "            break\n",
    "        last_height = new_height\n",
    "\n",
    "    time.sleep(10)\n",
    "    dic = get_product_company_details(driver)\n",
    "    driver.quit()\n",
    "    with open('crawler_output_files/result_'+product_name+'.json', 'w') as fp:\n",
    "        json.dump(dic, fp)\n",
    "    return product_name,dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def store_multiple_json_files(df,n=len(df)):\n",
    "    beg = time.time()\n",
    "    print(n)\n",
    "    full_json = {}\n",
    "    product_names = []\n",
    "    product_urls = []\n",
    "    file_names = []\n",
    "    count = 0 \n",
    "    i = 0\n",
    "    while count<n and i<len(df):\n",
    "        print(\"Crawling url no.:\",i)\n",
    "        try:\n",
    "            print(df['product_urls'][i])\n",
    "            product_name,dic = get_json_for_url(df['product_urls'][i])\n",
    "            product_names.append(product_name)\n",
    "            product_urls.append(df['product_urls'][i])\n",
    "            file_names.append('result_'+product_name+'.json')\n",
    "            full_json[i]={}\n",
    "            count+=1\n",
    "        except Exception as e:\n",
    "            print(\"Page not found\",e)\n",
    "            #full_json[i]='Not Found'\n",
    "            dic = 'Page not found'\n",
    "        full_json[i]=dic\n",
    "        i+=1\n",
    "        print(\"Got data from.:\",count,\"urls\")\n",
    "    output_df = pd.DataFrame({'product_name':product_names,'product_url':product_urls,'file_name':file_names})\n",
    "    print(\"Time taken for \",n,\"urls:\",(time.time()-beg),\"seconds\")\n",
    "    return output_df,full_json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Crawl Links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "begin = time.time()\n",
    "output_df,full_json = store_multiple_json_files(df,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('all_products_data.json','w') as file:\n",
    "    file.write(json.dumps(full_json, indent=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ldf = pd.DataFrame(full_json).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ldf.to_csv('file.csv',index = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ldf['product_details'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ldf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_json[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_df.to_csv('final_df.csv',index = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_size = len(full_json.keys())\n",
    "title_lst = [None]*list_size\n",
    "product_img_lst = [None]*list_size\n",
    "overview_lst = [None]*list_size\n",
    "our_services = [None]*list_size\n",
    "packaging_delivery = [None]*list_size\n",
    "faq = [None]*list_size\n",
    "image_content = [None]*list_size\n",
    "company_overview = [None]*list_size\n",
    "production_capacity = [None]*list_size\n",
    "r_d_capacity = [None]*list_size\n",
    "trade_capacity = [None]*list_size\n",
    "user_manual = [None]*list_size\n",
    "for key in full_json.keys():\n",
    "    if full_json[key]!='Page not found':\n",
    "        title_lst[key] = str(full_json[key]['product_details']['title'])\n",
    "        product_img_lst[key] = str(full_json[key]['product_details']['product_images'])\n",
    "        overview_lst[key] = str(full_json[key]['product_details']['overview'])\n",
    "        if full_json[key]['product_details'].get('user_manual'):\n",
    "            user_manual[key] = str(full_json[key]['product_details']['user_manual'])\n",
    "        if full_json[key]['product_details'].get('our_services'):\n",
    "            our_services[key] = str(full_json[key]['product_details']['our_services'])\n",
    "\n",
    "        if full_json[key]['product_details']['overview'].get('Packaging & Delivery'):\n",
    "            packaging_delivery[key] = str(full_json[key]['product_details']['overview']['Packaging & Delivery'])\n",
    "\n",
    "        if full_json[key]['product_details'].get('FAQ'):   \n",
    "            faq[key] = str(full_json[key]['product_details']['FAQ'])\n",
    "\n",
    "\n",
    "        image_content[key] = str(full_json[key]['product_details']['image_content'])\n",
    "\n",
    "        company_overview[key] = str(full_json[key]['company_profile']['company_overview'])\n",
    "        production_capacity[key] = str(full_json[key]['company_profile']['production_capacity'])\n",
    "        r_d_capacity[key] = str(full_json[key]['company_profile']['r&d_capacity'])\n",
    "        trade_capacity[key] = str(full_json[key]['company_profile']['trade_capacity'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dic = {'product_title':title_lst,\n",
    " 'product_images':product_img_lst,\n",
    " 'overview':overview_lst,\n",
    " 'user_manual':user_manual,\n",
    " 'our_services':our_services,\n",
    " 'packaging_delivery':packaging_delivery,\n",
    " 'FAQ': faq,\n",
    " 'image_content':image_content,\n",
    " 'company_overview':company_overview,\n",
    " 'production_capacity':production_capacity,\n",
    " 'r&d_capacity':r_d_capacity,\n",
    " 'trade_capacity':trade_capacity\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df1 = pd.DataFrame(df_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df1.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df1['product_url'] = df['product_urls']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df1['category'] = df['category']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df1.to_csv('final_output_file.csv',index = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}