FakeNews_model_final.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FakeNews-model-final.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "daab1c4f4f754a58baeea9b09ab21c36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cffca6798e8049ef82c103e846682484",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_75520c6f09b744ee99a4251db43c29c5",
              "IPY_MODEL_b70d73ee60e2477086dab11a69c93d69"
            ]
          }
        },
        "cffca6798e8049ef82c103e846682484": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "75520c6f09b744ee99a4251db43c29c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6b73ad83d59c4e24be113ee7d8054f5a",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 871891,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 871891,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9951a26d574443508cef4e52cfc3d6a4"
          }
        },
        "b70d73ee60e2477086dab11a69c93d69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_70a1a7f5d5f94c3f9506c0bec588f177",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 872k/872k [00:01&lt;00:00, 649kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_214c22368222442d8f246bebddc442db"
          }
        },
        "6b73ad83d59c4e24be113ee7d8054f5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9951a26d574443508cef4e52cfc3d6a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "70a1a7f5d5f94c3f9506c0bec588f177": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "214c22368222442d8f246bebddc442db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ec656a3f08634484a792cea9818ccbe5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_55056ea58e2547f5891b5452b59a18ca",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c82adb7f882948c0baae1e1bd8179adf",
              "IPY_MODEL_bd89214d587448bbbe02d05cbfeca515"
            ]
          }
        },
        "55056ea58e2547f5891b5452b59a18ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c82adb7f882948c0baae1e1bd8179adf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c199cf39dc464f998bf8409cb4990424",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f62c0fbd316a48179dc4ca41d4bc5242"
          }
        },
        "bd89214d587448bbbe02d05cbfeca515": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_56479aba064b4f2b96551f82de337e64",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:01&lt;00:00, 17.5B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d2364aa2765943658eb7218149ac1a6e"
          }
        },
        "c199cf39dc464f998bf8409cb4990424": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f62c0fbd316a48179dc4ca41d4bc5242": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "56479aba064b4f2b96551f82de337e64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d2364aa2765943658eb7218149ac1a6e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aa624cc045cf4dc8a2436f4b125d24b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a784c1deec11401a829e367dd399c579",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a0138461eec448739cbcaf0a264731d0",
              "IPY_MODEL_61a2141d42c34ddd8e8b01055697ec18"
            ]
          }
        },
        "a784c1deec11401a829e367dd399c579": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a0138461eec448739cbcaf0a264731d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2fa7d3288a6445f2a78c0882e6debe41",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1715180,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1715180,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a9ccead30acf4043bede1b48579d1b5e"
          }
        },
        "61a2141d42c34ddd8e8b01055697ec18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_34975406f3f54d1e9ec30176e6b78e30",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.72M/1.72M [00:00&lt;00:00, 5.94MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_07a184cd77a14834b118a54eb594124d"
          }
        },
        "2fa7d3288a6445f2a78c0882e6debe41": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a9ccead30acf4043bede1b48579d1b5e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "34975406f3f54d1e9ec30176e6b78e30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "07a184cd77a14834b118a54eb594124d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a0b4a7a05f3c4eb99076068d6a738f22": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1211462e2faa444e8883135a5012c2f6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_081451d244ab44f790eb7999c14fa3d0",
              "IPY_MODEL_b5225cf1534446298b9c8c50ded292b9"
            ]
          }
        },
        "1211462e2faa444e8883135a5012c2f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "081451d244ab44f790eb7999c14fa3d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_651a8342c0e046aba6bd2d864ac46179",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 625,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 625,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_37cc7d69e67a4a968102041421d58692"
          }
        },
        "b5225cf1534446298b9c8c50ded292b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f8a78a6ce88f405a9df4b01f79074d81",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 625/625 [00:00&lt;00:00, 10.7kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_21cc59f9714240eb8673ba0d19308353"
          }
        },
        "651a8342c0e046aba6bd2d864ac46179": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "37cc7d69e67a4a968102041421d58692": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f8a78a6ce88f405a9df4b01f79074d81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "21cc59f9714240eb8673ba0d19308353": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "eb1f490f8daa41e4986e47180cc1dd44": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1742885195ec489b89a2178156bc91fc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_863693a307234b5291ad057a30517239",
              "IPY_MODEL_93e1680a49834dc9bd65f14cb50c2fd5"
            ]
          }
        },
        "1742885195ec489b89a2178156bc91fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "863693a307234b5291ad057a30517239": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_07e1e46c84664b32bf0fdba1b890d94c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 672271273,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 672271273,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_caac574908c441bcbc57a4662a0a6c4f"
          }
        },
        "93e1680a49834dc9bd65f14cb50c2fd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ede467e203d948f3bf3cc3ced057649a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 672M/672M [00:24&lt;00:00, 27.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f9243e449bb447d7b2201a456ef6a828"
          }
        },
        "07e1e46c84664b32bf0fdba1b890d94c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "caac574908c441bcbc57a4662a0a6c4f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ede467e203d948f3bf3cc3ced057649a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f9243e449bb447d7b2201a456ef6a828": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Siddhartha15/Hindi-Fake-News-Detection/blob/main/FakeNews_model_final.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BFHmELDY98Qg",
        "outputId": "810857d6-6ac4-4635-e685-3b19c5bd6782"
      },
      "source": [
        "!git clone https://github.com/Jelwin13afc/FakeNewsDetection"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'FakeNewsDetection'...\n",
            "remote: Enumerating objects: 36, done.\u001b[K\n",
            "remote: Counting objects: 100% (36/36), done.\u001b[K\n",
            "remote: Compressing objects: 100% (29/29), done.\u001b[K\n",
            "remote: Total 36 (delta 18), reused 18 (delta 7), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (36/36), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gKNSQ1_3-K9w"
      },
      "source": [
        "import pandas as pd\n",
        "fake_news=pd.read_csv('/content/FakeNewsDetection/fake_news_basic.csv')\n",
        "true_news=pd.read_csv('/content/FakeNewsDetection/true_news_basic.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "id": "jze2XMjpDLCK",
        "outputId": "1b8e5134-19c9-4f54-a20b-c1ad4563004b"
      },
      "source": [
        "\"\"\"\n",
        "#add additional true news data\n",
        "df_data = pd.read_csv('/content/FakeNewsDetection/more_true_news.csv', sep=\"\\t\", encoding='utf-8', header=None)\n",
        "\n",
        "# true_news=true_news.drop(['long_description','full_title'],axis=1)\n",
        "fake_news=fake_news.drop(['short_description','full_title'],axis=1)\n",
        "\n",
        "df_data = df_data.drop(0,axis=1)\n",
        "df_data = df_data.rename(columns={1:\"long_description\"})\n",
        "df_data=df_data.dropna()\n",
        "true_news = df_data\n",
        "\n",
        "# frame = [true_news, df_data]\n",
        "# true_news = pd.concat(frame, ignore_index=True)\n",
        "\n",
        "# true_news=true_news.drop(['Unnamed: 0'],axis=1)\n",
        "fake_news=fake_news.drop(['Unnamed: 0'],axis=1)\n",
        "\"\"\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'\\n#add additional true news data\\ndf_data = pd.read_csv(\\'/content/FakeNewsDetection/more_true_news.csv\\', sep=\"\\t\", encoding=\\'utf-8\\', header=None)\\n\\n# true_news=true_news.drop([\\'long_description\\',\\'full_title\\'],axis=1)\\nfake_news=fake_news.drop([\\'short_description\\',\\'full_title\\'],axis=1)\\n\\ndf_data = df_data.drop(0,axis=1)\\ndf_data = df_data.rename(columns={1:\"long_description\"})\\ndf_data=df_data.dropna()\\ntrue_news = df_data\\n\\n# frame = [true_news, df_data]\\n# true_news = pd.concat(frame, ignore_index=True)\\n\\n# true_news=true_news.drop([\\'Unnamed: 0\\'],axis=1)\\nfake_news=fake_news.drop([\\'Unnamed: 0\\'],axis=1)\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "88i7ecU7Y7LB"
      },
      "source": [
        "true_news=true_news.drop(['Unnamed: 0'],axis=1)\n",
        "fake_news=fake_news.drop(['Unnamed: 0'],axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "id": "Wsd5N70L-hkY",
        "outputId": "3749cb0e-11e1-4847-e377-89023a9530df"
      },
      "source": [
        "fake_news.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>short_description</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>बूम पाय इमरान खान भारत सरकार आलोच रह वर्तमान श...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>सिख समुदाय के लोग हिंद साइन बोर्ड कालिख पोत दि...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>सोशल मीडिय प्लेटफ़ॉर्म फ़ेसबुक ट्विटर दाव के बड़ ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>दाव भाजप के मा सरकार जन के गलत रह ।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>मीडिय आउटलेट्स वायर एजेंस गलत तरीक दाव किय पाक...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                   short_description\n",
              "0  बूम पाय इमरान खान भारत सरकार आलोच रह वर्तमान श...\n",
              "1  सिख समुदाय के लोग हिंद साइन बोर्ड कालिख पोत दि...\n",
              "2  सोशल मीडिय प्लेटफ़ॉर्म फ़ेसबुक ट्विटर दाव के बड़ ...\n",
              "3               दाव भाजप के मा सरकार जन के गलत रह । \n",
              "4  मीडिय आउटलेट्स वायर एजेंस गलत तरीक दाव किय पाक..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "id": "WAYYwdqE-ml7",
        "outputId": "98917375-b8f8-474d-9987-aa8b8e48af0f"
      },
      "source": [
        "true_news.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>short_description</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>भारत ऑस्ट्रेलिय टेस्ट सिरीज़ ऐतिहासिक जीत दर्ज...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ब्रिस्बेन टेस्ट भारत दीवार कह वाल चेतेश्वर पुज...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>पाकिस्ता टीव प्रेजेंटर सैय्यद इक़रारुल हसन अप ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>पूछ जाए दिल्ल पहल गणतंत्र दिवस परेड कह इसक जवा...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>वेब सिरीज़ तांडव ले जार विवाद कार्टून</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                   short_description\n",
              "0  भारत ऑस्ट्रेलिय टेस्ट सिरीज़ ऐतिहासिक जीत दर्ज...\n",
              "1  ब्रिस्बेन टेस्ट भारत दीवार कह वाल चेतेश्वर पुज...\n",
              "2  पाकिस्ता टीव प्रेजेंटर सैय्यद इक़रारुल हसन अप ...\n",
              "3  पूछ जाए दिल्ल पहल गणतंत्र दिवस परेड कह इसक जवा...\n",
              "4             वेब सिरीज़ तांडव ले जार विवाद कार्टून "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FfPkwm3eRN-g",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4a2a25e9-331f-4472-a896-5bac7daace27"
      },
      "source": [
        "# make both length equal\n",
        "if len(fake_news) > len(true_news):\n",
        "  fake_news = fake_news.head(len(true_news))\n",
        "elif len(true_news) > len(fake_news):\n",
        "  true_news = true_news.head(len(fake_news))\n",
        "print(\"Length of fake news: \", len(fake_news))\n",
        "print(\"Length of true news: \", len(true_news))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Length of fake news:  760\n",
            "Length of true news:  760\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XX3HKYarAMQ9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "outputId": "d23713bb-7ed1-4577-d420-f4221b1bccc5"
      },
      "source": [
        "true_news['label']=1\n",
        "fake_news['label']=0\n",
        "news = pd.concat([fake_news, true_news])\n",
        "news.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>short_description</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>755</th>\n",
              "      <td>चालक कार रोक के बजाय चल जार रख ट्रैफ़िक पुलिसक...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>756</th>\n",
              "      <td>मिसाइल 400 किलोमीटर ज़्याद मार सक</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>757</th>\n",
              "      <td>आकर्षण केंद्र वो शक्तिशाल पहलवान जिनक संबंध प्...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>758</th>\n",
              "      <td>बिहार के जातिगत समीकरण बात लंब बीजेप हिंद पहचा...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>759</th>\n",
              "      <td>एनआईए 83 साल के फ़ादर स्टेन स्वाम गिरफ़्तार यू...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                     short_description  label\n",
              "755  चालक कार रोक के बजाय चल जार रख ट्रैफ़िक पुलिसक...      1\n",
              "756                 मिसाइल 400 किलोमीटर ज़्याद मार सक       1\n",
              "757  आकर्षण केंद्र वो शक्तिशाल पहलवान जिनक संबंध प्...      1\n",
              "758  बिहार के जातिगत समीकरण बात लंब बीजेप हिंद पहचा...      1\n",
              "759  एनआईए 83 साल के फ़ादर स्टेन स्वाम गिरफ़्तार यू...      1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vX7op-NmayPd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1638b1dd-50e9-432b-c81f-7f96c8bc4d80"
      },
      "source": [
        "import tensorflow as tf\n",
        "# Get the GPU device name.\n",
        "device_name = tf.test.gpu_device_name()\n",
        "# The device name should look like the following:\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yRM9eh-JvEeM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eb9ec97f-5fdb-4355-ac08-898199b2bc06"
      },
      "source": [
        "import torch\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla T4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-sqndMMF-cxl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 723
        },
        "outputId": "51d065e4-2652-40af-b1ab-eddf0ef89750"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.9.1-py3-none-any.whl (2.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.6 MB 9.6 MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 61.1 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub==0.0.12\n",
            "  Downloading huggingface_hub-0.0.12-py3-none-any.whl (37 kB)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 57.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (21.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.6.1)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 58.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.12->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.5.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Installing collected packages: tokenizers, sacremoses, pyyaml, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.0.12 pyyaml-5.4.1 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.9.1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "yaml"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RA7CM5SD-on4"
      },
      "source": [
        "#split train test\n",
        "from sklearn.model_selection import train_test_split\n",
        "train, test, labels, y_test = train_test_split(news, news['label'], test_size=0.2, stratify=news['label'],random_state=123, shuffle=True)\n",
        "\n",
        "sentences = train.short_description.values\n",
        "labels=labels.values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tzdW-2mi_TdK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 229,
          "referenced_widgets": [
            "daab1c4f4f754a58baeea9b09ab21c36",
            "cffca6798e8049ef82c103e846682484",
            "75520c6f09b744ee99a4251db43c29c5",
            "b70d73ee60e2477086dab11a69c93d69",
            "6b73ad83d59c4e24be113ee7d8054f5a",
            "9951a26d574443508cef4e52cfc3d6a4",
            "70a1a7f5d5f94c3f9506c0bec588f177",
            "214c22368222442d8f246bebddc442db",
            "ec656a3f08634484a792cea9818ccbe5",
            "55056ea58e2547f5891b5452b59a18ca",
            "c82adb7f882948c0baae1e1bd8179adf",
            "bd89214d587448bbbe02d05cbfeca515",
            "c199cf39dc464f998bf8409cb4990424",
            "f62c0fbd316a48179dc4ca41d4bc5242",
            "56479aba064b4f2b96551f82de337e64",
            "d2364aa2765943658eb7218149ac1a6e",
            "aa624cc045cf4dc8a2436f4b125d24b2",
            "a784c1deec11401a829e367dd399c579",
            "a0138461eec448739cbcaf0a264731d0",
            "61a2141d42c34ddd8e8b01055697ec18",
            "2fa7d3288a6445f2a78c0882e6debe41",
            "a9ccead30acf4043bede1b48579d1b5e",
            "34975406f3f54d1e9ec30176e6b78e30",
            "07a184cd77a14834b118a54eb594124d",
            "a0b4a7a05f3c4eb99076068d6a738f22",
            "1211462e2faa444e8883135a5012c2f6",
            "081451d244ab44f790eb7999c14fa3d0",
            "b5225cf1534446298b9c8c50ded292b9",
            "651a8342c0e046aba6bd2d864ac46179",
            "37cc7d69e67a4a968102041421d58692",
            "f8a78a6ce88f405a9df4b01f79074d81",
            "21cc59f9714240eb8673ba0d19308353"
          ]
        },
        "outputId": "21e34ed7-fc6b-4c1d-805d-f9bedbff04fd"
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...')\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased', do_lower_case=True) #bert-base-uncased"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading BERT tokenizer...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "daab1c4f4f754a58baeea9b09ab21c36",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=871891.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ec656a3f08634484a792cea9818ccbe5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "aa624cc045cf4dc8a2436f4b125d24b2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1715180.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a0b4a7a05f3c4eb99076068d6a738f22",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=625.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tReoDkLD_l1_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8a1264c3-dd9f-4582-ccbb-216569b71e29"
      },
      "source": [
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "for sent in sentences:\n",
        "    # `encode` will:  (1) Tokenize the sentence.   (2) Prepend the `[CLS]` token to the start.  (3) Append the `[SEP]` token to the end. (4) Map tokens to their IDs.\n",
        "    encoded_sent = tokenizer.encode(\n",
        "                        sent,                      \n",
        "                        add_special_tokens = True,  # Add '[CLS]' and '[SEP]'\n",
        "                        # max_length = 64,          # Truncate all sentences.\n",
        "                        #return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "    \n",
        "    # Add the encoded sentence to the list.\n",
        "    input_ids.append(encoded_sent)\n",
        "# Print sentence 0, now as a list of IDs.\n",
        "print('Original: ', sentences[0])\n",
        "print('Token IDs:', input_ids[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original:  विपक्ष कांग्रेस पार्ट सरकार स्थित स्पष्ट कह दूसर ओर सरकार कह स्थित उसक नज़र \n",
            "Token IDs: [101, 53836, 94464, 22965, 95490, 567, 17277, 13043, 39425, 24573, 95129, 547, 16080, 52254, 44239, 39425, 547, 16080, 24573, 34731, 566, 58246, 102]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "KFE2afr9XT34",
        "outputId": "586c294a-0709-424c-c620-1f36675b8645"
      },
      "source": [
        "seq_len = [len(sen) for sen in input_ids]\n",
        "pd.Series(seq_len).hist(bins = 30)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f8c80a55590>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAVc0lEQVR4nO3df5DcdX3H8edLohg4m4DRnTTJ9GKNOJgTJFvEwXH2oNUgDuEPhoahmjhxbtqixRpHgs6U9g+msW2kOLbOpJKCI8OJESUNomLkytApYA6RAwIlhYC5CYmUEHuYYk/f/WO/qctxyf747n5395PXYyZz+/18v7uf1y3fe/PZz35/KCIwM7O0vKbbAczMrP1c3M3MEuTibmaWIBd3M7MEubibmSVoTrcDACxYsCAGBwcL7fOll17i5JNPLrTPdnDu4vVrducuVjdyj4+PPx8Rb5ptXU8U98HBQXbu3Flon2NjY1QqlUL7bAfnLl6/ZnfuYnUjt6RnjrbO0zJmZglycTczS5CLu5lZglzczcwS5OJuZpYgF3czswS5uJuZJcjF3cwsQS7uZmYJ6okzVC09gxvueMXy+qFp1s5oA9iz8cKiIpkdVzxyNzNLkIu7mVmCXNzNzBLk4m5mlqC6xV3SFkkHJD0yo/0Tkh6X9Kikv6lpv1rSbklPSPpAJ0KbmdmxNXK0zI3Al4CvHmmQNAysAs6IiJclvTlrPx1YDbwD+G3gB5LeFhG/andwMzM7uroj94i4B3hhRvOfABsj4uVsmwNZ+ypgNCJejoingd3A2W3Ma2ZmDVBE1N9IGgS2R8TybPkh4HZgJfA/wKcj4keSvgTcFxFfy7a7AbgzIrbO8pojwAhAqVRaMTo62pZfqFFTU1MMDAwU2mc79EvuiclDr1guzYX9h1+93dCieQUlal2/vOczOXexupF7eHh4PCLKs61r9SSmOcCpwDnA7wG3SnpLMy8QEZuBzQDlcjmKvj2Vb+XVWTNPWFo/NM2miVfvbnsurxSUqHX98p7P5NzF6rXcrR4tsxe4LaoeAH4NLAAmgSU12y3O2szMrECtFvdvA8MAkt4GvA54HtgGrJZ0oqSlwDLggXYENTOzxtWdlpF0C1ABFkjaC1wDbAG2ZIdH/hJYE9XJ+0cl3Qo8BkwDV/hIGTOz4tUt7hFx2VFW/dFRtr8WuDZPKDMzy8dnqJqZJcjF3cwsQS7uZmYJcnE3M0uQi7uZWYJc3M3MEuTibmaWIBd3M7MEubibmSXIxd3MLEEu7mZmCXJxNzNLkIu7mVmCXNzNzBLk4m5mliAXdzOzBNUt7pK2SDqQ3XVp5rr1kkLSgmxZkr4oabekhyWd1YnQZmZ2bI2M3G8EVs5slLQEeD/wbE3zBVTvm7oMGAG+nD+imZk1q25xj4h7gBdmWXUd8BkgatpWAV+NqvuA+ZIWtiWpmZk1TNX7WtfZSBoEtkfE8mx5FXBeRFwpaQ9QjojnJW0HNkbEvdl2O4CrImLnLK85QnV0T6lUWjE6Otqe36hBU1NTDAwMFNpnO/RL7onJQ69YLs2F/Ydfvd3QonkFJWpdv7znMzl3sbqRe3h4eDwiyrOtq3uD7JkknQR8luqUTMsiYjOwGaBcLkelUsnzck0bGxuj6D7boV9yr91wxyuW1w9Ns2ni1bvbnssrBSVqXb+85zM5d7F6LXfTxR34XWAp8BNJAIuBByWdDUwCS2q2XZy1mZlZgZo+FDIiJiLizRExGBGDwF7grIh4DtgGfCQ7auYc4FBE7GtvZDMzq6eRQyFvAf4dOE3SXknrjrH5d4CngN3APwF/2paUZmbWlLrTMhFxWZ31gzWPA7gifywzM8vDZ6iamSXIxd3MLEEu7mZmCWrlUEizthmccTz80ezZeGGHk5ilxSN3M7MEubibmSXIxd3MLEGec7e+4Ll5s+Z45G5mliAXdzOzBLm4m5klyMXdzCxBLu5mZglycTczS5CLu5lZglzczcwS1MidmLZIOiDpkZq2v5X0uKSHJX1L0vyadVdL2i3pCUkf6FRwMzM7ukZG7jcCK2e03QUsj4h3Av8BXA0g6XRgNfCO7Dn/KOmEtqU1M7OG1C3uEXEP8MKMtu9HxHS2eB+wOHu8ChiNiJcj4mmq91I9u415zcysAare9rTORtIgsD0ils+y7l+Ar0fE1yR9CbgvIr6WrbsBuDMits7yvBFgBKBUKq0YHR3N83s0bWpqioGBgUL7bId+yT0xeegVy6W5sP9w5/sdWjSv7a/ZL+/5TM5drG7kHh4eHo+I8mzrcl04TNLngGng5mafGxGbgc0A5XI5KpVKnihNGxsbo+g+26Ffcq+dcaGv9UPTbJro/HXq9lxeaftr9st7PpNzF6vXcrf81yZpLfAh4Pz4zfB/ElhSs9nirM3MzArU0qGQklYCnwEuiohf1KzaBqyWdKKkpcAy4IH8Mc3MrBl1R+6SbgEqwAJJe4FrqB4dcyJwlySozrP/cUQ8KulW4DGq0zVXRMSvOhXezMxmV7e4R8RlszTfcIztrwWuzRPKzMzy8RmqZmYJ8m32LCm+HZ9ZlUfuZmYJcnE3M0uQi7uZWYJc3M3MEuTibmaWIBd3M7MEubibmSXIxd3MLEE+icma0uhJQmbWXR65m5klyMXdzCxBLu5mZglycTczS5CLu5lZguoWd0lbJB2Q9EhN26mS7pL0ZPbzlKxdkr4oabekhyWd1cnwZmY2u0ZG7jcCK2e0bQB2RMQyYEe2DHAB1fumLgNGgC+3J6aZmTWjbnGPiHuAF2Y0rwJuyh7fBFxc0/7VqLoPmC9pYbvCmplZYxQR9TeSBoHtEbE8W34xIuZnjwUcjIj5krYDGyPi3mzdDuCqiNg5y2uOUB3dUyqVVoyOjrbnN2rQ1NQUAwMDhfbZDt3OPTF5qKXnlebC/sNtDpPD0KJ5DW/b7fe8Vc5drG7kHh4eHo+I8mzrcp+hGhEhqf7/IV79vM3AZoByuRyVSiVvlKaMjY1RdJ/t0O3ca1s8Q3X90DSbJnrnhOg9l1ca3rbb73mrnLtYvZa71aNl9h+Zbsl+HsjaJ4ElNdstztrMzKxArRb3bcCa7PEa4Paa9o9kR82cAxyKiH05M5qZWZPqfk6WdAtQARZI2gtcA2wEbpW0DngGuDTb/DvAB4HdwC+Aj3Ygs5mZ1VG3uEfEZUdZdf4s2wZwRd5QZmaWj89QNTNLkIu7mVmCXNzNzBLk4m5mliAXdzOzBLm4m5klqHfOBzcrUDM3+r5x5ckdTGLWGR65m5klyMXdzCxBLu5mZglycTczS5CLu5lZglzczcwS5OJuZpYgF3czswTlOolJ0p8DHwMCmKB6c46FwCjwRmAc+HBE/DJnTuuwZk7qMbPe1/LIXdIi4M+AckQsB04AVgOfB66LiLcCB4F17QhqZmaNyzstMweYK2kOcBKwDzgP2Jqtvwm4OGcfZmbWpJaLe0RMAn8HPEu1qB+iOg3zYkRMZ5vtBRblDWlmZs1R9banLTxROgX4JvCHwIvAN6iO2P8ym5JB0hLgzmzaZubzR4ARgFKptGJ0dLSlHK2amppiYGCg0D7boVO5JyYPtf01a5Xmwv7DHe2iY5bOO8H7SoGcu3HDw8PjEVGebV2eL1R/H3g6In4GIOk24FxgvqQ52eh9MTA525MjYjOwGaBcLkelUskRpXljY2MU3Wc7dCr32g5/obp+aJpNE/15EdL1Q9Nsuvelutvt2XhhAWka5328WL2WO8+c+7PAOZJOkiTgfOAx4G7gkmybNcDt+SKamVmz8sy53091GuZBqodBvobqSPwq4FOSdlM9HPKGNuQ0M7Mm5PqcHBHXANfMaH4KODvP65qZWT4+Q9XMLEEu7mZmCXJxNzNLkIu7mVmCXNzNzBLk4m5mliAXdzOzBLm4m5klyMXdzCxBLu5mZglycTczS5CLu5lZglzczcwS5OJuZpYgF3czswS5uJuZJShXcZc0X9JWSY9L2iXpPZJOlXSXpCezn6e0K6yZmTUm78j9euC7EfF24AxgF7AB2BERy4Ad2bKZmRWo5eIuaR7wPrJ7pEbELyPiRWAVcFO22U3AxXlDmplZcxQRrT1ROpPqDbEfozpqHweuBCYjYn62jYCDR5ZnPH8EGAEolUorRkdHW8rRqqmpKQYGBgrtsx06lXti8lDbX7NWaS7sP9zRLjqm0exDi+Z1PkwTvI8Xqxu5h4eHxyOiPNu6PMW9DNwHnBsR90u6Hvg58InaYi7pYEQcc969XC7Hzp07W8rRqrGxMSqVSqF9tkOncg9uuKPtr1lr/dA0myZy3Y+9axrNvmfjhQWkaZz38WJ1I7ekoxb3PHPue4G9EXF/trwVOAvYL2lh1vFC4ECOPszMrAUtF/eIeA74qaTTsqbzqU7RbAPWZG1rgNtzJTQzs6bl/Zz8CeBmSa8DngI+SvV/GLdKWgc8A1yasw8zM2tSruIeEQ8Bs833nJ/ndc3MLB+foWpmliAXdzOzBLm4m5klyMXdzCxBLu5mZglycTczS5CLu5lZglzczcwS5OJuZpag/rxMnzWk01d6NLPe5ZG7mVmCXNzNzBLkaZke0sg0yvqhaSqdj2Jmfc4jdzOzBLm4m5klyMXdzCxBuYu7pBMk/VjS9mx5qaT7Je2W9PXsLk1mZlagdozcrwR21Sx/HrguIt4KHATWtaEPMzNrQq7iLmkxcCHwlWxZwHnA1myTm4CL8/RhZmbNU0S0/mRpK/DXwBuATwNrgfuyUTuSlgB3RsTyWZ47AowAlEqlFaOjoy3naMXU1BQDAwOF9lnPxOShutuU5sKbT53XttcrSmku7D/c7RStaTT70KLG/rsUpRf38UY4d+OGh4fHI2K2+1i3fpy7pA8BByJiXFKl2edHxGZgM0C5XI5KpemXyGVsbIyi+6xnbYPHuV/aYO5GXq8o64em2TTRn6dVNJp9z+WVzodpQi/u441w7vbI89d2LnCRpA8Crwd+C7gemC9pTkRMA4uByfwxzcysGS3PuUfE1RGxOCIGgdXADyPicuBu4JJsszXA7blTmplZUzpxnPtVwKck7QbeCNzQgT7MzOwY2jIJGhFjwFj2+Cng7Ha8rpmZtcZnqJqZJcjF3cwsQS7uZmYJcnE3M0tQf55VcpzzvVHNrB6P3M3MEuTibmaWIE/LmBWs0Wm1PRsv7HASS5mLu1mb+LsQ6yWeljEzS5CLu5lZglzczcwS5OJuZpYgF3czswS5uJuZJcjF3cwsQS0Xd0lLJN0t6TFJj0q6Mms/VdJdkp7Mfp7SvrhmZtaIPCP3aWB9RJwOnANcIel0YAOwIyKWATuyZTMzK1DLZ6hGxD5gX/b4vyXtAhYBq4BKttlNVG+/d1WulGZ2VEc7M3b90DRrWzhr1pc9SIMiIv+LSIPAPcBy4NmImJ+1Czh4ZHnGc0aAEYBSqbRidHQ0d45mTE1NMTAwUGif9UxMHqq7TWku7D9cQJg269fc0L3sQ4vmNbTd0fabVnM32m+n9OLfZiO6kXt4eHg8Isqzrctd3CUNAP8KXBsRt0l6sbaYSzoYEcecdy+Xy7Fz585cOZo1NjZGpVIptM96Grk2yfqhaTZN9N8lgfo1N3Qve6Mj6GON3FvJ3e2Rey/+bTaiG7klHbW459pjJb0W+CZwc0TcljXvl7QwIvZJWggcyNOH2fHKFyKzPPIcLSPgBmBXRHyhZtU2YE32eA1we+vxzMysFXlG7ucCHwYmJD2UtX0W2AjcKmkd8Axwab6I/c8jMDMrWp6jZe4FdJTV57f6umZmlp/PUDUzS5CLu5lZglzczcwS5OJuZpYgF3czswS5uJuZJcjF3cwsQS7uZmYJ6s8rOZlZT2j32dfdvmhZSo6r4l67Ix7rWtfewcys33laxswsQcfVyL3dfEEwM+tVHrmbmSXII3czewV/Ik2Di/ssvHObWb9zcTeznjG44Y5jHsl2hI9oq69jxV3SSuB64ATgKxGxsVN9mdnxpdFP13lvMt7q6/WCjnyhKukE4B+AC4DTgcsknd6JvszM7NU6NXI/G9gdEU8BSBoFVgGPtbsjz4+b2dF0sz50+9OAIqL9LypdAqyMiI9lyx8G3h0RH6/ZZgQYyRZPA55oe5BjWwA8X3Cf7eDcxevX7M5drG7k/p2IeNNsK7r2hWpEbAY2d6t/STsjotyt/lvl3MXr1+zOXaxey92pk5gmgSU1y4uzNjMzK0CnivuPgGWSlkp6HbAa2NahvszMbIaOTMtExLSkjwPfo3oo5JaIeLQTfeXQtSmhnJy7eP2a3bmL1VO5O/KFqpmZdZcvHGZmliAXdzOzBB0XxV3SFkkHJD1S03aqpLskPZn9PKWbGWcjaYmkuyU9JulRSVdm7T2dXdLrJT0g6SdZ7r/K2pdKul/Sbklfz75s7zmSTpD0Y0nbs+Wezy1pj6QJSQ9J2pm19fR+AiBpvqStkh6XtEvSe3o9t6TTsvf5yL+fS/pkr+U+Loo7cCOwckbbBmBHRCwDdmTLvWYaWB8RpwPnAFdkl3Ho9ewvA+dFxBnAmcBKSecAnweui4i3AgeBdV3MeCxXArtqlvsl93BEnFlzrHWv7ydQvf7UdyPi7cAZVN/3ns4dEU9k7/OZwArgF8C36LXcEXFc/AMGgUdqlp8AFmaPFwJPdDtjA7/D7cAf9FN24CTgQeDdVM/em5O1vwf4XrfzzZJ3MdU/zPOA7YD6JPceYMGMtp7eT4B5wNNkB3b0S+4ZWd8P/Fsv5j5eRu6zKUXEvuzxc0Cpm2HqkTQIvAu4nz7Ink1tPAQcAO4C/hN4MSKms032Aou6le8Y/h74DPDrbPmN9EfuAL4vaTy7tAf0/n6yFPgZ8M/ZNNhXJJ1M7+eutRq4JXvcU7mP5+L+/6L6v9qePSZU0gDwTeCTEfHz2nW9mj0ifhXVj62LqV5I7u1djlSXpA8BByJivNtZWvDeiDiL6pVYr5D0vtqVPbqfzAHOAr4cEe8CXmLGVEaP5gYg++7lIuAbM9f1Qu7jubjvl7QQIPt5oMt5ZiXptVQL+80RcVvW3BfZASLiReBuqtMZ8yUdOXGuFy9JcS5wkaQ9wCjVqZnr6f3cRMRk9vMA1fnfs+n9/WQvsDci7s+Wt1It9r2e+4gLgAcjYn+23FO5j+fivg1Ykz1eQ3U+u6dIEnADsCsivlCzqqezS3qTpPnZ47lUvyfYRbXIX5Jt1nO5I+LqiFgcEYNUP27/MCIup8dzSzpZ0huOPKY6D/wIPb6fRMRzwE8lnZY1nU/1suA9nbvGZfxmSgZ6LXe3v5Ao6EuPW4B9wP9SHS2sozqXugN4EvgBcGq3c86S+71UP9o9DDyU/ftgr2cH3gn8OMv9CPAXWftbgAeA3VQ/yp7Y7azH+B0qwPZ+yJ3l+0n271Hgc1l7T+8nWcYzgZ3ZvvJt4JQ+yX0y8F/AvJq2nsrtyw+YmSXoeJ6WMTNLlou7mVmCXNzNzBLk4m5mliAXdzOzBLm4m5klyMXdzCxB/wekAXI7dMWSzQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uNTTL_5rAPjA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "246d5550-71b6-40f0-93cb-ddad74ac51d9"
      },
      "source": [
        "print('Max sentence length: ', max([len(sen) for sen in input_ids]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Max sentence length:  73\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IpQWP4NDAaMH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "db86c594-87c2-43a7-fc56-d259b95f65dd"
      },
      "source": [
        "# We'll borrow the `pad_sequences` utility function to do padding.\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "# Set the maximum sequence length as 128 , as it should be slightly above 105\n",
        "MAX_LEN = 96\n",
        "print('\\nPadding/truncating all sentences to %d values...' % MAX_LEN)\n",
        "print('\\nPadding token: \"{:}\", ID: {:}'.format(tokenizer.pad_token, tokenizer.pad_token_id))\n",
        "# Pad our input tokens with value 0.\n",
        "# \"post\" indicates that we want to pad and truncate at the end of the sequence, as opposed to the beginning.\n",
        "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", \n",
        "                          value=0, truncating=\"post\", padding=\"post\")\n",
        "print('\\Done.')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Padding/truncating all sentences to 96 values...\n",
            "\n",
            "Padding token: \"[PAD]\", ID: 0\n",
            "\\Done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3T74PWbmAoz1"
      },
      "source": [
        "# Create attention masks\n",
        "attention_masks = []\n",
        "for sent in input_ids:\n",
        "    \n",
        "    # Create the attention mask. If a token ID is 0, then it's padding, set the mask to 0. If a token ID is > 0, then it's a real token, set the mask to 1.\n",
        "    att_mask = [int(token_id > 0) for token_id in sent]\n",
        "    \n",
        "    # Store the attention mask for this sentence.\n",
        "    attention_masks.append(att_mask)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NbTTRovIA2DG"
      },
      "source": [
        "# Use train_test_split to split our data into train and validation sets for training, 10% validation\n",
        "from sklearn.model_selection import train_test_split\n",
        "train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels, \n",
        "                                                            random_state=123, test_size=0.1)\n",
        "# Do the same for the masks.\n",
        "train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,\n",
        "                                             random_state=123, test_size=0.1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SiZaCWjjBQe3"
      },
      "source": [
        "# Convert all inputs and labels into torch tensors, the required datatype for our model.\n",
        "train_inputs = torch.tensor(train_inputs)\n",
        "validation_inputs = torch.tensor(validation_inputs)\n",
        "train_labels = torch.tensor(train_labels)\n",
        "validation_labels = torch.tensor(validation_labels)\n",
        "train_masks = torch.tensor(train_masks)\n",
        "validation_masks = torch.tensor(validation_masks)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6UVcwHGfBVZI"
      },
      "source": [
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "# The DataLoader needs to know our batch size for training, so we specify it here.\n",
        "batch_size = 32\n",
        "# Create the DataLoader for our training set.\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "# Create the DataLoader for our validation set.\n",
        "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ciRxZNxqBfhD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "eb1f490f8daa41e4986e47180cc1dd44",
            "1742885195ec489b89a2178156bc91fc",
            "863693a307234b5291ad057a30517239",
            "93e1680a49834dc9bd65f14cb50c2fd5",
            "07e1e46c84664b32bf0fdba1b890d94c",
            "caac574908c441bcbc57a4662a0a6c4f",
            "ede467e203d948f3bf3cc3ced057649a",
            "f9243e449bb447d7b2201a456ef6a828"
          ]
        },
        "outputId": "c2d4b706-8c6d-4961-991e-24cdb22f7ffb"
      },
      "source": [
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "# Load BertForSequenceClassification, the pretrained BERT model with a single linear classification layer on top. \n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-multilingual-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "    num_labels = 2, # The number of output labels--2 for binary classification.\n",
        "    output_attentions = False, # Whether the model returns attentions weights.\n",
        "    output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        ")\n",
        "# Tell pytorch to run this model on the GPU.\n",
        "model.cuda()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "eb1f490f8daa41e4986e47180cc1dd44",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=672271273.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-multilingual-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-multilingual-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(105879, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nHy65PAhBp8F",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fe5e7f0a-0dee-4d7b-b399-467257412f35"
      },
      "source": [
        "# Get all of the model's parameters as a list of tuples.\n",
        "params = list(model.named_parameters())\n",
        "print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
        "print('==== Embedding Layer ====\\n')\n",
        "for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "print('\\n==== First Transformer ====\\n')\n",
        "for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "print('\\n==== Output Layer ====\\n')\n",
        "for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The BERT model has 201 different named parameters.\n",
            "\n",
            "==== Embedding Layer ====\n",
            "\n",
            "bert.embeddings.word_embeddings.weight                  (105879, 768)\n",
            "bert.embeddings.position_embeddings.weight                (512, 768)\n",
            "bert.embeddings.token_type_embeddings.weight                (2, 768)\n",
            "bert.embeddings.LayerNorm.weight                              (768,)\n",
            "bert.embeddings.LayerNorm.bias                                (768,)\n",
            "\n",
            "==== First Transformer ====\n",
            "\n",
            "bert.encoder.layer.0.attention.self.query.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.query.bias                (768,)\n",
            "bert.encoder.layer.0.attention.self.key.weight            (768, 768)\n",
            "bert.encoder.layer.0.attention.self.key.bias                  (768,)\n",
            "bert.encoder.layer.0.attention.self.value.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.value.bias                (768,)\n",
            "bert.encoder.layer.0.attention.output.dense.weight        (768, 768)\n",
            "bert.encoder.layer.0.attention.output.dense.bias              (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.weight        (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.bias          (768,)\n",
            "bert.encoder.layer.0.intermediate.dense.weight           (3072, 768)\n",
            "bert.encoder.layer.0.intermediate.dense.bias                 (3072,)\n",
            "bert.encoder.layer.0.output.dense.weight                 (768, 3072)\n",
            "bert.encoder.layer.0.output.dense.bias                        (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.weight                  (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.bias                    (768,)\n",
            "\n",
            "==== Output Layer ====\n",
            "\n",
            "bert.pooler.dense.weight                                  (768, 768)\n",
            "bert.pooler.dense.bias                                        (768,)\n",
            "classifier.weight                                           (2, 768)\n",
            "classifier.bias                                                 (2,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nk0I-UxmBy1i"
      },
      "source": [
        "# Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 2e-5, # args.learning_rate \n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "epochs = 5\n",
        "# Total number of training steps is number of batches * number of epochs.\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0, \n",
        "                                            num_training_steps = total_steps)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c8NakeZ7B_bt"
      },
      "source": [
        "import numpy as np\n",
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kjYi3CpICQcm"
      },
      "source": [
        "import time\n",
        "import datetime\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yQy5MVUICTbE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4bd3454b-7b03-4bfb-e6ce-128ba216948c"
      },
      "source": [
        "import random\n",
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "# Set the seed value all over the place to make this reproducible.\n",
        "seed_val = 42\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "# Store the average loss after each epoch so we can plot them.\n",
        "loss_values = []\n",
        "start=time.time()\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "    t0 = time.time()\n",
        "    total_loss = 0\n",
        "    model.train()\n",
        "    # For each batch of training data...\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "        # Progress update every 40 batches.\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "        # Unpack this training batch from our dataloader. \n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids \n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "        # Always clear any previously calculated gradients before performing a backward pass.\n",
        "\n",
        "        model.zero_grad()        \n",
        "        # Perform a forward pass .This will return the loss .\n",
        "        outputs = model(b_input_ids, \n",
        "                    token_type_ids=None, \n",
        "                    attention_mask=b_input_mask, \n",
        "                    labels=b_labels)\n",
        "        \n",
        "        # The call to `model` always returns a tuple, so we need to pull the loss value out of the tuple.\n",
        "        loss = outputs[0]\n",
        "        # Accumulate the training loss over all of the batches so that we can calculate the average loss at the end. `loss` is a Tensor containing a single value\n",
        "        total_loss += loss.item()\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "        # Clip the norm of the gradients to 1.0. This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "        # Update parameters and take a step using the computed gradient.\n",
        "        optimizer.step()\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "    # Calculate the average loss over the training data.\n",
        "    avg_train_loss = total_loss / len(train_dataloader)            \n",
        "    \n",
        "    # Store the loss value for plotting the learning curve.\n",
        "    loss_values.append(avg_train_loss)\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(format_time(time.time() - t0)))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on our validation set.\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "    t0 = time.time()\n",
        "    # Put the model in evaluation mode--the dropout layers behave differently during evaluation.\n",
        "    model.eval()\n",
        "    eval_loss, eval_accuracy = 0, 0\n",
        "    nb_eval_steps, nb_eval_examples = 0, 0\n",
        "    for batch in validation_dataloader:\n",
        "        \n",
        "        # Add batch to GPU\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "        \n",
        "        # Unpack the inputs from our dataloader\n",
        "        b_input_ids, b_input_mask, b_labels = batch\n",
        "        \n",
        "        # Telling the model not to compute or store gradients, saving memory and  speeding up validation\n",
        "        with torch.no_grad():        \n",
        "            # Forward pass, calculate logit predictions. This will return the logits rather than the loss because we have not provided labels.\n",
        "            # token_type_ids is the same as the \"segment ids\"\n",
        "            outputs = model(b_input_ids, \n",
        "                            token_type_ids=None, \n",
        "                            attention_mask=b_input_mask)\n",
        "        \n",
        "        # Get the \"logits\" output by the model. \n",
        "        logits = outputs[0]\n",
        "        # Move logits and labels to CPU\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "        \n",
        "        # Calculate the accuracy for this batch of test sentences.\n",
        "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
        "        \n",
        "        # Accumulate the total accuracy.\n",
        "        eval_accuracy += tmp_eval_accuracy\n",
        "        # Track the number of batches\n",
        "        nb_eval_steps += 1\n",
        "    # Report the final accuracy for this validation run.\n",
        "    print(\"  Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n",
        "    print(\"  Validation took: {:}\".format(format_time(time.time() - t0)))\n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n",
        "end=time.time()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 5 ========\n",
            "Training...\n",
            "\n",
            "  Average training loss: 0.49\n",
            "  Training epcoh took: 0:00:19\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.92\n",
            "  Validation took: 0:00:01\n",
            "\n",
            "======== Epoch 2 / 5 ========\n",
            "Training...\n",
            "\n",
            "  Average training loss: 0.15\n",
            "  Training epcoh took: 0:00:18\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.92\n",
            "  Validation took: 0:00:01\n",
            "\n",
            "======== Epoch 3 / 5 ========\n",
            "Training...\n",
            "\n",
            "  Average training loss: 0.09\n",
            "  Training epcoh took: 0:00:19\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.92\n",
            "  Validation took: 0:00:01\n",
            "\n",
            "======== Epoch 4 / 5 ========\n",
            "Training...\n",
            "\n",
            "  Average training loss: 0.04\n",
            "  Training epcoh took: 0:00:19\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.96\n",
            "  Validation took: 0:00:01\n",
            "\n",
            "======== Epoch 5 / 5 ========\n",
            "Training...\n",
            "\n",
            "  Average training loss: 0.03\n",
            "  Training epcoh took: 0:00:19\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.93\n",
            "  Validation took: 0:00:01\n",
            "\n",
            "Training complete!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oHsQYdLHCntm",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "outputId": "55dab15f-aa3b-41f9-fc8d-82cb2aac6ef2"
      },
      "source": [
        "import plotly.express as px\n",
        "f = pd.DataFrame(loss_values)\n",
        "f.columns=['Loss']\n",
        "fig = px.line(f, x=f.index, y=f.Loss)\n",
        "fig.update_layout(title='Training loss of the Model',\n",
        "                   xaxis_title='Epoch',\n",
        "                   yaxis_title='Loss')\n",
        "fig.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"9e3fe9c1-06ed-44e5-ae65-709d8e9b4a46\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"9e3fe9c1-06ed-44e5-ae65-709d8e9b4a46\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        '9e3fe9c1-06ed-44e5-ae65-709d8e9b4a46',\n",
              "                        [{\"hoverlabel\": {\"namelength\": 0}, \"hovertemplate\": \"index=%{x}<br>Loss=%{y}\", \"legendgroup\": \"\", \"line\": {\"color\": \"#636efa\", \"dash\": \"solid\"}, \"mode\": \"lines\", \"name\": \"\", \"showlegend\": false, \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4], \"xaxis\": \"x\", \"y\": [0.48961302606122836, 0.14934865050017834, 0.0915987444509353, 0.042993506070758615, 0.02933444633547749], \"yaxis\": \"y\"}],\n",
              "                        {\"legend\": {\"tracegroupgap\": 0}, \"margin\": {\"t\": 60}, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Training loss of the Model\"}, \"xaxis\": {\"anchor\": \"y\", \"domain\": [0.0, 1.0], \"title\": {\"text\": \"Epoch\"}}, \"yaxis\": {\"anchor\": \"x\", \"domain\": [0.0, 1.0], \"title\": {\"text\": \"Loss\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('9e3fe9c1-06ed-44e5-ae65-709d8e9b4a46');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "micZoH6SDJiF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0451c284-735c-42c9-f3bf-276944d2caed"
      },
      "source": [
        "sentences = test.short_description.values\n",
        "labels = y_test.values\n",
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "for sent in sentences:\n",
        "  \n",
        "    encoded_sent = tokenizer.encode(\n",
        "                        sent,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = 256\n",
        "                   )\n",
        "    \n",
        "    input_ids.append(encoded_sent)\n",
        "# Pad our input tokens\n",
        "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, \n",
        "                          dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "# Create attention masks\n",
        "attention_masks = []\n",
        "# Create a mask of 1s for each token followed by 0s for padding\n",
        "for seq in input_ids:\n",
        "  seq_mask = [float(i>0) for i in seq]\n",
        "  attention_masks.append(seq_mask) \n",
        "# Convert to tensors.\n",
        "prediction_inputs = torch.tensor(input_ids)\n",
        "prediction_masks = torch.tensor(attention_masks)\n",
        "prediction_labels = torch.tensor(labels)\n",
        "# Set the batch size.  \n",
        "batch_size = 32  \n",
        "# Create the DataLoader.\n",
        "prediction_data = TensorDataset(prediction_inputs, prediction_masks, prediction_labels)\n",
        "prediction_sampler = SequentialSampler(prediction_data)\n",
        "prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "02d9i2HMWnko",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c2d68dea-57d5-4331-c623-91fcc06a27c1"
      },
      "source": [
        "print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs)))\n",
        "# Put model in evaluation mode\n",
        "model.eval()\n",
        "\n",
        "predictions , true_labels = [], []\n",
        "# Predict \n",
        "for batch in prediction_dataloader:\n",
        "  # Add batch to GPU\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  \n",
        "  # Unpack the inputs from our dataloader\n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "  \n",
        "\n",
        "  with torch.no_grad():\n",
        "      # Forward pass, calculate logit predictions\n",
        "      outputs = model(b_input_ids, token_type_ids=None, \n",
        "                      attention_mask=b_input_mask)\n",
        "  logits = outputs[0]\n",
        "  # Move logits and labels to CPU\n",
        "  logits = logits.detach().cpu().numpy()\n",
        "  label_ids = b_labels.to('cpu').numpy()\n",
        "  \n",
        "  # Store predictions and true labels\n",
        "  predictions.append(logits)\n",
        "  true_labels.append(label_ids)\n",
        "print('DONE.')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predicting labels for 304 test sentences...\n",
            "DONE.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "22YTI3YBW0Bp"
      },
      "source": [
        "count=0\n",
        "guesses=[]\n",
        "actual=[]\n",
        "# For each input batch...\n",
        "for i in range(len(true_labels)):\n",
        "  \n",
        "  # The predictions for this batch are a 2-column ndarray (one column for \"0\" and one column for \"1\"). Pick the label with the highest value and turn this in to a list of 0s and 1s.\n",
        "  pred_labels_i = np.argmax(predictions[i], axis=1).flatten()\n",
        "  \n",
        "\n",
        "  #calculate accuracy\n",
        "  for j in range(len(pred_labels_i)):\n",
        "    if(pred_labels_i[j]==true_labels[i][j]):\n",
        "      count+=1\n",
        "    guesses.append(pred_labels_i[j])\n",
        "    actual.append(true_labels[i][j])\n",
        "    \n",
        "\n",
        "accuracy = count/len(y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BuXyhYvIXv6F",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "83f74cb1-3f49-4c64-f102-05e6d843c903"
      },
      "source": [
        "print(accuracy)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.9703947368421053\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eBICY6o7eK5_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "51d9cea2-7ca1-4ea3-f215-3aaa1da0e27a"
      },
      "source": [
        "from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score\n",
        "\n",
        "\n",
        "print(accuracy_score(actual, guesses))\n",
        "print(recall_score(actual, guesses))\n",
        "print(precision_score(actual, guesses))\n",
        "print(f1_score(actual, guesses))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.9703947368421053\n",
            "0.9605263157894737\n",
            "0.9798657718120806\n",
            "0.9700996677740865\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I2i-_YS3uaLV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8bf8cdfc-5f25-4d3b-b83d-32cc803b5950"
      },
      "source": [
        "print(end-start)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "96.57519102096558\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}