04_unsupervised_learning_language_models.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyODxhZxzQSt3PzHE3YqYSXn",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "48524f233dab40a1a8a4f49de10c741d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6dceb6e249274cd78400c5b135a32b27",
              "IPY_MODEL_b559ab6d7def43a8899b5c4a1db15214",
              "IPY_MODEL_8f1aebfa6daa494485040a5a1d2f711b"
            ],
            "layout": "IPY_MODEL_ae26b1205f61423b924f1cba03abed94"
          }
        },
        "6dceb6e249274cd78400c5b135a32b27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_193392a07f9743268eeb86e5d52dd2cf",
            "placeholder": "​",
            "style": "IPY_MODEL_48019fbf73814a58adf78b0c6099eda7",
            "value": "Downloading readme: 100%"
          }
        },
        "b559ab6d7def43a8899b5c4a1db15214": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b4c963851c8c465386ec03ca55ce8b71",
            "max": 10464,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2891c35063a546cb9667dc3deb382527",
            "value": 10464
          }
        },
        "8f1aebfa6daa494485040a5a1d2f711b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4a0f169deeab41208e68e7e5a6693eb8",
            "placeholder": "​",
            "style": "IPY_MODEL_8aa654b2c2634e1897a37a68861aab90",
            "value": " 10.5k/10.5k [00:00&lt;00:00, 614kB/s]"
          }
        },
        "ae26b1205f61423b924f1cba03abed94": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "193392a07f9743268eeb86e5d52dd2cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "48019fbf73814a58adf78b0c6099eda7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b4c963851c8c465386ec03ca55ce8b71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2891c35063a546cb9667dc3deb382527": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4a0f169deeab41208e68e7e5a6693eb8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8aa654b2c2634e1897a37a68861aab90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "34cdb78c2b164747ad86e1d6aa8a0702": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c05bfea913cc4309a8e2378d2f91fa0a",
              "IPY_MODEL_c172fdf108bc43a3a0f35851ca52640a",
              "IPY_MODEL_085ed8e03f094297aa087e8160b16141"
            ],
            "layout": "IPY_MODEL_589d0a459df64eadae77cc6d06128196"
          }
        },
        "c05bfea913cc4309a8e2378d2f91fa0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_413df2a280304bebbe5f1849adf70f41",
            "placeholder": "​",
            "style": "IPY_MODEL_cdbe10d96c534544b6857adf289ff8b9",
            "value": "Downloading data: 100%"
          }
        },
        "c172fdf108bc43a3a0f35851ca52640a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_657fa5fc13e541e29d5219bd0a8ae4db",
            "max": 732610,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bb1272540b9244cca0e9f5726cc597ba",
            "value": 732610
          }
        },
        "085ed8e03f094297aa087e8160b16141": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c337a8fbccf84fd182d70c013b837bbc",
            "placeholder": "​",
            "style": "IPY_MODEL_4c54e305a63e4261b0a4b916daaa3736",
            "value": " 733k/733k [00:01&lt;00:00, 642kB/s]"
          }
        },
        "589d0a459df64eadae77cc6d06128196": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "413df2a280304bebbe5f1849adf70f41": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cdbe10d96c534544b6857adf289ff8b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "657fa5fc13e541e29d5219bd0a8ae4db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb1272540b9244cca0e9f5726cc597ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c337a8fbccf84fd182d70c013b837bbc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c54e305a63e4261b0a4b916daaa3736": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d919be2767584000b248de99eaae4e30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d8543decb9ab4f7d8729c1672e102554",
              "IPY_MODEL_7405bc3b596c4b778fe3a22dac4229da",
              "IPY_MODEL_0635f88c0036409da976a84e26f54e17"
            ],
            "layout": "IPY_MODEL_9000b335d4174443b6fc509447b5ecb8"
          }
        },
        "d8543decb9ab4f7d8729c1672e102554": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b22b883fdbe44be8bccd1b161eed26f8",
            "placeholder": "​",
            "style": "IPY_MODEL_6bff9295fd5f4d8a9708c427704962f5",
            "value": "Downloading data: 100%"
          }
        },
        "7405bc3b596c4b778fe3a22dac4229da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a4acd9bcb8ff4bca874c944aa5060bcd",
            "max": 6357543,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6d2c67cf56b747178de36bad4d6c3a2f",
            "value": 6357543
          }
        },
        "0635f88c0036409da976a84e26f54e17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2372463cb8244ad28f321a1608ab3d17",
            "placeholder": "​",
            "style": "IPY_MODEL_4bd68037c5a344dd8ebe06572ab91302",
            "value": " 6.36M/6.36M [00:01&lt;00:00, 3.74MB/s]"
          }
        },
        "9000b335d4174443b6fc509447b5ecb8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b22b883fdbe44be8bccd1b161eed26f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6bff9295fd5f4d8a9708c427704962f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a4acd9bcb8ff4bca874c944aa5060bcd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6d2c67cf56b747178de36bad4d6c3a2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2372463cb8244ad28f321a1608ab3d17": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4bd68037c5a344dd8ebe06572ab91302": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "86f5aace1dc4454786cb1b92b9b72b99": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_52675d7edbbd4e3f8aad5f775f39adc8",
              "IPY_MODEL_4ba773bcb1b14821934728a39bef3efa",
              "IPY_MODEL_552d1789ceaa43ab867a7e78e440e878"
            ],
            "layout": "IPY_MODEL_db6507119fbc42c7996ddb48f63e3d6d"
          }
        },
        "52675d7edbbd4e3f8aad5f775f39adc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7d4bdf46f1f748ff94fa0748f8b829e3",
            "placeholder": "​",
            "style": "IPY_MODEL_bc21cfdbdef3452e8d7792e92ed38599",
            "value": "Downloading data: 100%"
          }
        },
        "4ba773bcb1b14821934728a39bef3efa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0c80b9fce8704a6ba3c38b4dc5732a08",
            "max": 657209,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_563e51df03804a10a06395755f616358",
            "value": 657209
          }
        },
        "552d1789ceaa43ab867a7e78e440e878": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3c6b76b3491e4eed977648ea15bbfcca",
            "placeholder": "​",
            "style": "IPY_MODEL_3865265fbaf0439e94b8d659c8536dbc",
            "value": " 657k/657k [00:00&lt;00:00, 1.90MB/s]"
          }
        },
        "db6507119fbc42c7996ddb48f63e3d6d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7d4bdf46f1f748ff94fa0748f8b829e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc21cfdbdef3452e8d7792e92ed38599": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0c80b9fce8704a6ba3c38b4dc5732a08": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "563e51df03804a10a06395755f616358": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3c6b76b3491e4eed977648ea15bbfcca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3865265fbaf0439e94b8d659c8536dbc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "68527ced1ee24ef8bfab018f26f6b252": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_51ee70fe9bf447149c74ce455fb9c284",
              "IPY_MODEL_f75fca6750484e24ad795f7b4a753f83",
              "IPY_MODEL_227235ae048447a38845149767a151ab"
            ],
            "layout": "IPY_MODEL_c034f0044f7540cc8888922247d07a42"
          }
        },
        "51ee70fe9bf447149c74ce455fb9c284": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_12e77da23c2e4124ae516be1fc3c16a4",
            "placeholder": "​",
            "style": "IPY_MODEL_2d5f72f265cc415ab410675401c7e0ef",
            "value": "Generating test split: 100%"
          }
        },
        "f75fca6750484e24ad795f7b4a753f83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5f7bfc69f04c41ce91a2e97001956ca8",
            "max": 4358,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c62142c7358c4e82a5061a6865344051",
            "value": 4358
          }
        },
        "227235ae048447a38845149767a151ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3c372b225e314a8aa177108144011590",
            "placeholder": "​",
            "style": "IPY_MODEL_b4f46d56c81d4aceb95c01677690679d",
            "value": " 4358/4358 [00:00&lt;00:00, 52885.85 examples/s]"
          }
        },
        "c034f0044f7540cc8888922247d07a42": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "12e77da23c2e4124ae516be1fc3c16a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2d5f72f265cc415ab410675401c7e0ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5f7bfc69f04c41ce91a2e97001956ca8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c62142c7358c4e82a5061a6865344051": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3c372b225e314a8aa177108144011590": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b4f46d56c81d4aceb95c01677690679d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "22c070b0b377435c857fd4f833f09a4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_efa119514c4e49ad9670147d24ab8fcb",
              "IPY_MODEL_c02fae38ad334841b4f45ab7c0e65699",
              "IPY_MODEL_ce14a2dbdeb347749c386c6b844f2788"
            ],
            "layout": "IPY_MODEL_ae14636242974eaa9f1113530cd9e448"
          }
        },
        "efa119514c4e49ad9670147d24ab8fcb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_611c1b601d3c4d7ea0b347c4d67f5b9a",
            "placeholder": "​",
            "style": "IPY_MODEL_cbba4afb7dc547d79ea70f519dce833b",
            "value": "Generating train split: 100%"
          }
        },
        "c02fae38ad334841b4f45ab7c0e65699": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_334dcfa8b22a4a7b84e53f51a55622c5",
            "max": 36718,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_004e395ac0e94e55b79a6650f8675431",
            "value": 36718
          }
        },
        "ce14a2dbdeb347749c386c6b844f2788": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f019da15bdcb4c8a8b5d932b8811756d",
            "placeholder": "​",
            "style": "IPY_MODEL_c61ca0378c474b2ea344fad86f7823a3",
            "value": " 36718/36718 [00:00&lt;00:00, 160026.86 examples/s]"
          }
        },
        "ae14636242974eaa9f1113530cd9e448": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "611c1b601d3c4d7ea0b347c4d67f5b9a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cbba4afb7dc547d79ea70f519dce833b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "334dcfa8b22a4a7b84e53f51a55622c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "004e395ac0e94e55b79a6650f8675431": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f019da15bdcb4c8a8b5d932b8811756d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c61ca0378c474b2ea344fad86f7823a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3f44a7f2775849d68d52b2394a1ddda3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_90d0dd00fe624c1b9090516dd9853b63",
              "IPY_MODEL_34783f19709f457aa705cd6ae92e94eb",
              "IPY_MODEL_4c91acefdd9e4b0da6cc91339e22afc0"
            ],
            "layout": "IPY_MODEL_ef9795b1c6884da9b149b9219bbeefd1"
          }
        },
        "90d0dd00fe624c1b9090516dd9853b63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_86bf7196c4044c7dab43ce20687b34d0",
            "placeholder": "​",
            "style": "IPY_MODEL_300bf51d8b364a55b2c9c9dac8671333",
            "value": "Generating validation split: 100%"
          }
        },
        "34783f19709f457aa705cd6ae92e94eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c0c16811131b4a938e6be65cb85c00b8",
            "max": 3760,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ce693953b5c24e2a8b10b9ea210bfab2",
            "value": 3760
          }
        },
        "4c91acefdd9e4b0da6cc91339e22afc0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_63f677cb8dee4c2f997bcdb974fa9bcd",
            "placeholder": "​",
            "style": "IPY_MODEL_461dd46eccfe4ec8b576f6670957c387",
            "value": " 3760/3760 [00:00&lt;00:00, 79970.91 examples/s]"
          }
        },
        "ef9795b1c6884da9b149b9219bbeefd1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "86bf7196c4044c7dab43ce20687b34d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "300bf51d8b364a55b2c9c9dac8671333": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c0c16811131b4a938e6be65cb85c00b8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce693953b5c24e2a8b10b9ea210bfab2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "63f677cb8dee4c2f997bcdb974fa9bcd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "461dd46eccfe4ec8b576f6670957c387": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5e8a51d913e540769b5c2af95660484b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fffae0d2d03444f2b99c89c6e9bee971",
              "IPY_MODEL_93e78ce6d7524d379865065ec880ec53",
              "IPY_MODEL_2242bfbe777649dab1e28404aa3d1dd5"
            ],
            "layout": "IPY_MODEL_9d09fa14b84b4d0d8cb9dfc6e548a197"
          }
        },
        "fffae0d2d03444f2b99c89c6e9bee971": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a02295b5353448e9ed4b2ee87c478a2",
            "placeholder": "​",
            "style": "IPY_MODEL_f7dd3fcf382d49ce9055875f717a9841",
            "value": "config.json: 100%"
          }
        },
        "93e78ce6d7524d379865065ec880ec53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f13d35d41b1c4c35bccd1b3d7a3da829",
            "max": 757,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e701e154ccfe453eadc38a8161b5e3e2",
            "value": 757
          }
        },
        "2242bfbe777649dab1e28404aa3d1dd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1751341bc4214b1abf34921728acbcf8",
            "placeholder": "​",
            "style": "IPY_MODEL_283b4d8c3d8b4cce90d8a5690f4c795b",
            "value": " 757/757 [00:00&lt;00:00, 48.3kB/s]"
          }
        },
        "9d09fa14b84b4d0d8cb9dfc6e548a197": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2a02295b5353448e9ed4b2ee87c478a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f7dd3fcf382d49ce9055875f717a9841": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f13d35d41b1c4c35bccd1b3d7a3da829": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e701e154ccfe453eadc38a8161b5e3e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1751341bc4214b1abf34921728acbcf8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "283b4d8c3d8b4cce90d8a5690f4c795b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5397da56986f48e5b412f85df5f33ee2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4807b7cd22824426958430cb4551f2a4",
              "IPY_MODEL_0ad0d0bf6dc242db92177032ea885cf4",
              "IPY_MODEL_ac9a795df6524c73a3410d2a3d88171a"
            ],
            "layout": "IPY_MODEL_6c78305129a84fd8905686c9f731201d"
          }
        },
        "4807b7cd22824426958430cb4551f2a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6bb187303ab94d73aef80c0986f37131",
            "placeholder": "​",
            "style": "IPY_MODEL_9b4e50cfa7ab435db2132806c4e1aee1",
            "value": "model.safetensors: 100%"
          }
        },
        "0ad0d0bf6dc242db92177032ea885cf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_92c50a5c77644b789ede48a9ab88b6ee",
            "max": 4150880304,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c5463c26f41e47d6ba0285b316182072",
            "value": 4150880304
          }
        },
        "ac9a795df6524c73a3410d2a3d88171a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_06169e68f2d94f6dbd51f8f00610730e",
            "placeholder": "​",
            "style": "IPY_MODEL_6a72b73b5ec746289df213fff1626020",
            "value": " 4.15G/4.15G [04:23&lt;00:00, 16.8MB/s]"
          }
        },
        "6c78305129a84fd8905686c9f731201d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6bb187303ab94d73aef80c0986f37131": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9b4e50cfa7ab435db2132806c4e1aee1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "92c50a5c77644b789ede48a9ab88b6ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c5463c26f41e47d6ba0285b316182072": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "06169e68f2d94f6dbd51f8f00610730e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6a72b73b5ec746289df213fff1626020": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b8228d9041c5491597c96401d65796dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4f25acab8b64415eaa189a88759ee039",
              "IPY_MODEL_6278d7a52fde4bb68ef7be5e59fb052b",
              "IPY_MODEL_07ae580ea6454ec9a1c4e58cb7bdeacf"
            ],
            "layout": "IPY_MODEL_933717fa89354548bc1995d8f74d9cdd"
          }
        },
        "4f25acab8b64415eaa189a88759ee039": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dd4c3d33776e4d5b89baa43b17e84c89",
            "placeholder": "​",
            "style": "IPY_MODEL_c0481ed5b5694a97955b0baf46604a77",
            "value": "generation_config.json: 100%"
          }
        },
        "6278d7a52fde4bb68ef7be5e59fb052b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_30b9674faacb485ba6b4cc62f24cb51a",
            "max": 116,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4705d60711764ecd99f36b105281f4af",
            "value": 116
          }
        },
        "07ae580ea6454ec9a1c4e58cb7bdeacf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc5145fec71a45ff9820982bb6df5324",
            "placeholder": "​",
            "style": "IPY_MODEL_fc1ef45dfbbf4d76bd05781b5479b5fc",
            "value": " 116/116 [00:00&lt;00:00, 7.60kB/s]"
          }
        },
        "933717fa89354548bc1995d8f74d9cdd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dd4c3d33776e4d5b89baa43b17e84c89": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c0481ed5b5694a97955b0baf46604a77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "30b9674faacb485ba6b4cc62f24cb51a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4705d60711764ecd99f36b105281f4af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bc5145fec71a45ff9820982bb6df5324": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fc1ef45dfbbf4d76bd05781b5479b5fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5574c9b17b7b4a4e9bb87e3d03e0dc13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_33748abc61c940aaac11d6dccaa9f1b0",
              "IPY_MODEL_ed5a797a35da4491b3c075fbb33b52f6",
              "IPY_MODEL_f39d8173026b4023b8be9a1bd8def980"
            ],
            "layout": "IPY_MODEL_c2a678410ac64b3bb4ed8cb2e19e0e10"
          }
        },
        "33748abc61c940aaac11d6dccaa9f1b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_038ae9d6da14495db8a362184b045f37",
            "placeholder": "​",
            "style": "IPY_MODEL_716f23bb032d4773aedb5bb250d8070a",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "ed5a797a35da4491b3c075fbb33b52f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_89d6b4c10cc846b7ac400372a59d3ba8",
            "max": 962,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a0fd4fd17b864f93823cbd3ff7b1627e",
            "value": 962
          }
        },
        "f39d8173026b4023b8be9a1bd8def980": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f11754700fcd49e5b8c4365b8271afe1",
            "placeholder": "​",
            "style": "IPY_MODEL_5845dbe4675c414faa2670a06d6b785a",
            "value": " 962/962 [00:00&lt;00:00, 60.6kB/s]"
          }
        },
        "c2a678410ac64b3bb4ed8cb2e19e0e10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "038ae9d6da14495db8a362184b045f37": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "716f23bb032d4773aedb5bb250d8070a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "89d6b4c10cc846b7ac400372a59d3ba8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0fd4fd17b864f93823cbd3ff7b1627e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f11754700fcd49e5b8c4365b8271afe1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5845dbe4675c414faa2670a06d6b785a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "800d96d21ef64899b3183312cc0eb1fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_099e0fccfa4b4cafbb4a5026ace9839a",
              "IPY_MODEL_a93c96cf1d6c4abeaf8706fcb9969a3c",
              "IPY_MODEL_2ca5a801d6f04629b7fdd1d5e7ab307e"
            ],
            "layout": "IPY_MODEL_6f236be00cdc42fcac662c7d43c7b672"
          }
        },
        "099e0fccfa4b4cafbb4a5026ace9839a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_00ab8ec5b2de431d97678894213c142c",
            "placeholder": "​",
            "style": "IPY_MODEL_2710aaebaa8b497c9924898212fbaa9f",
            "value": "tokenizer.model: 100%"
          }
        },
        "a93c96cf1d6c4abeaf8706fcb9969a3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5f1b929fd4c64f27a8bd64e60fa89e02",
            "max": 493443,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_08643a3fdb4940c688bce7f5c406c320",
            "value": 493443
          }
        },
        "2ca5a801d6f04629b7fdd1d5e7ab307e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d115ff9c42464306a02229ea3625bb11",
            "placeholder": "​",
            "style": "IPY_MODEL_a0416e48e00d45dfa9dba46b3eef39bd",
            "value": " 493k/493k [00:03&lt;00:00, 159kB/s]"
          }
        },
        "6f236be00cdc42fcac662c7d43c7b672": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "00ab8ec5b2de431d97678894213c142c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2710aaebaa8b497c9924898212fbaa9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5f1b929fd4c64f27a8bd64e60fa89e02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "08643a3fdb4940c688bce7f5c406c320": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d115ff9c42464306a02229ea3625bb11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0416e48e00d45dfa9dba46b3eef39bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9953bfe8affd4a8ba74894d1787b7372": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e258b238299d41ffb9d2ffe63faf16c8",
              "IPY_MODEL_b0a64f0f7a3e4fd595e8b1a487b6d323",
              "IPY_MODEL_72798c51abc04899b4d0fde0739bae07"
            ],
            "layout": "IPY_MODEL_4ca1327667084b1d99fc6cb1dda7c3c9"
          }
        },
        "e258b238299d41ffb9d2ffe63faf16c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5e84091bc7514e76b200a4db23e5de15",
            "placeholder": "​",
            "style": "IPY_MODEL_73f25d8e7e194ac3af759afbe54385ea",
            "value": "tokenizer.json: 100%"
          }
        },
        "b0a64f0f7a3e4fd595e8b1a487b6d323": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_15cf767ff4374f06b6ccafea00daff0b",
            "max": 1795300,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_db9a9a91c2e941a7a68f1b2387ba2b32",
            "value": 1795300
          }
        },
        "72798c51abc04899b4d0fde0739bae07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2d7bf1a1a9984970976458ea087de707",
            "placeholder": "​",
            "style": "IPY_MODEL_cfd896d3647b4936b090a9a37a452b49",
            "value": " 1.80M/1.80M [00:00&lt;00:00, 29.7MB/s]"
          }
        },
        "4ca1327667084b1d99fc6cb1dda7c3c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5e84091bc7514e76b200a4db23e5de15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "73f25d8e7e194ac3af759afbe54385ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "15cf767ff4374f06b6ccafea00daff0b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "db9a9a91c2e941a7a68f1b2387ba2b32": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2d7bf1a1a9984970976458ea087de707": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cfd896d3647b4936b090a9a37a452b49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1f88dd4d9a2d48caaf78b50536792c37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_379bc80fbbc8479384a73c4c6449eb68",
              "IPY_MODEL_f66048d816f04d6dbbcfbed995e8716e",
              "IPY_MODEL_af1ceec3e66b44b083c2983d26a46068"
            ],
            "layout": "IPY_MODEL_5fdd96706c6c43be9219d97e84e76a43"
          }
        },
        "379bc80fbbc8479384a73c4c6449eb68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d3f78544d98a4431a5dcda77083f4727",
            "placeholder": "​",
            "style": "IPY_MODEL_b1d10cb261f94e17829847c783f627a2",
            "value": "added_tokens.json: 100%"
          }
        },
        "f66048d816f04d6dbbcfbed995e8716e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2d7198b84d184ce9b40937efaa930d35",
            "max": 42,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0e04f2bf60f94b798e230035e5ac94d9",
            "value": 42
          }
        },
        "af1ceec3e66b44b083c2983d26a46068": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f8073adc11c7492299fe5c5e0868d030",
            "placeholder": "​",
            "style": "IPY_MODEL_0a6f177b9763493996bb5c5658048f8f",
            "value": " 42.0/42.0 [00:00&lt;00:00, 2.67kB/s]"
          }
        },
        "5fdd96706c6c43be9219d97e84e76a43": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d3f78544d98a4431a5dcda77083f4727": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b1d10cb261f94e17829847c783f627a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2d7198b84d184ce9b40937efaa930d35": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0e04f2bf60f94b798e230035e5ac94d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f8073adc11c7492299fe5c5e0868d030": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a6f177b9763493996bb5c5658048f8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3f89d6b0190d4891a933fb0c1e67255f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_24656013428b4a7e961cbc29a837f9ea",
              "IPY_MODEL_ae38d05fbc05469d96dbe6fa039dd9df",
              "IPY_MODEL_4748a9f59cb74ef4897b96401fcfc968"
            ],
            "layout": "IPY_MODEL_9d494edf5a8f48f184a251ca36dcbf96"
          }
        },
        "24656013428b4a7e961cbc29a837f9ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fbf13624fbec4c9ead029be700634fec",
            "placeholder": "​",
            "style": "IPY_MODEL_fa50ec515d144cabb4138c8e84c1927d",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "ae38d05fbc05469d96dbe6fa039dd9df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b64332d4bd1846e6b16519b2bf92c973",
            "max": 72,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_159efa92982c41c99b4da656564ed8b3",
            "value": 72
          }
        },
        "4748a9f59cb74ef4897b96401fcfc968": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1deac6009ee04d40aa66e20c606b22c0",
            "placeholder": "​",
            "style": "IPY_MODEL_639c1f47215a44bda4eaca62e5b889ae",
            "value": " 72.0/72.0 [00:00&lt;00:00, 5.17kB/s]"
          }
        },
        "9d494edf5a8f48f184a251ca36dcbf96": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fbf13624fbec4c9ead029be700634fec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa50ec515d144cabb4138c8e84c1927d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b64332d4bd1846e6b16519b2bf92c973": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "159efa92982c41c99b4da656564ed8b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1deac6009ee04d40aa66e20c606b22c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "639c1f47215a44bda4eaca62e5b889ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/badlogic/genai-workshop/blob/main/04_unsupervised_learning_language_models.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "> **Note:** Before you continue, make sure the notebook runtime on Google Colab is set to T4. Click on the arrow to the right of the \"RAM/Disk\" area in the top right corner, then click \"Runtime Type\" and select T4. This will start the notebook with a GPU-enabled runtime."
      ],
      "metadata": {
        "id": "KzaUfi9RAT_a"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Language Models\n",
        "Language models are at the core of the recent AI hype, brought to you by OpenAI's ChatGPT, products like GitHub's Copilot, or open-source models like Llama.\n",
        "\n",
        "As described in [the unsupervised learning overview](https://colab.research.google.com/drive/10tlC17BRVoX9aPp66orqiI16iUzLx-4p?usp=sharing), the goal of language models is it to predict the likelihood of a sequence of **tokens** in a language. A token can be a word or punctuation in the simplest case. There are more sophisticated tokenizers that break words down into sub-units, like [binary pair encoding](https://en.wikipedia.org/wiki/Byte_pair_encoding) or [SentencePiece](https://github.com/google/sentencepiece).\n",
        "\n",
        "A language model learns patterns and structures based on large amounts of text data to generate or understand text. The patterns are learned in an unsupervised manner directly from the training data.\n",
        "\n",
        "Such models can then be used to predict the next token in a sequence, or fill in a token in-between other tokens of a text.\n",
        "\n",
        "Let's create a simple language model of our own, to build an initial intution about such models, before we dive into the current state-of-the-art in more depth."
      ],
      "metadata": {
        "id": "3SPT9uhDv-D-"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# A naive n-gram language model\n",
        "The simplest language model we can build is an [n-gram model](https://en.wikipedia.org/wiki/Word_n-gram_language_model), based on word tokens.\n"
      ],
      "metadata": {
        "id": "YtIVoNLUyVnf"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data Preparation\n",
        "Before we start building the model, we need some data to learn from. We'll use a small [Wikipedia dataset](https://huggingface.co/datasets/wikitext/viewer/wikitext-2-raw-v1), consisting of about ~44k paragraphs from Wikipedia articles, of which we use ~36k to train the language model (\"train\" split)."
      ],
      "metadata": {
        "id": "XQ6E62TF60L6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip -q install datasets"
      ],
      "metadata": {
        "id": "mBulGBoMBaRn"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "import pandas as pd\n",
        "\n",
        "paragraphs = load_dataset('wikitext', 'wikitext-2-raw-v1')[\"train\"];\n",
        "print(f'{len(paragraphs)} paragraphs')\n",
        "pd.DataFrame(paragraphs[\"text\"][:100], columns=[\"text\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 789,
          "referenced_widgets": [
            "48524f233dab40a1a8a4f49de10c741d",
            "6dceb6e249274cd78400c5b135a32b27",
            "b559ab6d7def43a8899b5c4a1db15214",
            "8f1aebfa6daa494485040a5a1d2f711b",
            "ae26b1205f61423b924f1cba03abed94",
            "193392a07f9743268eeb86e5d52dd2cf",
            "48019fbf73814a58adf78b0c6099eda7",
            "b4c963851c8c465386ec03ca55ce8b71",
            "2891c35063a546cb9667dc3deb382527",
            "4a0f169deeab41208e68e7e5a6693eb8",
            "8aa654b2c2634e1897a37a68861aab90",
            "34cdb78c2b164747ad86e1d6aa8a0702",
            "c05bfea913cc4309a8e2378d2f91fa0a",
            "c172fdf108bc43a3a0f35851ca52640a",
            "085ed8e03f094297aa087e8160b16141",
            "589d0a459df64eadae77cc6d06128196",
            "413df2a280304bebbe5f1849adf70f41",
            "cdbe10d96c534544b6857adf289ff8b9",
            "657fa5fc13e541e29d5219bd0a8ae4db",
            "bb1272540b9244cca0e9f5726cc597ba",
            "c337a8fbccf84fd182d70c013b837bbc",
            "4c54e305a63e4261b0a4b916daaa3736",
            "d919be2767584000b248de99eaae4e30",
            "d8543decb9ab4f7d8729c1672e102554",
            "7405bc3b596c4b778fe3a22dac4229da",
            "0635f88c0036409da976a84e26f54e17",
            "9000b335d4174443b6fc509447b5ecb8",
            "b22b883fdbe44be8bccd1b161eed26f8",
            "6bff9295fd5f4d8a9708c427704962f5",
            "a4acd9bcb8ff4bca874c944aa5060bcd",
            "6d2c67cf56b747178de36bad4d6c3a2f",
            "2372463cb8244ad28f321a1608ab3d17",
            "4bd68037c5a344dd8ebe06572ab91302",
            "86f5aace1dc4454786cb1b92b9b72b99",
            "52675d7edbbd4e3f8aad5f775f39adc8",
            "4ba773bcb1b14821934728a39bef3efa",
            "552d1789ceaa43ab867a7e78e440e878",
            "db6507119fbc42c7996ddb48f63e3d6d",
            "7d4bdf46f1f748ff94fa0748f8b829e3",
            "bc21cfdbdef3452e8d7792e92ed38599",
            "0c80b9fce8704a6ba3c38b4dc5732a08",
            "563e51df03804a10a06395755f616358",
            "3c6b76b3491e4eed977648ea15bbfcca",
            "3865265fbaf0439e94b8d659c8536dbc",
            "68527ced1ee24ef8bfab018f26f6b252",
            "51ee70fe9bf447149c74ce455fb9c284",
            "f75fca6750484e24ad795f7b4a753f83",
            "227235ae048447a38845149767a151ab",
            "c034f0044f7540cc8888922247d07a42",
            "12e77da23c2e4124ae516be1fc3c16a4",
            "2d5f72f265cc415ab410675401c7e0ef",
            "5f7bfc69f04c41ce91a2e97001956ca8",
            "c62142c7358c4e82a5061a6865344051",
            "3c372b225e314a8aa177108144011590",
            "b4f46d56c81d4aceb95c01677690679d",
            "22c070b0b377435c857fd4f833f09a4d",
            "efa119514c4e49ad9670147d24ab8fcb",
            "c02fae38ad334841b4f45ab7c0e65699",
            "ce14a2dbdeb347749c386c6b844f2788",
            "ae14636242974eaa9f1113530cd9e448",
            "611c1b601d3c4d7ea0b347c4d67f5b9a",
            "cbba4afb7dc547d79ea70f519dce833b",
            "334dcfa8b22a4a7b84e53f51a55622c5",
            "004e395ac0e94e55b79a6650f8675431",
            "f019da15bdcb4c8a8b5d932b8811756d",
            "c61ca0378c474b2ea344fad86f7823a3",
            "3f44a7f2775849d68d52b2394a1ddda3",
            "90d0dd00fe624c1b9090516dd9853b63",
            "34783f19709f457aa705cd6ae92e94eb",
            "4c91acefdd9e4b0da6cc91339e22afc0",
            "ef9795b1c6884da9b149b9219bbeefd1",
            "86bf7196c4044c7dab43ce20687b34d0",
            "300bf51d8b364a55b2c9c9dac8671333",
            "c0c16811131b4a938e6be65cb85c00b8",
            "ce693953b5c24e2a8b10b9ea210bfab2",
            "63f677cb8dee4c2f997bcdb974fa9bcd",
            "461dd46eccfe4ec8b576f6670957c387"
          ]
        },
        "id": "kPaA8qvZBY07",
        "outputId": "a5cf4f7e-9f55-493c-82cf-728f274a0331"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/10.5k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "48524f233dab40a1a8a4f49de10c741d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/733k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "34cdb78c2b164747ad86e1d6aa8a0702"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/6.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d919be2767584000b248de99eaae4e30"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/657k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "86f5aace1dc4454786cb1b92b9b72b99"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating test split:   0%|          | 0/4358 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "68527ced1ee24ef8bfab018f26f6b252"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split:   0%|          | 0/36718 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "22c070b0b377435c857fd4f833f09a4d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating validation split:   0%|          | 0/3760 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3f44a7f2775849d68d52b2394a1ddda3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "36718 paragraphs\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                 text\n",
              "0                                                    \n",
              "1                      = Valkyria Chronicles III = \\n\n",
              "2                                                    \n",
              "3    Senjō no Valkyria 3 : Unrecorded Chronicles (...\n",
              "4    The game began development in 2010 , carrying...\n",
              "..                                                ...\n",
              "95   75 @,@ 000 buck & ball cartridges - percussio...\n",
              "96       14 @,@ 000 buck & ball cartridges - flint \\n\n",
              "97                                 275 paper fuzes \\n\n",
              "98        117 rounds , 6 @-@ pounder canister shot \\n\n",
              "99            130 rounds , 6 @-@ pounder ball shot \\n\n",
              "\n",
              "[100 rows x 1 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0cb2174c-8c1c-4a5e-9a9d-2b3097017e8f\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>= Valkyria Chronicles III = \\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Senjō no Valkyria 3 : Unrecorded Chronicles (...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The game began development in 2010 , carrying...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95</th>\n",
              "      <td>75 @,@ 000 buck &amp; ball cartridges - percussio...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>14 @,@ 000 buck &amp; ball cartridges - flint \\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>275 paper fuzes \\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>117 rounds , 6 @-@ pounder canister shot \\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>130 rounds , 6 @-@ pounder ball shot \\n</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>100 rows × 1 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0cb2174c-8c1c-4a5e-9a9d-2b3097017e8f')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-0cb2174c-8c1c-4a5e-9a9d-2b3097017e8f button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-0cb2174c-8c1c-4a5e-9a9d-2b3097017e8f');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-12479e56-be9e-4f07-a861-892b92ae8963\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-12479e56-be9e-4f07-a861-892b92ae8963')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-12479e56-be9e-4f07-a861-892b92ae8963 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"pd\",\n  \"rows\": 100,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 76,\n        \"samples\": [\n          \" It met with positive sales in Japan , and was praised by both Japanese and western critics . After release , it received downloadable content , along with an expanded edition in November of that year . It was also adapted into manga and an original video animation series . Due to low sales of Valkyria Chronicles II , Valkyria Chronicles III was not localized , but a fan translation compatible with the game 's expanded edition was released in 2014 . Media.Vision would return to the franchise with the development of Valkyria : Azure Revolution for the PlayStation 4 . \\n\",\n          \" = = Construction = = \\n\",\n          \" The game takes place during the Second Europan War . Gallian Army Squad 422 , also known as \\\" The Nameless \\\" , are a penal military unit composed of criminals , foreign deserters , and military offenders whose real names are erased from the records and thereon officially referred to by numbers . Ordered by the Gallian military to perform the most dangerous missions that the Regular Army and Militia will not do , they are nevertheless up to the task , exemplified by their motto , Altaha Abilia , meaning \\\" Always Ready . \\\" The three main characters are No.7 Kurt Irving , an army officer falsely accused of treason who wishes to redeem himself ; Ace No.1 Imca , a female Darcsen heavy weapons specialist who seeks revenge against the Valkyria who destroyed her home ; and No.13 Riela Marcellis , a seemingly jinxed young woman who is unknowingly a descendant of the Valkyria . Together with their fellow squad members , these three are tasked to fight against a mysterious Imperial unit known as Calamity Raven , consisting of mostly Darcsen soldiers . \\n\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Next, we need to split each paragraph into sentences, and each sentence into word tokens. We use the sentence splitter and tokenizer from [NLTK](https://www.nltk.org/) to achieve this."
      ],
      "metadata": {
        "id": "kBDoHakF6-h-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.tokenize import word_tokenize, sent_tokenize\n",
        "import nltk\n",
        "\n",
        "nltk.download('punkt')\n",
        "sentences = [word_tokenize(sentence) for text in paragraphs['text'] for sentence in sent_tokenize(text)]"
      ],
      "metadata": {
        "id": "tZDYpsv6DRLi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5af93450-adf9-4318-ebd0-efbd13aec75e"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Here's one tokenized sentence."
      ],
      "metadata": {
        "id": "SAevoDdBu24t"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pd.DataFrame(sentences[10], columns=[\"token\"])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 614
        },
        "id": "Ct9l-zvk7xBP",
        "outputId": "635c7260-7668-4b1d-bd12-3e8c1ab465d1"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       token\n",
              "0         It\n",
              "1        met\n",
              "2       with\n",
              "3   positive\n",
              "4      sales\n",
              "5         in\n",
              "6      Japan\n",
              "7          ,\n",
              "8        and\n",
              "9        was\n",
              "10   praised\n",
              "11        by\n",
              "12      both\n",
              "13  Japanese\n",
              "14       and\n",
              "15   western\n",
              "16   critics\n",
              "17         ."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-53280ab2-57ff-4fb5-8c65-0c072047f06d\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>token</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>It</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>met</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>with</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>sales</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>in</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Japan</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>,</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>and</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>was</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>praised</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>by</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>both</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>Japanese</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>and</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>western</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>critics</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-53280ab2-57ff-4fb5-8c65-0c072047f06d')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-53280ab2-57ff-4fb5-8c65-0c072047f06d button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-53280ab2-57ff-4fb5-8c65-0c072047f06d');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-9736928c-a77b-4b55-84cd-7a7cc355416a\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9736928c-a77b-4b55-84cd-7a7cc355416a')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-9736928c-a77b-4b55-84cd-7a7cc355416a button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"pd\",\n  \"rows\": 18,\n  \"fields\": [\n    {\n      \"column\": \"token\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 17,\n        \"samples\": [\n          \"It\",\n          \"met\",\n          \"in\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training\n",
        "\n",
        "Time to train the language model. The algorithm may look a bit daunting, but the concept is very simply.\n",
        "\n",
        "We first check that the provided parameter `n` is valid, that is, is bigger or equal to 1.\n",
        "\n",
        "Next we instantiate a dictionary called `n_gram_stats`, which maps from a prefix to a dictionary. The dictionary for a prefix contains all the tokens and their counts that were observed in the training data following the prefix.\n",
        "\n",
        "Next, we iterate through all sentences. For each sentence, we iterate through all token positions.\n",
        "\n",
        "For each token position, we initialize the prefix to the empty string. We then iterate through all token positions from the current token position to the current token position + `n`. In each step, we get the current token for the position and add one to the dictionary entry of its prefix. We then expand the prefix with this token, and move on to the next token position.\n",
        "\n",
        "Once we've collected all statistics, we convert each dictionary for a prefix to a list, and sort the `(token, count)` tuples in descending order by count. This will later help us speed up generation.\n",
        "\n",
        "> **Note:** standard n-gram models are probabilistic in nature and would normalize the counts to model probability distributions. For our purposes, it's sufficient to stick to the raw counts.\n",
        "\n",
        "Execute the code below to see the training algorithm build a model over the single sentence `\"The quick brown fox jumped over the lazy dog\"` using `n=5`. The debug parameter is set to `True`, so the training code will output all n-grams it records counts for."
      ],
      "metadata": {
        "id": "PcPzjOKp8c26"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import defaultdict\n",
        "\n",
        "def train_lm(sentences, n, debug=False):\n",
        "  if (n < 1):\n",
        "    raise ValueError(\"n must be >= 1\")\n",
        "  n_gram_stats = defaultdict(lambda: defaultdict(int))\n",
        "  for sentence in sentences:\n",
        "    for i in range(len(sentence)):\n",
        "      prefix = \"\"\n",
        "      for j in range(0, n):\n",
        "        if i + j < len(sentence):\n",
        "          token = sentence[i + j]\n",
        "          n_gram_stats[prefix][token] += 1\n",
        "          if (debug):\n",
        "            print(f'{prefix} -> {token}')\n",
        "          prefix += (\" \" if prefix else \"\") + token\n",
        "\n",
        "  for prefix, token_counts in n_gram_stats.items():\n",
        "    sorted_token_counts = sorted(token_counts.items(), key=lambda item: item[1], reverse=True)\n",
        "    n_gram_stats[prefix] = sorted_token_counts\n",
        "\n",
        "  return n_gram_stats\n",
        "\n",
        "sentence = [\"The\", \"quick\", \"brown\", \"fox\", \"jumped\", \"over\", \"the\", \"lazy\", \"dog\", \".\"];\n",
        "model = train_lm([sentence], 5, True)"
      ],
      "metadata": {
        "id": "yjFz5FZkMd2d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0f803a91-fa7f-4d6d-fe58-63e6cf06d162"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " -> The\n",
            "The -> quick\n",
            "The quick -> brown\n",
            "The quick brown -> fox\n",
            "The quick brown fox -> jumped\n",
            " -> quick\n",
            "quick -> brown\n",
            "quick brown -> fox\n",
            "quick brown fox -> jumped\n",
            "quick brown fox jumped -> over\n",
            " -> brown\n",
            "brown -> fox\n",
            "brown fox -> jumped\n",
            "brown fox jumped -> over\n",
            "brown fox jumped over -> the\n",
            " -> fox\n",
            "fox -> jumped\n",
            "fox jumped -> over\n",
            "fox jumped over -> the\n",
            "fox jumped over the -> lazy\n",
            " -> jumped\n",
            "jumped -> over\n",
            "jumped over -> the\n",
            "jumped over the -> lazy\n",
            "jumped over the lazy -> dog\n",
            " -> over\n",
            "over -> the\n",
            "over the -> lazy\n",
            "over the lazy -> dog\n",
            "over the lazy dog -> .\n",
            " -> the\n",
            "the -> lazy\n",
            "the lazy -> dog\n",
            "the lazy dog -> .\n",
            " -> lazy\n",
            "lazy -> dog\n",
            "lazy dog -> .\n",
            " -> dog\n",
            "dog -> .\n",
            " -> .\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can also output the recorded counts for each `prefix -> token` pair the model learned."
      ],
      "metadata": {
        "id": "5rKjdXL5IUnu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for prefix, tokens in model.items():\n",
        "  for token, count in tokens:\n",
        "    print(f'\"{prefix}\", \"{token}\" -> {count}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_VOkscr7IZtz",
        "outputId": "3b862400-6ff4-446f-ec77-134ce7333a34"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\"\", \"The\" -> 1\n",
            "\"\", \"quick\" -> 1\n",
            "\"\", \"brown\" -> 1\n",
            "\"\", \"fox\" -> 1\n",
            "\"\", \"jumped\" -> 1\n",
            "\"\", \"over\" -> 1\n",
            "\"\", \"the\" -> 1\n",
            "\"\", \"lazy\" -> 1\n",
            "\"\", \"dog\" -> 1\n",
            "\"\", \".\" -> 1\n",
            "\"The\", \"quick\" -> 1\n",
            "\"The quick\", \"brown\" -> 1\n",
            "\"The quick brown\", \"fox\" -> 1\n",
            "\"The quick brown fox\", \"jumped\" -> 1\n",
            "\"quick\", \"brown\" -> 1\n",
            "\"quick brown\", \"fox\" -> 1\n",
            "\"quick brown fox\", \"jumped\" -> 1\n",
            "\"quick brown fox jumped\", \"over\" -> 1\n",
            "\"brown\", \"fox\" -> 1\n",
            "\"brown fox\", \"jumped\" -> 1\n",
            "\"brown fox jumped\", \"over\" -> 1\n",
            "\"brown fox jumped over\", \"the\" -> 1\n",
            "\"fox\", \"jumped\" -> 1\n",
            "\"fox jumped\", \"over\" -> 1\n",
            "\"fox jumped over\", \"the\" -> 1\n",
            "\"fox jumped over the\", \"lazy\" -> 1\n",
            "\"jumped\", \"over\" -> 1\n",
            "\"jumped over\", \"the\" -> 1\n",
            "\"jumped over the\", \"lazy\" -> 1\n",
            "\"jumped over the lazy\", \"dog\" -> 1\n",
            "\"over\", \"the\" -> 1\n",
            "\"over the\", \"lazy\" -> 1\n",
            "\"over the lazy\", \"dog\" -> 1\n",
            "\"over the lazy dog\", \".\" -> 1\n",
            "\"the\", \"lazy\" -> 1\n",
            "\"the lazy\", \"dog\" -> 1\n",
            "\"the lazy dog\", \".\" -> 1\n",
            "\"lazy\", \"dog\" -> 1\n",
            "\"lazy dog\", \".\" -> 1\n",
            "\"dog\", \".\" -> 1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "With `n=5`, we learn the counts of sequences with up to `n` tokens. The model can not learn the counts of longer sequences. We can increase `n`, but will eventually run out of memory.\n",
        "\n",
        "Let's build the model over all sentences from our Wikipedia training set using a token window size of `n=8`. We also want to measure the approximate RAM consumption of the resulting model, as well as the execution time of the training."
      ],
      "metadata": {
        "id": "GTROhOxyGIYC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import psutil\n",
        "import os\n",
        "import gc\n",
        "import time\n",
        "\n",
        "# Function to get current RAM usage in MB\n",
        "def get_ram_usage():\n",
        "    process = psutil.Process(os.getpid())\n",
        "    return f\"{process.memory_info().rss / 1024 / 1024: .2f} MB\"\n",
        "\n",
        "for i in range(20):\n",
        "  gc.collect()\n",
        "print(f'Before: {get_ram_usage()}')\n",
        "start_time = time.time()\n",
        "\n",
        "model = train_lm(sentences, 8)\n",
        "\n",
        "print(f'After: {get_ram_usage()}')\n",
        "print(f'Took: {time.time() - start_time} seconds')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W1WTQS6Y-dSP",
        "outputId": "b518607c-1da2-4b77-b860-a05110454e3d"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Before:  555.16 MB\n",
            "After:  3760.29 MB\n",
            "Took: 41.685065269470215 seconds\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Execution takes around 30-60 seconds, while the model approximately takes up ~3.2GB of RAM. That's a lot for such a small dataset and token window size!\n",
        "\n",
        "We could improve both execution time and RAM usage by employing a smarter encoding scheme. For the sake or brevity, that is left as an exercise to the reader. We should appreciate better what large language models are capable of though. For only 2 to 4 time that amount of (V)RAM, we can run an infinitely more capable large language model like [Llama 7B](https://huggingface.co/meta-llama/Llama-2-7b)."
      ],
      "metadata": {
        "id": "WpG00d15Lw14"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Generating sentences\n",
        "Given a token sequence, we can now predict the next most likely token using our language model. Let's write a function called `predict_lm` that does just that.\n",
        "\n",
        "It takes as input the model, the sequence for which we want to predict the next token, and two optional parameters `top_k`, which defines how many of he most probable tokens to pick, and `randomize`, which defines if a token should be picked at random from the `top_k` candidate tokens, or if the most probable token should be picked. These two optional parameters define how \"creative\" our language model is allowed to get.\n",
        "\n",
        "The function first tokenizes the input sequence.\n",
        "\n",
        "It then tries to find the longest prefix from the end of the sequence in the model for which a token list is available.\n",
        "\n",
        "If no token list can be found, it fetches the token list for the prefix `\"\"`, which is the list of unigram counts.\n",
        "\n",
        "Next, it selects the top-k tokens from the list based on their count.\n",
        "\n",
        "If randomization is enabled, it will pick a token at random from this list. Otherwise, it will pick the token with the highest count."
      ],
      "metadata": {
        "id": "iyJUSCNaOM3q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "\n",
        "def predict_lm(model, sequence, top_k=5, randomize=True, debug=False):\n",
        "    input_tokens = word_tokenize(sequence)\n",
        "    n = max(1, len(input_tokens))\n",
        "    token_list = None\n",
        "\n",
        "    while n > 0 and token_list is None:\n",
        "        prefix = \" \".join(input_tokens[-n:])\n",
        "        if debug:\n",
        "          print(f'Probing prefix \"{prefix}\"')\n",
        "        token_list = model.get(prefix)\n",
        "        if token_list is None:\n",
        "          n -= 1\n",
        "        else:\n",
        "          if debug:\n",
        "            print(f'Full token list for prefix \"{prefix}\": {token_list}')\n",
        "\n",
        "    if token_list is None:\n",
        "      token_list = model.get(\"\")\n",
        "\n",
        "    top_k_tokens = token_list[:top_k]\n",
        "    if debug:\n",
        "      print(f'Top-k tokens for prefix \"{prefix}\": {top_k_tokens}')\n",
        "    if randomize:\n",
        "        next_token = random.choice(top_k_tokens)[0]\n",
        "    else:\n",
        "        next_token = token_list[0]\n",
        "    return next_token"
      ],
      "metadata": {
        "id": "gIWTI7mvXzTi"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's see what the model predicts for \"The cat in the\""
      ],
      "metadata": {
        "id": "Lfm7qREAW6jy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "predict_lm(model, \"The cat in\", 10, True, True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "id": "VU_K7l91W5Ug",
        "outputId": "a79dddcf-7c9c-43b8-da1f-520b1285ad26"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Probing prefix \"The cat in\"\n",
            "Probing prefix \"cat in\"\n",
            "Full token list for prefix \"cat in\": [('the', 1), ('every', 1), ('its', 1)]\n",
            "Top-k tokens for prefix \"cat in\": [('the', 1), ('every', 1), ('its', 1)]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'every'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's write a function called `complete` which generates a complete sentence based on an input query.\n",
        "\n",
        "It takes the same inputs as `predict_lm`, plus an additional parameter `max_tokens`, which defines the maximum number of tokens to generate.\n",
        "\n",
        "The function will call `predict_lm` in a loop, until either a \".\" was predicted, or the maximal number of tokens was generated.\n",
        "\n",
        "The generated token is appended to the current sequence. This extended sequence is then fed back into the model prediction function to generate the next token.\n",
        "\n",
        "This is also how large language models work at the core."
      ],
      "metadata": {
        "id": "6h7Zl5d0Yz8o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def complete_lm(model, sequence, max_tokens=40, top_k=5, randomize=True, debug=False):\n",
        "  for i in range(max_tokens):\n",
        "    next = predict_lm(model, sequence, top_k, randomize, debug)\n",
        "    sequence += \" \" + next\n",
        "    if (next == \".\"):\n",
        "      break;\n",
        "\n",
        "  return sequence"
      ],
      "metadata": {
        "id": "9HkA1WIlh9j4"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "complete_lm(model, \"The president of\", 40, 5, True, True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "ymu8DOR3NI9B",
        "outputId": "b052908a-1bf4-489c-bfa4-9b221a5b6cc8"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Probing prefix \"The president of\"\n",
            "Full token list for prefix \"The president of\": [('the', 3)]\n",
            "Top-k tokens for prefix \"The president of\": [('the', 3)]\n",
            "Probing prefix \"The president of the\"\n",
            "Full token list for prefix \"The president of the\": [('Supreme', 1), ('Constitutional', 1), ('university', 1)]\n",
            "Top-k tokens for prefix \"The president of the\": [('Supreme', 1), ('Constitutional', 1), ('university', 1)]\n",
            "Probing prefix \"The president of the university\"\n",
            "Full token list for prefix \"The president of the university\": [('later', 1)]\n",
            "Top-k tokens for prefix \"The president of the university\": [('later', 1)]\n",
            "Probing prefix \"The president of the university later\"\n",
            "Full token list for prefix \"The president of the university later\": [('recalled', 1)]\n",
            "Top-k tokens for prefix \"The president of the university later\": [('recalled', 1)]\n",
            "Probing prefix \"The president of the university later recalled\"\n",
            "Full token list for prefix \"The president of the university later recalled\": [('that', 1)]\n",
            "Top-k tokens for prefix \"The president of the university later recalled\": [('that', 1)]\n",
            "Probing prefix \"The president of the university later recalled that\"\n",
            "Probing prefix \"president of the university later recalled that\"\n",
            "Full token list for prefix \"president of the university later recalled that\": [('he', 1)]\n",
            "Top-k tokens for prefix \"president of the university later recalled that\": [('he', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he\"\n",
            "Probing prefix \"president of the university later recalled that he\"\n",
            "Probing prefix \"of the university later recalled that he\"\n",
            "Full token list for prefix \"of the university later recalled that he\": [('was', 1)]\n",
            "Top-k tokens for prefix \"of the university later recalled that he\": [('was', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he was\"\n",
            "Probing prefix \"president of the university later recalled that he was\"\n",
            "Probing prefix \"of the university later recalled that he was\"\n",
            "Probing prefix \"the university later recalled that he was\"\n",
            "Full token list for prefix \"the university later recalled that he was\": [('often', 1)]\n",
            "Top-k tokens for prefix \"the university later recalled that he was\": [('often', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he was often\"\n",
            "Probing prefix \"president of the university later recalled that he was often\"\n",
            "Probing prefix \"of the university later recalled that he was often\"\n",
            "Probing prefix \"the university later recalled that he was often\"\n",
            "Probing prefix \"university later recalled that he was often\"\n",
            "Full token list for prefix \"university later recalled that he was often\": [('mentally', 1)]\n",
            "Top-k tokens for prefix \"university later recalled that he was often\": [('mentally', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he was often mentally\"\n",
            "Probing prefix \"president of the university later recalled that he was often mentally\"\n",
            "Probing prefix \"of the university later recalled that he was often mentally\"\n",
            "Probing prefix \"the university later recalled that he was often mentally\"\n",
            "Probing prefix \"university later recalled that he was often mentally\"\n",
            "Probing prefix \"later recalled that he was often mentally\"\n",
            "Full token list for prefix \"later recalled that he was often mentally\": [('jumbled', 1)]\n",
            "Top-k tokens for prefix \"later recalled that he was often mentally\": [('jumbled', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he was often mentally jumbled\"\n",
            "Probing prefix \"president of the university later recalled that he was often mentally jumbled\"\n",
            "Probing prefix \"of the university later recalled that he was often mentally jumbled\"\n",
            "Probing prefix \"the university later recalled that he was often mentally jumbled\"\n",
            "Probing prefix \"university later recalled that he was often mentally jumbled\"\n",
            "Probing prefix \"later recalled that he was often mentally jumbled\"\n",
            "Probing prefix \"recalled that he was often mentally jumbled\"\n",
            "Full token list for prefix \"recalled that he was often mentally jumbled\": [('and', 1)]\n",
            "Top-k tokens for prefix \"recalled that he was often mentally jumbled\": [('and', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he was often mentally jumbled and\"\n",
            "Probing prefix \"president of the university later recalled that he was often mentally jumbled and\"\n",
            "Probing prefix \"of the university later recalled that he was often mentally jumbled and\"\n",
            "Probing prefix \"the university later recalled that he was often mentally jumbled and\"\n",
            "Probing prefix \"university later recalled that he was often mentally jumbled and\"\n",
            "Probing prefix \"later recalled that he was often mentally jumbled and\"\n",
            "Probing prefix \"recalled that he was often mentally jumbled and\"\n",
            "Probing prefix \"that he was often mentally jumbled and\"\n",
            "Full token list for prefix \"that he was often mentally jumbled and\": [('disorganized', 1)]\n",
            "Top-k tokens for prefix \"that he was often mentally jumbled and\": [('disorganized', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he was often mentally jumbled and disorganized\"\n",
            "Probing prefix \"president of the university later recalled that he was often mentally jumbled and disorganized\"\n",
            "Probing prefix \"of the university later recalled that he was often mentally jumbled and disorganized\"\n",
            "Probing prefix \"the university later recalled that he was often mentally jumbled and disorganized\"\n",
            "Probing prefix \"university later recalled that he was often mentally jumbled and disorganized\"\n",
            "Probing prefix \"later recalled that he was often mentally jumbled and disorganized\"\n",
            "Probing prefix \"recalled that he was often mentally jumbled and disorganized\"\n",
            "Probing prefix \"that he was often mentally jumbled and disorganized\"\n",
            "Probing prefix \"he was often mentally jumbled and disorganized\"\n",
            "Full token list for prefix \"he was often mentally jumbled and disorganized\": [('near', 1)]\n",
            "Top-k tokens for prefix \"he was often mentally jumbled and disorganized\": [('near', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he was often mentally jumbled and disorganized near\"\n",
            "Probing prefix \"president of the university later recalled that he was often mentally jumbled and disorganized near\"\n",
            "Probing prefix \"of the university later recalled that he was often mentally jumbled and disorganized near\"\n",
            "Probing prefix \"the university later recalled that he was often mentally jumbled and disorganized near\"\n",
            "Probing prefix \"university later recalled that he was often mentally jumbled and disorganized near\"\n",
            "Probing prefix \"later recalled that he was often mentally jumbled and disorganized near\"\n",
            "Probing prefix \"recalled that he was often mentally jumbled and disorganized near\"\n",
            "Probing prefix \"that he was often mentally jumbled and disorganized near\"\n",
            "Probing prefix \"he was often mentally jumbled and disorganized near\"\n",
            "Probing prefix \"was often mentally jumbled and disorganized near\"\n",
            "Full token list for prefix \"was often mentally jumbled and disorganized near\": [('the', 1)]\n",
            "Top-k tokens for prefix \"was often mentally jumbled and disorganized near\": [('the', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he was often mentally jumbled and disorganized near the\"\n",
            "Probing prefix \"president of the university later recalled that he was often mentally jumbled and disorganized near the\"\n",
            "Probing prefix \"of the university later recalled that he was often mentally jumbled and disorganized near the\"\n",
            "Probing prefix \"the university later recalled that he was often mentally jumbled and disorganized near the\"\n",
            "Probing prefix \"university later recalled that he was often mentally jumbled and disorganized near the\"\n",
            "Probing prefix \"later recalled that he was often mentally jumbled and disorganized near the\"\n",
            "Probing prefix \"recalled that he was often mentally jumbled and disorganized near the\"\n",
            "Probing prefix \"that he was often mentally jumbled and disorganized near the\"\n",
            "Probing prefix \"he was often mentally jumbled and disorganized near the\"\n",
            "Probing prefix \"was often mentally jumbled and disorganized near the\"\n",
            "Probing prefix \"often mentally jumbled and disorganized near the\"\n",
            "Full token list for prefix \"often mentally jumbled and disorganized near the\": [('end', 1)]\n",
            "Top-k tokens for prefix \"often mentally jumbled and disorganized near the\": [('end', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he was often mentally jumbled and disorganized near the end\"\n",
            "Probing prefix \"president of the university later recalled that he was often mentally jumbled and disorganized near the end\"\n",
            "Probing prefix \"of the university later recalled that he was often mentally jumbled and disorganized near the end\"\n",
            "Probing prefix \"the university later recalled that he was often mentally jumbled and disorganized near the end\"\n",
            "Probing prefix \"university later recalled that he was often mentally jumbled and disorganized near the end\"\n",
            "Probing prefix \"later recalled that he was often mentally jumbled and disorganized near the end\"\n",
            "Probing prefix \"recalled that he was often mentally jumbled and disorganized near the end\"\n",
            "Probing prefix \"that he was often mentally jumbled and disorganized near the end\"\n",
            "Probing prefix \"he was often mentally jumbled and disorganized near the end\"\n",
            "Probing prefix \"was often mentally jumbled and disorganized near the end\"\n",
            "Probing prefix \"often mentally jumbled and disorganized near the end\"\n",
            "Probing prefix \"mentally jumbled and disorganized near the end\"\n",
            "Full token list for prefix \"mentally jumbled and disorganized near the end\": [('of', 1)]\n",
            "Top-k tokens for prefix \"mentally jumbled and disorganized near the end\": [('of', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he was often mentally jumbled and disorganized near the end of\"\n",
            "Probing prefix \"president of the university later recalled that he was often mentally jumbled and disorganized near the end of\"\n",
            "Probing prefix \"of the university later recalled that he was often mentally jumbled and disorganized near the end of\"\n",
            "Probing prefix \"the university later recalled that he was often mentally jumbled and disorganized near the end of\"\n",
            "Probing prefix \"university later recalled that he was often mentally jumbled and disorganized near the end of\"\n",
            "Probing prefix \"later recalled that he was often mentally jumbled and disorganized near the end of\"\n",
            "Probing prefix \"recalled that he was often mentally jumbled and disorganized near the end of\"\n",
            "Probing prefix \"that he was often mentally jumbled and disorganized near the end of\"\n",
            "Probing prefix \"he was often mentally jumbled and disorganized near the end of\"\n",
            "Probing prefix \"was often mentally jumbled and disorganized near the end of\"\n",
            "Probing prefix \"often mentally jumbled and disorganized near the end of\"\n",
            "Probing prefix \"mentally jumbled and disorganized near the end of\"\n",
            "Probing prefix \"jumbled and disorganized near the end of\"\n",
            "Full token list for prefix \"jumbled and disorganized near the end of\": [('his', 1)]\n",
            "Top-k tokens for prefix \"jumbled and disorganized near the end of\": [('his', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he was often mentally jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"president of the university later recalled that he was often mentally jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"of the university later recalled that he was often mentally jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"the university later recalled that he was often mentally jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"university later recalled that he was often mentally jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"later recalled that he was often mentally jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"recalled that he was often mentally jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"that he was often mentally jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"he was often mentally jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"was often mentally jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"often mentally jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"mentally jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"jumbled and disorganized near the end of his\"\n",
            "Probing prefix \"and disorganized near the end of his\"\n",
            "Full token list for prefix \"and disorganized near the end of his\": [('employment', 1)]\n",
            "Top-k tokens for prefix \"and disorganized near the end of his\": [('employment', 1)]\n",
            "Probing prefix \"The president of the university later recalled that he was often mentally jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"president of the university later recalled that he was often mentally jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"of the university later recalled that he was often mentally jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"the university later recalled that he was often mentally jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"university later recalled that he was often mentally jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"later recalled that he was often mentally jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"recalled that he was often mentally jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"that he was often mentally jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"he was often mentally jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"was often mentally jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"often mentally jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"mentally jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"jumbled and disorganized near the end of his employment\"\n",
            "Probing prefix \"and disorganized near the end of his employment\"\n",
            "Probing prefix \"disorganized near the end of his employment\"\n",
            "Full token list for prefix \"disorganized near the end of his employment\": [('.', 1)]\n",
            "Top-k tokens for prefix \"disorganized near the end of his employment\": [('.', 1)]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'The president of the university later recalled that he was often mentally jumbled and disorganized near the end of his employment .'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "From the debug output, we can see that the model has memorized longer sequences, which it will faithfully reproduce, for lack of alternatives. Let's try a few more."
      ],
      "metadata": {
        "id": "rw1bbu2Xa1gZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "complete_lm(model, \"Cassini\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "G8_MF4dYvcnh",
        "outputId": "28afc2c2-0dc7-4195-adb0-d6a131ff0974"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Cassini probe approached the planet in 2000 and took very detailed images of its atmosphere .'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "complete_lm(model, \"Schwarzenegger\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "pAsehkV3vnkK",
        "outputId": "7185de7b-908e-4e25-c7f2-3dc21429ed83"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Schwarzenegger character Terminator , he prompts Leslie to do that impression as well .'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "complete_lm(model, \"And then the\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "XTFtX49jvszX",
        "outputId": "e47f67e6-6100-41be-9e36-fe2f93e6057f"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'And then the federal Treasury stopped paying out gold at face value .'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "complete_lm(model, \"Aliens have\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "1oEhAfl_v4zK",
        "outputId": "82a9efcc-2240-4805-a3e4-8e9d2f20ad90"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Aliens have to wait `` , referring to a scene in the `` New York `` episode where Kurt discussed the planned move with Rachel .'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let us contrast this with a large language model."
      ],
      "metadata": {
        "id": "Mytw9IjEvc6y"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Playing with pre-trained large language models\n",
        "Even our small n-gram language model takes up considerable amount of RAM and a surprising amount of training time.\n",
        "\n",
        "More capable large language models take many orders of magnitude more training time (days to months), as they are trained on trillions of tokens instead of just a few hundred thousand tokens. Their architecture is much more complex, with the number of model parameters ranging from a few hundred millions (BERT) to a trillion (like Google Brain's [latest entry](https://aibusiness.com/nlp/google-brain-unveils-trillion-parameter-ai-language-model-the-largest-yet)). They are also not just learning simple n-gram statistics, but complex latent variables, that take into account linguistic and semantic properties of text at a much, much deeper level.\n",
        "\n",
        "As such, the community around large language models has started the trend of publishing **pre-trained large language models**, which allows organizations and individuals without the necessary resources to use and adapt these models.\n",
        "\n",
        "A popular hub for exchanging models is [Hugging Face](https://huggingface.co/models), where pre-trained, **open-weights models** like Llama, Mistral, and others can be found. The models are published togehter with the model architecture, which allows them to be run via [Hugging Face transformers](https://huggingface.co/docs/transformers/en/index) and similar libraries and frameworks.\n",
        "\n",
        "We will dive into the details for large language model architecture later. For now, we want to load a \"small\" large language model and compare its output with our own language model above. Ultimately, large language models and our punny n-gram do the same thing: predict the next token for a sequence.\n",
        "\n",
        "## GPU support\n",
        "Running such large language models on the CPU is possible, but not the most enjoybale past time in terms of speed. For any productive work, GPU support is mandatory.\n",
        "\n",
        "To run a large language model on the GPU in Google Colab, we need to select a runtime with GPU support, like T4 (free), A100 (paid), or V100 (paid). Click on `Runtime -> Change Runtime` above, switch to a GPU runtime, and restart the session."
      ],
      "metadata": {
        "id": "2Y5SYNmDbCha"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Quantization\n",
        "![](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/blog/bitsandbytes/FP8-scheme.png)\n",
        "\n",
        "The free Google Colab T4 runtime features a GPU with 16GB of RAM. How much RAM a model requires can be ballparked via its parameter count.\n",
        "\n",
        "The number of parameters is usually part of the model name, e.g. the Mistral-7B model has 7 billion parameters. How big is one parameter? All of these models are made up of deep neural networks. Like the one we built in the supervised learning section. One parameter is a weight (or bias) in those deep neural networks. Weights and biases are encoded as floating point numbers. Floating point numbers can have a bit depth ranging from [4-bit](https://arxiv.org/abs/2310.16836) to 64-bit. Most open-weight models are published with [16-bit wide float](https://en.wikipedia.org/wiki/Half-precision_floating-point_format) or [bfloat](https://en.wikipedia.org/wiki/Bfloat16_floating-point_format) weights and biases.\n",
        "\n",
        "A 7B model with weights encoded as 16-bit (b)float thus requires at a minimum 14GB of RAM. This does not include working memory required to hold auxiliary data.\n",
        "\n",
        "Getting a model to fit into a consumer grade GPU, or a Google Colab T4 instance is hard to impossible.\n",
        "\n",
        "Thus, pracitioners have come up with quantization schemes that reduce the bit-depth of model weights to 8- or even 4-bits. This reduces memory requirements by 2x-3x, and can also speeds up inference. Of course, there are also downsides: since the weights are compressed lossily, the model may suffer some quality degradation. For many models, this degradation is often acceptable though, especially during development.\n",
        "\n",
        "Once such quantization method is called [AutoAWQ](https://github.com/casper-hansen/AutoAWQ). It quantizes model weights to 4-bit, while trying to maintain as much fidelity as possible. AutoAWQ is compatible with the Huggin Face Transformers library, which we'll use to load and use AWQ quantized LLMs.\n",
        "\n",
        "Let's install AutoAWQ:\n"
      ],
      "metadata": {
        "id": "Cdd4Q56Bf5Ho"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip -q install autoawq"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a0yRwCaPKkY4",
        "outputId": "87a478bd-5e27-4a09-bc90-391e6a9013f6"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.8/76.8 kB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m280.0/280.0 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m38.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m33.4/33.4 MB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Loading Mistral-7B\n",
        "A handful of notorious users on Hugging Face seem to have it made their lifegoal to quantize any and all open-weight LLM models in existance. The most prominent is \"TheBloke\", who also provides an AWQ quantized version of the [Mistral-7B model](https://huggingface.co/TheBloke/Mistral-7B-v0.1-AWQ).\n",
        "\n",
        "Mistral is a family of LLMs by [Mistral AI](https://mistral.ai/contact/), a french company working in the AI space. Mistral models are regularly at the top of [LLM model benchmarks](https://huggingface.co/spaces/lmsys/chatbot-arena-leaderboard).\n",
        "\n",
        "Like most LLM families, Mistral comes in [several flavours](https://huggingface.co/mistralai). Generally, we differentiate between pre-trained models with and without **instruction fine-tuning**. Models without instruction fine-tuning are sort of \"raw\". They can generally not follow instructions given to them, but will merely try to come up with the most likely next token, given a sequence. Just like our n-gram model above. Just way smarter.\n",
        "\n",
        "Let's load the AWQ quantized Mistral-7B from Hugging Face. To do so, we use the Hugging Face Transformers class `AutoModelForCausalLLM`. The class name already tells us, what the class does.\n",
        "\n",
        "`AutoModel` means, that the class knows how to load a specific model's architecture and its corresponding parameters.\n",
        "\n",
        "The `ForCausalLLM` part means that the model to be loaded is expected to be a large language model for next word prediction (also known as causal language modeling). Check out the [Auto Classes](https://huggingface.co/docs/transformers/model_doc/auto) documentation for more details.\n",
        "\n",
        "The `AutoModelForCausalLM` class has a static method `from_pretrained()` which takes the identifier of a causal large language model to be downloaded from the Hugging Face Hub. The identifier is composed of the name of the user or organization that has uploaded the model to the hub, e.g. \"TheBloke\", and the model name, e.g. \"Mistral-7B-v0.1-AWQ\", separated by a slash. You can also specify a local directory instead.\n",
        "\n",
        "The method will cache the model locally, so subsequent loads are faster. Once downloaded, the `from_pretrained` method then instantiates the concrete model class, in our case [MistralForCausalLM](https://huggingface.co/docs/transformers/v4.38.1/en/model_doc/mistral#transformers.MistralForCausalLM). This class knows how to instantiate all the modules and layers of the model. Once instantiated, the model parameters from the downloaded (or local) model file(s) are loaded into the model.\n",
        "\n",
        "The model can be instantiated CPU-side, or GPU-side. We can let the `from_pretrained` method pick the best option by passing `device_map=\"cuda\"` as a parameter. [CUDA](https://developer.nvidia.com/cuda-zone) by NVIDIA is the defacto framework when it comes to doing computations on a GPU. It also only works for NVIDIA GPUs. Setting the parameter to \"cuda\" will make the method try to put the model on a CUDA enabled GPU if available.\n",
        "\n",
        "A lot of words for 3 lines of code. Here's how we load the model:\n"
      ],
      "metadata": {
        "id": "9t5t8QC7l_rC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoModelForCausalLM\n",
        "\n",
        "model_name = \"TheBloke/Mistral-7B-v0.1-AWQ\"\n",
        "llm = AutoModelForCausalLM.from_pretrained(model_name, device_map=\"cuda\")"
      ],
      "metadata": {
        "id": "zzCQdsVO5Elb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "5e8a51d913e540769b5c2af95660484b",
            "fffae0d2d03444f2b99c89c6e9bee971",
            "93e78ce6d7524d379865065ec880ec53",
            "2242bfbe777649dab1e28404aa3d1dd5",
            "9d09fa14b84b4d0d8cb9dfc6e548a197",
            "2a02295b5353448e9ed4b2ee87c478a2",
            "f7dd3fcf382d49ce9055875f717a9841",
            "f13d35d41b1c4c35bccd1b3d7a3da829",
            "e701e154ccfe453eadc38a8161b5e3e2",
            "1751341bc4214b1abf34921728acbcf8",
            "283b4d8c3d8b4cce90d8a5690f4c795b",
            "5397da56986f48e5b412f85df5f33ee2",
            "4807b7cd22824426958430cb4551f2a4",
            "0ad0d0bf6dc242db92177032ea885cf4",
            "ac9a795df6524c73a3410d2a3d88171a",
            "6c78305129a84fd8905686c9f731201d",
            "6bb187303ab94d73aef80c0986f37131",
            "9b4e50cfa7ab435db2132806c4e1aee1",
            "92c50a5c77644b789ede48a9ab88b6ee",
            "c5463c26f41e47d6ba0285b316182072",
            "06169e68f2d94f6dbd51f8f00610730e",
            "6a72b73b5ec746289df213fff1626020",
            "b8228d9041c5491597c96401d65796dc",
            "4f25acab8b64415eaa189a88759ee039",
            "6278d7a52fde4bb68ef7be5e59fb052b",
            "07ae580ea6454ec9a1c4e58cb7bdeacf",
            "933717fa89354548bc1995d8f74d9cdd",
            "dd4c3d33776e4d5b89baa43b17e84c89",
            "c0481ed5b5694a97955b0baf46604a77",
            "30b9674faacb485ba6b4cc62f24cb51a",
            "4705d60711764ecd99f36b105281f4af",
            "bc5145fec71a45ff9820982bb6df5324",
            "fc1ef45dfbbf4d76bd05781b5479b5fc"
          ]
        },
        "outputId": "f5811c61-3a07-4247-c03f-9e4750fb0a15"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/757 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5e8a51d913e540769b5c2af95660484b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/4.15G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5397da56986f48e5b412f85df5f33ee2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/116 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b8228d9041c5491597c96401d65796dc"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Printing the model will tell us about its internal architecture. Constrast and compare this to our own little deepish neural network."
      ],
      "metadata": {
        "id": "LzxTOuAxsvGf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(llm)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z3KdblJcBuGk",
        "outputId": "f15979c8-63d9-421d-8b4e-4e387728801e"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "MistralForCausalLM(\n",
            "  (model): MistralModel(\n",
            "    (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n",
            "    (layers): ModuleList(\n",
            "      (0-31): 32 x MistralDecoderLayer(\n",
            "        (self_attn): MistralAttention(\n",
            "          (q_proj): WQLinear_GEMM(in_features=4096, out_features=4096, bias=False, w_bit=4, group_size=128)\n",
            "          (k_proj): WQLinear_GEMM(in_features=4096, out_features=1024, bias=False, w_bit=4, group_size=128)\n",
            "          (v_proj): WQLinear_GEMM(in_features=4096, out_features=1024, bias=False, w_bit=4, group_size=128)\n",
            "          (o_proj): WQLinear_GEMM(in_features=4096, out_features=4096, bias=False, w_bit=4, group_size=128)\n",
            "          (rotary_emb): MistralRotaryEmbedding()\n",
            "        )\n",
            "        (mlp): MistralMLP(\n",
            "          (gate_proj): WQLinear_GEMM(in_features=4096, out_features=14336, bias=False, w_bit=4, group_size=128)\n",
            "          (up_proj): WQLinear_GEMM(in_features=4096, out_features=14336, bias=False, w_bit=4, group_size=128)\n",
            "          (down_proj): WQLinear_GEMM(in_features=14336, out_features=4096, bias=False, w_bit=4, group_size=128)\n",
            "          (act_fn): SiLU()\n",
            "        )\n",
            "        (input_layernorm): MistralRMSNorm()\n",
            "        (post_attention_layernorm): MistralRMSNorm()\n",
            "      )\n",
            "    )\n",
            "    (norm): MistralRMSNorm()\n",
            "  )\n",
            "  (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Tokenization\n",
        "While large language models deal with text, they can not directly work with direct text input. Just as in case of our n-gram model, we need to use a tokenizer to preprocess any text we want to pass to the large language model.\n",
        "\n",
        "Tokenizers for large language model are a science in their own right. We've already mentioned [binary pair encoding](https://en.wikipedia.org/wiki/Byte_pair_encoding) and many other tokenizers, which do not operate on word boundaries, like in our n-gram case.\n",
        "\n",
        "Whatever model we use, we must ensure to use the same tokenizer that was used for pre-training it. Thankfully, the information which tokenizer is needed for a model is part of the model configuration. Which means, we can automatically load it, just like we did with with the model itself. The Hugging Face Transformers library has another auto class for that, `AutoTokenizer`. Here is how we load the tokenizer for the mistral model:"
      ],
      "metadata": {
        "id": "IR5fUjXCs5lD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)"
      ],
      "metadata": {
        "id": "9C0B71F1yN4c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177,
          "referenced_widgets": [
            "5574c9b17b7b4a4e9bb87e3d03e0dc13",
            "33748abc61c940aaac11d6dccaa9f1b0",
            "ed5a797a35da4491b3c075fbb33b52f6",
            "f39d8173026b4023b8be9a1bd8def980",
            "c2a678410ac64b3bb4ed8cb2e19e0e10",
            "038ae9d6da14495db8a362184b045f37",
            "716f23bb032d4773aedb5bb250d8070a",
            "89d6b4c10cc846b7ac400372a59d3ba8",
            "a0fd4fd17b864f93823cbd3ff7b1627e",
            "f11754700fcd49e5b8c4365b8271afe1",
            "5845dbe4675c414faa2670a06d6b785a",
            "800d96d21ef64899b3183312cc0eb1fe",
            "099e0fccfa4b4cafbb4a5026ace9839a",
            "a93c96cf1d6c4abeaf8706fcb9969a3c",
            "2ca5a801d6f04629b7fdd1d5e7ab307e",
            "6f236be00cdc42fcac662c7d43c7b672",
            "00ab8ec5b2de431d97678894213c142c",
            "2710aaebaa8b497c9924898212fbaa9f",
            "5f1b929fd4c64f27a8bd64e60fa89e02",
            "08643a3fdb4940c688bce7f5c406c320",
            "d115ff9c42464306a02229ea3625bb11",
            "a0416e48e00d45dfa9dba46b3eef39bd",
            "9953bfe8affd4a8ba74894d1787b7372",
            "e258b238299d41ffb9d2ffe63faf16c8",
            "b0a64f0f7a3e4fd595e8b1a487b6d323",
            "72798c51abc04899b4d0fde0739bae07",
            "4ca1327667084b1d99fc6cb1dda7c3c9",
            "5e84091bc7514e76b200a4db23e5de15",
            "73f25d8e7e194ac3af759afbe54385ea",
            "15cf767ff4374f06b6ccafea00daff0b",
            "db9a9a91c2e941a7a68f1b2387ba2b32",
            "2d7bf1a1a9984970976458ea087de707",
            "cfd896d3647b4936b090a9a37a452b49",
            "1f88dd4d9a2d48caaf78b50536792c37",
            "379bc80fbbc8479384a73c4c6449eb68",
            "f66048d816f04d6dbbcfbed995e8716e",
            "af1ceec3e66b44b083c2983d26a46068",
            "5fdd96706c6c43be9219d97e84e76a43",
            "d3f78544d98a4431a5dcda77083f4727",
            "b1d10cb261f94e17829847c783f627a2",
            "2d7198b84d184ce9b40937efaa930d35",
            "0e04f2bf60f94b798e230035e5ac94d9",
            "f8073adc11c7492299fe5c5e0868d030",
            "0a6f177b9763493996bb5c5658048f8f",
            "3f89d6b0190d4891a933fb0c1e67255f",
            "24656013428b4a7e961cbc29a837f9ea",
            "ae38d05fbc05469d96dbe6fa039dd9df",
            "4748a9f59cb74ef4897b96401fcfc968",
            "9d494edf5a8f48f184a251ca36dcbf96",
            "fbf13624fbec4c9ead029be700634fec",
            "fa50ec515d144cabb4138c8e84c1927d",
            "b64332d4bd1846e6b16519b2bf92c973",
            "159efa92982c41c99b4da656564ed8b3",
            "1deac6009ee04d40aa66e20c606b22c0",
            "639c1f47215a44bda4eaca62e5b889ae"
          ]
        },
        "outputId": "f12be5a0-3958-4643-87bc-9e75518bb092"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/962 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5574c9b17b7b4a4e9bb87e3d03e0dc13"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.model:   0%|          | 0.00/493k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "800d96d21ef64899b3183312cc0eb1fe"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.80M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9953bfe8affd4a8ba74894d1787b7372"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "added_tokens.json:   0%|          | 0.00/42.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1f88dd4d9a2d48caaf78b50536792c37"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/72.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3f89d6b0190d4891a933fb0c1e67255f"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Tokenizing a text string is equally simple:"
      ],
      "metadata": {
        "id": "J0IoPdAqyaZb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokens = tokenizer(\"Peach is in a different castle\")\n",
        "tokens"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CJUnh6Icyg4E",
        "outputId": "3905ef14-53a2-48db-a63d-49d98f83c211"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [1, 3242, 595, 349, 297, 264, 1581, 19007], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1]}"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The tokenizer returns a set of ids. The first id with value `1` signals the start of the sequence. The remaining ids map to one token from the vocabulary the tokenizer has. This tokenizer has a total of 32000 tokens in its vocabulary. When a string is tokenized, it gets broken down into ids from that vocabulary.\n",
        "\n",
        "We can decode the ids back to text:"
      ],
      "metadata": {
        "id": "aIW1JaWzyoTz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(len(tokens[\"input_ids\"])):\n",
        "  print(tokenizer.decode(tokens[\"input_ids\"][i]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z4jy3zmQy6bp",
        "outputId": "2c1a18d5-5dc4-4305-ce11-4d2adfb9eeb1"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<s>\n",
            "Pe\n",
            "ach\n",
            "is\n",
            "in\n",
            "a\n",
            "different\n",
            "castle\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Where `<s>` is the textual encoding for the special \"beginning of stream\" token.\n",
        "\n",
        "The `attention_mask` is used to tell the model which of the tokens it can ignore (`0`) and which tokens it should consider (`1`). For our purpose, we'll always have an attention mask full of `1`s when passing input to the LLM.\n",
        "\n",
        "The values returned by the `tokenizer` call above live CPU-side. However, our model lives GPU-side. We need to invoke the tokenizer with an additional parameter `return_tensors`, so it returns PyTorch tensors. We then transfer all those tensors to the GPU via `value.to(\"cuda\")`."
      ],
      "metadata": {
        "id": "1vt19NEJzQBV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize_llm(input, max_length=4096):\n",
        "  inputs = tokenizer(input, return_tensors=\"pt\", max_length=1024, truncation=True);\n",
        "  inputs = {key: value.to(\"cuda\") for key, value in inputs.items()}\n",
        "  return inputs\n",
        "\n",
        "tokenize_llm(\"Peach is in a different castle\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IiBOJ-jCz6Np",
        "outputId": "fc5e6375-5e17-4471-cba3-85cc1ebdf82c"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': tensor([[    1,  3242,   595,   349,   297,   264,  1581, 19007]],\n",
              "        device='cuda:0'),\n",
              " 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1]], device='cuda:0')}"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The input ids and attention mask have been converted to PyTorch tensors, which live on the GPU. We are ready to pass input to the LLM for it to predict the next word in a sequence."
      ],
      "metadata": {
        "id": "7EMuBjQi0zS0"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Logits and predicting the next word\n",
        "Let's start by tokenizing an input we also used with our n-gram model:"
      ],
      "metadata": {
        "id": "MI0KelXC0Nng"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "input = tokenize_llm(\"Cassini\")\n",
        "input"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RUwG90x81Iup",
        "outputId": "534dde52-203b-4c5c-94e4-eb9990652070"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': tensor([[    1, 13367,  3494]], device='cuda:0'),\n",
              " 'attention_mask': tensor([[1, 1, 1]], device='cuda:0')}"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The `MistralModelForCausalLM` instance is actually a PyTorch `Module`, like our deep-ish neural network from the previous section. As such, we can make it predict output via:"
      ],
      "metadata": {
        "id": "WRlMAFL21Sir"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "llm.eval()\n",
        "output = llm(**input)\n",
        "print(type(output))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KdumclmZ1SF3",
        "outputId": "6365d7b5-e071-4037-dfd7-b1ddbae84b6f"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'transformers.modeling_outputs.CausalLMOutputWithPast'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The function returns an instance of [CausalLMOutputWithPast](https://huggingface.co/docs/transformers/v4.38.1/en/main_classes/output#transformers.modeling_outputs.CausalLMOutputWithPast). We are only interested in the `logits`, so lets have a look at what they are."
      ],
      "metadata": {
        "id": "76vqaGTa3RGe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "output.logits.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CeRw830w3fiX",
        "outputId": "d8198121-404c-4322-f313-e2a2242be458"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 3, 32000])"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "It's a PyTorch tensor, with a very weird shape. Let's break it down.\n",
        "\n",
        "The first dimension of size `1` is the batch size, which is equivalent to the number of inputs we submitted. We only submitted a single input, so its size is 1. In production, we'd submit multiple inputs at once, which is called batching. This way we use all (or most) of the available resources in parallel.\n",
        "\n",
        "The next dimension (`3`) is the sequence length. We passed in the tokens for `Cassini`, which got tokenized to `3` tokens (including the beginning of stream token with id `1`).\n",
        "\n",
        "The final dimension has size `32000`. That is equivalent to the number of tokens in the vocabulary of the tokenizer.\n",
        "\n",
        "We thus get one vector with 32000 dimensions for each input token. Each element in this vector corresponds to one token in the vocabulary the tokenizer understands. The index of the element is equal to the id of the corresponding token in the vocabulary. And the value of a vector element gives us the (unnormalized) log-probability that this token is the next token in the sequence after the current token! This (unnormalized) log-probability for a token is also called the **logit score**.\n",
        "\n",
        "This is very similar to the token list we fetch for a prefix in our n-gram model. Except, we do it for the longest prefix match, instead of for every token position in the input.\n",
        "\n",
        "We can visualize the top 5 predicted tokens for each position in the input sequence."
      ],
      "metadata": {
        "id": "hnyOKC4P3glL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "data = []\n",
        "for i in range(output.logits.shape[1]):\n",
        "    logits_for_position = output.logits[0, i, :]\n",
        "    topk_values, topk_indices = torch.topk(logits_for_position, 5)\n",
        "    topk_tokens_with_scores = [(tokenizer.decode([idx.item()], skip_special_tokens=True), score.item()) for idx, score in zip(topk_indices, topk_values)]\n",
        "    for rank, (token, score) in enumerate(topk_tokens_with_scores, start=1):\n",
        "        data.append({\n",
        "            \"Position\": i,\n",
        "            \"Input Token\": tokenizer.decode(input[\"input_ids\"][0][i]),\n",
        "            \"Predicted Next Token\": token,\n",
        "            \"Logit Score\": score\n",
        "        })\n",
        "pd.DataFrame(data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 519
        },
        "id": "B-9eXNoA6QMZ",
        "outputId": "5e0aa2b8-7fc6-4cd2-9265-9664825f750b"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    Position Input Token Predicted Next Token  Logit Score\n",
              "0          0         <s>                    #    12.250000\n",
              "1          0         <s>                   ##    11.101562\n",
              "2          0         <s>                  ###    10.703125\n",
              "3          0         <s>                  The    10.062500\n",
              "4          0         <s>                 User    10.039062\n",
              "5          1        Cass                   ie    11.312500\n",
              "6          1        Cass                andra    11.164062\n",
              "7          1        Cass                  ini    10.843750\n",
              "8          1        Cass                  idy    10.445312\n",
              "9          1        Cass                  per    10.351562\n",
              "10         2         ini                 Sign    11.593750\n",
              "11         2         ini                   is    10.632812\n",
              "12         2         ini                    ’    10.000000\n",
              "13         2         ini                    '     9.671875\n",
              "14         2         ini                    -     9.460938"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-23497b09-1fc1-4067-96a6-142ebacbb59f\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Position</th>\n",
              "      <th>Input Token</th>\n",
              "      <th>Predicted Next Token</th>\n",
              "      <th>Logit Score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>&lt;s&gt;</td>\n",
              "      <td>#</td>\n",
              "      <td>12.250000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>&lt;s&gt;</td>\n",
              "      <td>##</td>\n",
              "      <td>11.101562</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>&lt;s&gt;</td>\n",
              "      <td>###</td>\n",
              "      <td>10.703125</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>&lt;s&gt;</td>\n",
              "      <td>The</td>\n",
              "      <td>10.062500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>&lt;s&gt;</td>\n",
              "      <td>User</td>\n",
              "      <td>10.039062</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1</td>\n",
              "      <td>Cass</td>\n",
              "      <td>ie</td>\n",
              "      <td>11.312500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1</td>\n",
              "      <td>Cass</td>\n",
              "      <td>andra</td>\n",
              "      <td>11.164062</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>1</td>\n",
              "      <td>Cass</td>\n",
              "      <td>ini</td>\n",
              "      <td>10.843750</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>1</td>\n",
              "      <td>Cass</td>\n",
              "      <td>idy</td>\n",
              "      <td>10.445312</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1</td>\n",
              "      <td>Cass</td>\n",
              "      <td>per</td>\n",
              "      <td>10.351562</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>2</td>\n",
              "      <td>ini</td>\n",
              "      <td>Sign</td>\n",
              "      <td>11.593750</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>2</td>\n",
              "      <td>ini</td>\n",
              "      <td>is</td>\n",
              "      <td>10.632812</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>2</td>\n",
              "      <td>ini</td>\n",
              "      <td>’</td>\n",
              "      <td>10.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>2</td>\n",
              "      <td>ini</td>\n",
              "      <td>'</td>\n",
              "      <td>9.671875</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>2</td>\n",
              "      <td>ini</td>\n",
              "      <td>-</td>\n",
              "      <td>9.460938</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-23497b09-1fc1-4067-96a6-142ebacbb59f')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-23497b09-1fc1-4067-96a6-142ebacbb59f button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-23497b09-1fc1-4067-96a6-142ebacbb59f');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-900ea703-1d7b-483e-9f85-dbb336792211\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-900ea703-1d7b-483e-9f85-dbb336792211')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-900ea703-1d7b-483e-9f85-dbb336792211 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"pd\",\n  \"rows\": 15,\n  \"fields\": [\n    {\n      \"column\": \"Position\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 2,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          0,\n          1,\n          2\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Input Token\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"<s>\",\n          \"Cass\",\n          \"ini\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Predicted Next Token\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 15,\n        \"samples\": [\n          \"per\",\n          \"is\",\n          \"#\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Logit Score\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.7578136505882282,\n        \"min\": 9.4609375,\n        \"max\": 12.25,\n        \"num_unique_values\": 15,\n        \"samples\": [\n          10.3515625,\n          10.6328125,\n          12.25\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "A few interesting observations:\n",
        "\n",
        "* For the beginning of stream token `<s>`, that is for an empty input sequence, the model really wants to output \"#\", \"##\", \"###\". These are clearly markdown headings, which were likely very abundant in the training data.\n",
        "* For the next token `Cass`, the model thinks common names like `Cassie` or `Cassandra` would be appropriate.\n",
        "* For the last token in our input sequence, `ini`, the model predicts `Sign` as the most likely next token, followed by `is`\n",
        "\n",
        "For our final trick, we'll write a `complete_llm` function, that completes an input sequence up to a maximum number of generated tokens. We'll use the model's `generate` method for that, to make our life a little easier."
      ],
      "metadata": {
        "id": "-UJ8hDi2606o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def complete_llm(llm, input, max_length = 40):\n",
        "    input = tokenize_llm(input)\n",
        "    output_ids = llm.generate(**input, max_length=max_length, num_return_sequences=1)\n",
        "    generated_text = tokenizer.batch_decode(output_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]\n",
        "\n",
        "    return generated_text"
      ],
      "metadata": {
        "id": "gGUl-BjzAEh_"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "complete_llm(llm, \"The Cassini probe\")"
      ],
      "metadata": {
        "id": "hnxEYi0iAoPh",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "d64b0b1f-a90e-4e9d-9897-e67f33086687"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'The Cassini probe has been orbiting Saturn for over a decade, and has been sending back some amazing images of the planet and its moons. But the probe is running out of fuel'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "complete_llm(llm, \"And then the\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "id": "m_faKJan89qr",
        "outputId": "daa74ea7-139d-4088-cd8f-08ee396656f8"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'And then the world changed.\\n\\nI’m not talking about the pandemic. I’m talking about the murder of George Floyd.\\n\\nI’m not talking about the protests'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "complete_llm(llm, \"Aliens have\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "id": "E4g5nAu585S-",
        "outputId": "f5b6bed2-a0d3-4101-ecb0-903e7f922991"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Aliens have been a part of the Star Wars universe since the very beginning. In fact, the first Star Wars movie was originally titled Star Wars: Episode IV – The Star Wars.\\n\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can also check logits on OpenAI's playground.\n",
        "\n",
        "![](https://marioslab.io/uploads/genai/openai-logits.png)"
      ],
      "metadata": {
        "id": "GRdoZysIN2b5"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Running LLMs locally\n",
        "If you have a PC with a recent NVIDIA GPU and enough VRAM (Minimum 8GB) or a MacBook with an M1 or above (Minimum 16GB), you can try to run these models on your machine as well.\n",
        "\n",
        "There are different ways to run the models. All of them fetch the models from Hugging Face.\n",
        "\n",
        "* [LM Studio](https://lmstudio.ai/) a comprehensive GUI application that makes downloading and chatting with open-weight LLMs simple. Also supports serving models via an OpenAI compatible REST API.\n",
        "* [Ollama](https://ollama.com/), more or less a command line tool to quickly pull models from Hugging Face and run them locally. Also allows serving those files via a REST API that is compatible with the OpenAI API.\n",
        "* [llama.cpp](https://github.com/ggerganov/llama.cpp), a C++ library to load and run many open-weight models. Comes with its own file format (GGUF), and includes CLI tools for all kinds of tasks. Models can also be served via an OpenAI compatible REST API through [llama-cpp-python](https://llama-cpp-python.readthedocs.io/en/latest/server/).\n",
        "\n",
        "llama.cpp is the underpinning of many local open-weight model runners, including LM Studio and Ollama. It recently got support for [**LoRA fine-tuning**](https://rentry.org/cpu-lora) though only on the CPU, which is ... not great.\n",
        "\n",
        "If you use Ollama, you can get a local OpenAI compatible server running as follows:\n",
        "\n",
        "```\n",
        "ollama pull mixtral:latest\n",
        "ollama serve\n",
        "```\n",
        "\n",
        "You can then use the OpenAI Python module to talk to your local model or an OpenAI model, and thereby compare and contrast the performance of both:\n",
        "\n",
        "```\n",
        "use_local_model = True\n",
        "if use_local_model:\n",
        "  client = OpenAI(\n",
        "    base_url = 'http://localhost:11434/v1',\n",
        "    api_key='ollama', # required, but unused\n",
        "  )\n",
        "  model_name=\"mixtral:latest\"\n",
        "else:\n",
        "  client = OpenAI(api_key=\"<your OpenAI API key>\")\n",
        "  model_name=\"gpt-3.5-turbo\"\n",
        "\n",
        "# The rest of your code stays the same!\n",
        "completion = client.chat.completions.create(\n",
        "  model=model_name,\n",
        "  messages=[\n",
        "    {\"role\": \"system\", \"content\": \"You are a poetic assistant, skilled in explaining complex programming concepts with creative flair.\"},\n",
        "    {\"role\": \"user\", \"content\": \"Compose a poem that explains the concept of recursion in programming.\"}\n",
        "  ]\n",
        ")\n",
        "```\n",
        "\n",
        "You can use this trick for any OpenAI API based code in the subsequent sections!\n"
      ],
      "metadata": {
        "id": "cRgDEWNR-yHw"
      }
    }
  ]
}