jepa-vitl16_ink1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 71547,
     "status": "ok",
     "timestamp": 1719372316927,
     "user": {
      "displayName": "深川梓",
      "userId": "06822277665795432790"
     },
     "user_tz": -540
    },
    "id": "6Jp7uGzdU75A",
    "outputId": "209cb124-54bc-41d0-9fef-2fb283797ee2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/sagemaker-user/content/jepa\n",
      "running install\n",
      "/opt/conda/lib/python3.10/site-packages/setuptools/_distutils/cmd.py:66: SetuptoolsDeprecationWarning: setup.py install is deprecated.\n",
      "!!\n",
      "\n",
      "        ********************************************************************************\n",
      "        Please avoid running ``setup.py`` directly.\n",
      "        Instead, use pypa/build, pypa/installer or other\n",
      "        standards-based tools.\n",
      "\n",
      "        See https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html for details.\n",
      "        ********************************************************************************\n",
      "\n",
      "!!\n",
      "  self.initialize_options()\n",
      "/opt/conda/lib/python3.10/site-packages/setuptools/_distutils/cmd.py:66: EasyInstallDeprecationWarning: easy_install command is deprecated.\n",
      "!!\n",
      "\n",
      "        ********************************************************************************\n",
      "        Please avoid running ``setup.py`` and ``easy_install``.\n",
      "        Instead, use pypa/build, pypa/installer or other\n",
      "        standards-based tools.\n",
      "\n",
      "        See https://github.com/pypa/setuptools/issues/917 for details.\n",
      "        ********************************************************************************\n",
      "\n",
      "!!\n",
      "  self.initialize_options()\n",
      "running bdist_egg\n",
      "running egg_info\n",
      "writing src/jepa.egg-info/PKG-INFO\n",
      "writing dependency_links to src/jepa.egg-info/dependency_links.txt\n",
      "writing requirements to src/jepa.egg-info/requires.txt\n",
      "writing top-level names to src/jepa.egg-info/top_level.txt\n",
      "reading manifest file 'src/jepa.egg-info/SOURCES.txt'\n",
      "adding license file 'LICENSE'\n",
      "writing manifest file 'src/jepa.egg-info/SOURCES.txt'\n",
      "installing library code to build/bdist.linux-x86_64/egg\n",
      "running install_lib\n",
      "running build_py\n",
      "creating build\n",
      "creating build/lib\n",
      "creating build/lib/datasets\n",
      "copying src/datasets/data_manager.py -> build/lib/datasets\n",
      "copying src/datasets/image_dataset.py -> build/lib/datasets\n",
      "copying src/datasets/video_dataset.py -> build/lib/datasets\n",
      "creating build/lib/masks\n",
      "copying src/masks/default.py -> build/lib/masks\n",
      "copying src/masks/multiblock3d.py -> build/lib/masks\n",
      "copying src/masks/random_tube.py -> build/lib/masks\n",
      "copying src/masks/utils.py -> build/lib/masks\n",
      "creating build/lib/models\n",
      "copying src/models/attentive_pooler.py -> build/lib/models\n",
      "copying src/models/predictor.py -> build/lib/models\n",
      "copying src/models/vision_transformer.py -> build/lib/models\n",
      "creating build/lib/utils\n",
      "copying src/utils/distributed.py -> build/lib/utils\n",
      "copying src/utils/logging.py -> build/lib/utils\n",
      "copying src/utils/monitoring.py -> build/lib/utils\n",
      "copying src/utils/schedulers.py -> build/lib/utils\n",
      "copying src/utils/tensors.py -> build/lib/utils\n",
      "creating build/lib/datasets/utils\n",
      "copying src/datasets/utils/weighted_sampler.py -> build/lib/datasets/utils\n",
      "creating build/lib/datasets/utils/video\n",
      "copying src/datasets/utils/video/functional.py -> build/lib/datasets/utils/video\n",
      "copying src/datasets/utils/video/randaugment.py -> build/lib/datasets/utils/video\n",
      "copying src/datasets/utils/video/randerase.py -> build/lib/datasets/utils/video\n",
      "copying src/datasets/utils/video/transforms.py -> build/lib/datasets/utils/video\n",
      "copying src/datasets/utils/video/volume_transforms.py -> build/lib/datasets/utils/video\n",
      "creating build/lib/models/utils\n",
      "copying src/models/utils/modules.py -> build/lib/models/utils\n",
      "copying src/models/utils/multimask.py -> build/lib/models/utils\n",
      "copying src/models/utils/patch_embed.py -> build/lib/models/utils\n",
      "copying src/models/utils/pos_embs.py -> build/lib/models/utils\n",
      "creating build/bdist.linux-x86_64\n",
      "creating build/bdist.linux-x86_64/egg\n",
      "creating build/bdist.linux-x86_64/egg/datasets\n",
      "copying build/lib/datasets/data_manager.py -> build/bdist.linux-x86_64/egg/datasets\n",
      "copying build/lib/datasets/image_dataset.py -> build/bdist.linux-x86_64/egg/datasets\n",
      "copying build/lib/datasets/video_dataset.py -> build/bdist.linux-x86_64/egg/datasets\n",
      "creating build/bdist.linux-x86_64/egg/datasets/utils\n",
      "copying build/lib/datasets/utils/weighted_sampler.py -> build/bdist.linux-x86_64/egg/datasets/utils\n",
      "creating build/bdist.linux-x86_64/egg/datasets/utils/video\n",
      "copying build/lib/datasets/utils/video/functional.py -> build/bdist.linux-x86_64/egg/datasets/utils/video\n",
      "copying build/lib/datasets/utils/video/randaugment.py -> build/bdist.linux-x86_64/egg/datasets/utils/video\n",
      "copying build/lib/datasets/utils/video/randerase.py -> build/bdist.linux-x86_64/egg/datasets/utils/video\n",
      "copying build/lib/datasets/utils/video/transforms.py -> build/bdist.linux-x86_64/egg/datasets/utils/video\n",
      "copying build/lib/datasets/utils/video/volume_transforms.py -> build/bdist.linux-x86_64/egg/datasets/utils/video\n",
      "creating build/bdist.linux-x86_64/egg/masks\n",
      "copying build/lib/masks/default.py -> build/bdist.linux-x86_64/egg/masks\n",
      "copying build/lib/masks/multiblock3d.py -> build/bdist.linux-x86_64/egg/masks\n",
      "copying build/lib/masks/random_tube.py -> build/bdist.linux-x86_64/egg/masks\n",
      "copying build/lib/masks/utils.py -> build/bdist.linux-x86_64/egg/masks\n",
      "creating build/bdist.linux-x86_64/egg/models\n",
      "copying build/lib/models/attentive_pooler.py -> build/bdist.linux-x86_64/egg/models\n",
      "copying build/lib/models/predictor.py -> build/bdist.linux-x86_64/egg/models\n",
      "copying build/lib/models/vision_transformer.py -> build/bdist.linux-x86_64/egg/models\n",
      "creating build/bdist.linux-x86_64/egg/models/utils\n",
      "copying build/lib/models/utils/modules.py -> build/bdist.linux-x86_64/egg/models/utils\n",
      "copying build/lib/models/utils/multimask.py -> build/bdist.linux-x86_64/egg/models/utils\n",
      "copying build/lib/models/utils/patch_embed.py -> build/bdist.linux-x86_64/egg/models/utils\n",
      "copying build/lib/models/utils/pos_embs.py -> build/bdist.linux-x86_64/egg/models/utils\n",
      "creating build/bdist.linux-x86_64/egg/utils\n",
      "copying build/lib/utils/distributed.py -> build/bdist.linux-x86_64/egg/utils\n",
      "copying build/lib/utils/logging.py -> build/bdist.linux-x86_64/egg/utils\n",
      "copying build/lib/utils/monitoring.py -> build/bdist.linux-x86_64/egg/utils\n",
      "copying build/lib/utils/schedulers.py -> build/bdist.linux-x86_64/egg/utils\n",
      "copying build/lib/utils/tensors.py -> build/bdist.linux-x86_64/egg/utils\n",
      "byte-compiling build/bdist.linux-x86_64/egg/datasets/data_manager.py to data_manager.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/datasets/image_dataset.py to image_dataset.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/datasets/video_dataset.py to video_dataset.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/datasets/utils/weighted_sampler.py to weighted_sampler.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/datasets/utils/video/functional.py to functional.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/datasets/utils/video/randaugment.py to randaugment.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/datasets/utils/video/randerase.py to randerase.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/datasets/utils/video/transforms.py to transforms.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/datasets/utils/video/volume_transforms.py to volume_transforms.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/masks/default.py to default.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/masks/multiblock3d.py to multiblock3d.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/masks/random_tube.py to random_tube.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/masks/utils.py to utils.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/models/attentive_pooler.py to attentive_pooler.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/models/predictor.py to predictor.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/models/vision_transformer.py to vision_transformer.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/models/utils/modules.py to modules.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/models/utils/multimask.py to multimask.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/models/utils/patch_embed.py to patch_embed.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/models/utils/pos_embs.py to pos_embs.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/utils/distributed.py to distributed.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/utils/logging.py to logging.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/utils/monitoring.py to monitoring.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/utils/schedulers.py to schedulers.cpython-310.pyc\n",
      "byte-compiling build/bdist.linux-x86_64/egg/utils/tensors.py to tensors.cpython-310.pyc\n",
      "creating build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying src/jepa.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying src/jepa.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying src/jepa.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying src/jepa.egg-info/requires.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "copying src/jepa.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
      "zip_safe flag not set; analyzing archive contents...\n",
      "creating dist\n",
      "creating 'dist/jepa-0.0.1-py3.10.egg' and adding 'build/bdist.linux-x86_64/egg' to it\n",
      "removing 'build/bdist.linux-x86_64/egg' (and everything under it)\n",
      "Processing jepa-0.0.1-py3.10.egg\n",
      "Copying jepa-0.0.1-py3.10.egg to /opt/conda/lib/python3.10/site-packages\n",
      "Adding jepa 0.0.1 to easy-install.pth file\n",
      "\n",
      "Installed /opt/conda/lib/python3.10/site-packages/jepa-0.0.1-py3.10.egg\n",
      "Processing dependencies for jepa==0.0.1\n",
      "Searching for beartype\n",
      "Reading https://pypi.org/simple/beartype/\n",
      "Downloading https://files.pythonhosted.org/packages/4f/26/246c6b26c960090f7b9c63933aa331be11a63eadeee58eadd7d97b73ae79/beartype-0.19.0rc0-py3-none-any.whl#sha256=81426615988a22c01feda958cbd7bdf94dd140cea9ce36d2becc3d5ef24c4d45\n",
      "Best match: beartype 0.19.0rc0\n",
      "Processing beartype-0.19.0rc0-py3-none-any.whl\n",
      "Installing beartype-0.19.0rc0-py3-none-any.whl to /opt/conda/lib/python3.10/site-packages\n",
      "Adding beartype 0.19.0rc0 to easy-install.pth file\n",
      "detected new path './jepa-0.0.1-py3.10.egg'\n",
      "\n",
      "Installed /opt/conda/lib/python3.10/site-packages/beartype-0.19.0rc0-py3.10.egg\n",
      "Searching for einops\n",
      "Reading https://pypi.org/simple/einops/\n",
      "Downloading https://files.pythonhosted.org/packages/44/5a/f0b9ad6c0a9017e62d4735daaeb11ba3b6c009d69a26141b258cd37b5588/einops-0.8.0-py3-none-any.whl#sha256=9572fb63046264a862693b0a87088af3bdc8c068fde03de63453cbbde245465f\n",
      "Best match: einops 0.8.0\n",
      "Processing einops-0.8.0-py3-none-any.whl\n",
      "Installing einops-0.8.0-py3-none-any.whl to /opt/conda/lib/python3.10/site-packages\n",
      "Adding einops 0.8.0 to easy-install.pth file\n",
      "detected new path './beartype-0.19.0rc0-py3.10.egg'\n",
      "\n",
      "Installed /opt/conda/lib/python3.10/site-packages/einops-0.8.0-py3.10.egg\n",
      "Searching for decord\n",
      "Reading https://pypi.org/simple/decord/\n",
      "Downloading https://files.pythonhosted.org/packages/11/79/936af42edf90a7bd4e41a6cac89c913d4b47fa48a26b042d5129a9242ee3/decord-0.6.0-py3-none-manylinux2010_x86_64.whl#sha256=51997f20be8958e23b7c4061ba45d0efcd86bffd5fe81c695d0befee0d442976\n",
      "Best match: decord 0.6.0\n",
      "Processing decord-0.6.0-py3-none-manylinux2010_x86_64.whl\n",
      "Installing decord-0.6.0-py3-none-manylinux2010_x86_64.whl to /opt/conda/lib/python3.10/site-packages\n",
      "Adding decord 0.6.0 to easy-install.pth file\n",
      "detected new path './einops-0.8.0-py3.10.egg'\n",
      "\n",
      "Installed /opt/conda/lib/python3.10/site-packages/decord-0.6.0-py3.10-linux-x86_64.egg\n",
      "Searching for webdataset\n",
      "Reading https://pypi.org/simple/webdataset/\n",
      "Downloading https://files.pythonhosted.org/packages/9e/86/5614338e7bdd9335b92a855c7caf4713d9c91ee1a586659f2bcfe5dc946b/webdataset-0.2.86-py3-none-any.whl#sha256=843a2b57c6356ebba25e811adf38a476da8e176f1b192f8bd5c8270daf1a6989\n",
      "Best match: webdataset 0.2.86\n",
      "Processing webdataset-0.2.86-py3-none-any.whl\n",
      "Installing webdataset-0.2.86-py3-none-any.whl to /opt/conda/lib/python3.10/site-packages\n",
      "Adding webdataset 0.2.86 to easy-install.pth file\n",
      "detected new path './decord-0.6.0-py3.10-linux-x86_64.egg'\n",
      "Installing widsindex script to /opt/conda/bin\n",
      "\n",
      "Installed /opt/conda/lib/python3.10/site-packages/webdataset-0.2.86-py3.10.egg\n",
      "Searching for braceexpand\n",
      "Reading https://pypi.org/simple/braceexpand/\n",
      "Downloading https://files.pythonhosted.org/packages/fa/93/e8c04e80e82391a6e51f218ca49720f64236bc824e92152a2633b74cf7ab/braceexpand-0.1.7-py2.py3-none-any.whl#sha256=91332d53de7828103dcae5773fb43bc34950b0c8160e35e0f44c4427a3b85014\n",
      "Best match: braceexpand 0.1.7\n",
      "Processing braceexpand-0.1.7-py2.py3-none-any.whl\n",
      "Installing braceexpand-0.1.7-py2.py3-none-any.whl to /opt/conda/lib/python3.10/site-packages\n",
      "Adding braceexpand 0.1.7 to easy-install.pth file\n",
      "detected new path './webdataset-0.2.86-py3.10.egg'\n",
      "\n",
      "Installed /opt/conda/lib/python3.10/site-packages/braceexpand-0.1.7-py3.10.egg\n",
      "Searching for submitit\n",
      "Reading https://pypi.org/simple/submitit/\n",
      "Downloading https://files.pythonhosted.org/packages/eb/db/64ade58ecdd41b2fae6174ab9d3dd62112c0cc0b3f71d5672cd29877f197/submitit-1.5.1-py3-none-any.whl#sha256=c62f3d50bf5070f89b94d79d83d1dca260eefd22a5b3a97eb1f3cd6a5a7b771c\n",
      "Best match: submitit 1.5.1\n",
      "Processing submitit-1.5.1-py3-none-any.whl\n",
      "Installing submitit-1.5.1-py3-none-any.whl to /opt/conda/lib/python3.10/site-packages\n",
      "Adding submitit 1.5.1 to easy-install.pth file\n",
      "detected new path './braceexpand-0.1.7-py3.10.egg'\n",
      "\n",
      "Installed /opt/conda/lib/python3.10/site-packages/submitit-1.5.1-py3.10.egg\n",
      "Searching for opencv-python\n",
      "Reading https://pypi.org/simple/opencv-python/\n",
      "Downloading https://files.pythonhosted.org/packages/3f/a4/d2537f47fd7fcfba966bd806e3ec18e7ee1681056d4b0a9c8d983983e4d5/opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=9ace140fc6d647fbe1c692bcb2abce768973491222c067c131d80957c595b71f\n",
      "Best match: opencv-python 4.10.0.84\n",
      "Processing opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n",
      "Installing opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl to /opt/conda/lib/python3.10/site-packages\n",
      "Adding opencv-python 4.10.0.84 to easy-install.pth file\n",
      "detected new path './submitit-1.5.1-py3.10.egg'\n",
      "\n",
      "Installed /opt/conda/lib/python3.10/site-packages/opencv_python-4.10.0.84-py3.10-linux-x86_64.egg\n",
      "Searching for pandas==2.1.4\n",
      "Best match: pandas 2.1.4\n",
      "Adding pandas 2.1.4 to easy-install.pth file\n",
      "detected new path './opencv_python-4.10.0.84-py3.10-linux-x86_64.egg'\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for timm==0.9.16\n",
      "Best match: timm 0.9.16\n",
      "Adding timm 0.9.16 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for numpy==1.26.4\n",
      "Best match: numpy 1.26.4\n",
      "Adding numpy 1.26.4 to easy-install.pth file\n",
      "Installing f2py script to /opt/conda/bin\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for PyYAML==6.0.1\n",
      "Best match: PyYAML 6.0.1\n",
      "Adding PyYAML 6.0.1 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for torchvision==0.15.2a0+ab7b3e6\n",
      "Best match: torchvision 0.15.2a0+ab7b3e6\n",
      "Adding torchvision 0.15.2a0+ab7b3e6 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for torch==2.0.0.post104\n",
      "Best match: torch 2.0.0.post104\n",
      "Adding torch 2.0.0.post104 to easy-install.pth file\n",
      "Installing convert-caffe2-to-onnx script to /opt/conda/bin\n",
      "Installing convert-onnx-to-caffe2 script to /opt/conda/bin\n",
      "Installing torchrun script to /opt/conda/bin\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for tzdata==2024.1\n",
      "Best match: tzdata 2024.1\n",
      "Adding tzdata 2024.1 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for pytz==2023.3\n",
      "Best match: pytz 2023.3\n",
      "Adding pytz 2023.3 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for python-dateutil==2.9.0\n",
      "Best match: python-dateutil 2.9.0\n",
      "Adding python-dateutil 2.9.0 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for safetensors==0.4.3\n",
      "Best match: safetensors 0.4.3\n",
      "Adding safetensors 0.4.3 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for huggingface-hub==0.23.0\n",
      "Best match: huggingface-hub 0.23.0\n",
      "Adding huggingface-hub 0.23.0 to easy-install.pth file\n",
      "Installing huggingface-cli script to /opt/conda/bin\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for typing-extensions==4.11.0\n",
      "Best match: typing-extensions 4.11.0\n",
      "Adding typing-extensions 4.11.0 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for cloudpickle==2.2.1\n",
      "Best match: cloudpickle 2.2.1\n",
      "Adding cloudpickle 2.2.1 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for pillow==10.3.0\n",
      "Best match: pillow 10.3.0\n",
      "Adding pillow 10.3.0 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for requests==2.31.0\n",
      "Best match: requests 2.31.0\n",
      "Adding requests 2.31.0 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for jinja2==3.1.4\n",
      "Best match: jinja2 3.1.4\n",
      "Adding jinja2 3.1.4 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for networkx==3.3\n",
      "Best match: networkx 3.3\n",
      "Adding networkx 3.3 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for sympy==1.12\n",
      "Best match: sympy 1.12\n",
      "Adding sympy 1.12 to easy-install.pth file\n",
      "Installing isympy script to /opt/conda/bin\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for filelock==3.14.0\n",
      "Best match: filelock 3.14.0\n",
      "Adding filelock 3.14.0 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for six==1.16.0\n",
      "Best match: six 1.16.0\n",
      "Adding six 1.16.0 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for tqdm==4.66.4\n",
      "Best match: tqdm 4.66.4\n",
      "Adding tqdm 4.66.4 to easy-install.pth file\n",
      "Installing tqdm script to /opt/conda/bin\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for packaging==23.2\n",
      "Best match: packaging 23.2\n",
      "Adding packaging 23.2 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for fsspec==2023.6.0\n",
      "Best match: fsspec 2023.6.0\n",
      "Adding fsspec 2023.6.0 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for certifi==2024.2.2\n",
      "Best match: certifi 2024.2.2\n",
      "Adding certifi 2024.2.2 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for urllib3==1.26.18\n",
      "Best match: urllib3 1.26.18\n",
      "Adding urllib3 1.26.18 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for idna==3.7\n",
      "Best match: idna 3.7\n",
      "Adding idna 3.7 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for charset-normalizer==3.3.2\n",
      "Best match: charset-normalizer 3.3.2\n",
      "Adding charset-normalizer 3.3.2 to easy-install.pth file\n",
      "Installing normalizer script to /opt/conda/bin\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for MarkupSafe==2.1.5\n",
      "Best match: MarkupSafe 2.1.5\n",
      "Adding MarkupSafe 2.1.5 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Searching for mpmath==1.3.0\n",
      "Best match: mpmath 1.3.0\n",
      "Adding mpmath 1.3.0 to easy-install.pth file\n",
      "\n",
      "Using /opt/conda/lib/python3.10/site-packages\n",
      "Finished processing dependencies for jepa==0.0.1\n"
     ]
    }
   ],
   "source": [
    "%cd ~/content/jepa\n",
    "\n",
    "!python setup.py install\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/sagemaker-user/content/jepa\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n",
      "INFO:sagemaker.image_uris:image_uri is not presented, retrieving image_uri based on instance_type, framework etc.\n",
      "INFO:sagemaker:Creating training-job with name: pytorch-training-2024-07-12-12-10-09-779\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-07-12 12:13:41 Starting - Starting the training job...\n",
      "2024-07-12 12:13:58 Starting - Preparing the instances for training...\n",
      "2024-07-12 12:14:26 Downloading - Downloading input data.........\n",
      "2024-07-12 12:15:57 Downloading - Downloading the training image...............\n",
      "2024-07-12 12:18:24 Training - Training image download completed. Training in progress.\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2024-07-12 12:18:33,312 sagemaker-training-toolkit INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2024-07-12 12:18:33,333 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2024-07-12 12:18:33,344 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2024-07-12 12:18:33,348 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2024-07-12 12:19:03,650 sagemaker-training-toolkit INFO     Installing module with the following command:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python3.9 -m pip install . -r requirements.txt\u001b[0m\n",
      "\u001b[34mProcessing /opt/ml/code\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): started\u001b[0m\n",
      "\u001b[34mPreparing metadata (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCollecting torch>=2 (from -r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading torch-2.3.1-cp39-cp39-manylinux1_x86_64.whl.metadata (26 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: torchvision in /opt/conda/lib/python3.9/site-packages (from -r requirements.txt (line 2)) (0.14.1+cu117)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pyyaml in /opt/conda/lib/python3.9/site-packages (from -r requirements.txt (line 3)) (6.0.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: numpy in /opt/conda/lib/python3.9/site-packages (from -r requirements.txt (line 4)) (1.23.5)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: opencv-python in /opt/conda/lib/python3.9/site-packages (from -r requirements.txt (line 5)) (4.10.0.84)\u001b[0m\n",
      "\u001b[34mCollecting submitit (from -r requirements.txt (line 6))\u001b[0m\n",
      "\u001b[34mDownloading submitit-1.5.1-py3-none-any.whl.metadata (8.0 kB)\u001b[0m\n",
      "\u001b[34mCollecting braceexpand (from -r requirements.txt (line 7))\u001b[0m\n",
      "\u001b[34mDownloading braceexpand-0.1.7-py2.py3-none-any.whl.metadata (3.0 kB)\u001b[0m\n",
      "\u001b[34mCollecting webdataset (from -r requirements.txt (line 8))\u001b[0m\n",
      "\u001b[34mDownloading webdataset-0.2.86-py3-none-any.whl.metadata (29 kB)\u001b[0m\n",
      "\u001b[34mCollecting timm (from -r requirements.txt (line 9))\u001b[0m\n",
      "\u001b[34mDownloading timm-1.0.7-py3-none-any.whl.metadata (47 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 47.5/47.5 kB 1.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mCollecting decord (from -r requirements.txt (line 10))\u001b[0m\n",
      "\u001b[34mDownloading decord-0.6.0-py3-none-manylinux2010_x86_64.whl.metadata (422 bytes)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pandas in /opt/conda/lib/python3.9/site-packages (from -r requirements.txt (line 11)) (2.0.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: einops in /opt/conda/lib/python3.9/site-packages (from -r requirements.txt (line 12)) (0.6.1)\u001b[0m\n",
      "\u001b[34mCollecting beartype (from -r requirements.txt (line 13))\u001b[0m\n",
      "\u001b[34mDownloading beartype-0.18.5-py3-none-any.whl.metadata (30 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: filelock in /opt/conda/lib/python3.9/site-packages (from torch>=2->-r requirements.txt (line 1)) (3.12.2)\u001b[0m\n",
      "\u001b[34mCollecting typing-extensions>=4.8.0 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: sympy in /opt/conda/lib/python3.9/site-packages (from torch>=2->-r requirements.txt (line 1)) (1.12)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: networkx in /opt/conda/lib/python3.9/site-packages (from torch>=2->-r requirements.txt (line 1)) (3.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jinja2 in /opt/conda/lib/python3.9/site-packages (from torch>=2->-r requirements.txt (line 1)) (3.1.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: fsspec in /opt/conda/lib/python3.9/site-packages (from torch>=2->-r requirements.txt (line 1)) (2023.6.0)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-cublas-cu12==12.1.3.1 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-cufft-cu12==11.0.2.54 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-curand-cu12==10.3.2.106 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-nccl-cu12==2.20.5 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl.metadata (1.8 kB)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-nvtx-cu12==12.1.105 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\u001b[0m\n",
      "\u001b[34mCollecting triton==2.3.1 (from torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading triton-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.4 kB)\u001b[0m\n",
      "\u001b[34mCollecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=2->-r requirements.txt (line 1))\u001b[0m\n",
      "\u001b[34mDownloading nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: requests in /opt/conda/lib/python3.9/site-packages (from torchvision->-r requirements.txt (line 2)) (2.32.3)\u001b[0m\n",
      "\u001b[34mINFO: pip is looking at multiple versions of torchvision to determine which version is compatible with other requirements. This could take a while.\u001b[0m\n",
      "\u001b[34mCollecting torchvision (from -r requirements.txt (line 2))\u001b[0m\n",
      "\u001b[34mDownloading torchvision-0.18.1-cp39-cp39-manylinux1_x86_64.whl.metadata (6.6 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/conda/lib/python3.9/site-packages (from torchvision->-r requirements.txt (line 2)) (10.4.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: cloudpickle>=1.2.1 in /opt/conda/lib/python3.9/site-packages (from submitit->-r requirements.txt (line 6)) (2.2.1)\u001b[0m\n",
      "\u001b[34mCollecting huggingface_hub (from timm->-r requirements.txt (line 9))\u001b[0m\n",
      "\u001b[34mDownloading huggingface_hub-0.23.4-py3-none-any.whl.metadata (12 kB)\u001b[0m\n",
      "\u001b[34mCollecting safetensors (from timm->-r requirements.txt (line 9))\u001b[0m\n",
      "\u001b[34mDownloading safetensors-0.4.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.8 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.9/site-packages (from pandas->-r requirements.txt (line 11)) (2.8.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.9/site-packages (from pandas->-r requirements.txt (line 11)) (2023.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.9/site-packages (from pandas->-r requirements.txt (line 11)) (2023.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.9/site-packages (from python-dateutil>=2.8.2->pandas->-r requirements.txt (line 11)) (1.16.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.9/site-packages (from huggingface_hub->timm->-r requirements.txt (line 9)) (23.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tqdm>=4.42.1 in /opt/conda/lib/python3.9/site-packages (from huggingface_hub->timm->-r requirements.txt (line 9)) (4.66.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.9/site-packages (from jinja2->torch>=2->-r requirements.txt (line 1)) (2.1.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.9/site-packages (from sympy->torch>=2->-r requirements.txt (line 1)) (1.3.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.9/site-packages (from requests->torchvision->-r requirements.txt (line 2)) (3.2.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.9/site-packages (from requests->torchvision->-r requirements.txt (line 2)) (3.7)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.9/site-packages (from requests->torchvision->-r requirements.txt (line 2)) (1.26.19)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.9/site-packages (from requests->torchvision->-r requirements.txt (line 2)) (2023.7.22)\u001b[0m\n",
      "\u001b[34mDownloading torch-2.3.1-cp39-cp39-manylinux1_x86_64.whl (779.1 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 779.1/779.1 MB 1.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 410.6/410.6 MB 3.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 14.1/14.1 MB 87.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 23.7/23.7 MB 77.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 823.6/823.6 kB 73.2 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 731.7/731.7 MB 1.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 121.6/121.6 MB 27.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 56.5/56.5 MB 51.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 124.2/124.2 MB 26.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 196.0/196.0 MB 15.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 176.2/176.2 MB 19.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 99.1/99.1 kB 17.8 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading triton-2.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (168.1 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 168.1/168.1 MB 20.0 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading torchvision-0.18.1-cp39-cp39-manylinux1_x86_64.whl (7.0 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.0/7.0 MB 103.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading submitit-1.5.1-py3-none-any.whl (74 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 74.7/74.7 kB 15.6 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading braceexpand-0.1.7-py2.py3-none-any.whl (5.9 kB)\u001b[0m\n",
      "\u001b[34mDownloading webdataset-0.2.86-py3-none-any.whl (70 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 70.4/70.4 kB 17.1 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading timm-1.0.7-py3-none-any.whl (2.3 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.3/2.3 MB 104.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading decord-0.6.0-py3-none-manylinux2010_x86_64.whl (13.6 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 13.6/13.6 MB 81.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading beartype-0.18.5-py3-none-any.whl (917 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 917.8/917.8 kB 85.7 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading typing_extensions-4.12.2-py3-none-any.whl (37 kB)\u001b[0m\n",
      "\u001b[34mDownloading huggingface_hub-0.23.4-py3-none-any.whl (402 kB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 402.6/402.6 kB 61.4 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading safetensors-0.4.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.2/1.2 MB 81.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mDownloading nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_x86_64.whl (21.3 MB)\u001b[0m\n",
      "\u001b[34m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 21.3/21.3 MB 83.9 MB/s eta 0:00:00\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: jepa\u001b[0m\n",
      "\u001b[34mBuilding wheel for jepa (setup.py): started\u001b[0m\n",
      "\u001b[34mBuilding wheel for jepa (setup.py): finished with status 'done'\u001b[0m\n",
      "\u001b[34mCreated wheel for jepa: filename=jepa-0.0.1-py3-none-any.whl size=51322 sha256=21c6d56de5629ff6c1420b0835bfe3df1c381ef51e7e786b0a059387b05d82ee\u001b[0m\n",
      "\u001b[34mStored in directory: /tmp/pip-ephem-wheel-cache-z6vd2tb7/wheels/40/03/3a/5f39818cea87b3c154b54d046a775b3da4b8ed9b642b8d50e6\u001b[0m\n",
      "\u001b[34mSuccessfully built jepa\u001b[0m\n",
      "\u001b[34mInstalling collected packages: braceexpand, webdataset, typing-extensions, triton, safetensors, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, decord, beartype, submitit, nvidia-cusparse-cu12, nvidia-cudnn-cu12, huggingface_hub, nvidia-cusolver-cu12, torch, torchvision, timm, jepa\u001b[0m\n",
      "\u001b[34mAttempting uninstall: typing-extensions\u001b[0m\n",
      "\u001b[34mFound existing installation: typing_extensions 4.7.1\u001b[0m\n",
      "\u001b[34mUninstalling typing_extensions-4.7.1:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled typing_extensions-4.7.1\u001b[0m\n",
      "\u001b[34mAttempting uninstall: triton\u001b[0m\n",
      "\u001b[34mFound existing installation: triton 2.0.0.dev20221202\u001b[0m\n",
      "\u001b[34mUninstalling triton-2.0.0.dev20221202:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled triton-2.0.0.dev20221202\u001b[0m\n",
      "\u001b[34mAttempting uninstall: torch\u001b[0m\n",
      "\u001b[34mFound existing installation: torch 1.13.1+cu117\u001b[0m\n",
      "\u001b[34mUninstalling torch-1.13.1+cu117:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled torch-1.13.1+cu117\u001b[0m\n",
      "\u001b[34mAttempting uninstall: torchvision\u001b[0m\n",
      "\u001b[34mFound existing installation: torchvision 0.14.1+cu117\u001b[0m\n",
      "\u001b[34mUninstalling torchvision-0.14.1+cu117:\u001b[0m\n",
      "\u001b[34mSuccessfully uninstalled torchvision-0.14.1+cu117\u001b[0m\n",
      "\u001b[34mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\u001b[0m\n",
      "\u001b[34mfastai 2.7.11 requires torch<1.14,>=1.7, but you have torch 2.3.1 which is incompatible.\u001b[0m\n",
      "\u001b[34mtorchaudio 0.13.1+cu117 requires torch==1.13.1, but you have torch 2.3.1 which is incompatible.\u001b[0m\n",
      "\u001b[34mtorchdata 0.5.1 requires torch==1.13.1, but you have torch 2.3.1 which is incompatible.\u001b[0m\n",
      "\u001b[34mSuccessfully installed beartype-0.18.5 braceexpand-0.1.7 decord-0.6.0 huggingface_hub-0.23.4 jepa-0.0.1 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.82 nvidia-nvtx-cu12-12.1.105 safetensors-0.4.3 submitit-1.5.1 timm-1.0.7 torch-2.3.1 torchvision-0.18.1 triton-2.3.1 typing-extensions-4.12.2 webdataset-0.2.86\u001b[0m\n",
      "\u001b[34mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable.It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.\u001b[0m\n",
      "\u001b[34m2024-07-12 12:21:42,767 sagemaker-training-toolkit INFO     Waiting for the process to finish and give a return code.\u001b[0m\n",
      "\u001b[34m2024-07-12 12:21:42,767 sagemaker-training-toolkit INFO     Done waiting for a return code. Received 0 from exiting process.\u001b[0m\n",
      "\u001b[34m2024-07-12 12:21:42,821 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2024-07-12 12:21:42,868 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2024-07-12 12:21:42,912 sagemaker-training-toolkit INFO     No Neurons detected (normal if no neurons installed)\u001b[0m\n",
      "\u001b[34m2024-07-12 12:21:42,928 sagemaker-training-toolkit INFO     Invoking user script\u001b[0m\n",
      "\u001b[34mTraining Env:\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {\n",
      "        \"pretrained_model\": \"/opt/ml/input/data/pretrained_model\",\n",
      "        \"train\": \"/opt/ml/input/data/train\",\n",
      "        \"val\": \"/opt/ml/input/data/val\"\n",
      "    },\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"current_instance_group\": \"homogeneousCluster\",\n",
      "    \"current_instance_group_hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"current_instance_type\": \"ml.g4dn.xlarge\",\n",
      "    \"distribution_hosts\": [],\n",
      "    \"distribution_instance_groups\": [],\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"devices\": [\n",
      "            \"cuda:0\"\n",
      "        ],\n",
      "        \"fname\": \"configs/evals/vitl16_in1k.yaml\"\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {\n",
      "        \"pretrained_model\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        },\n",
      "        \"train\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        },\n",
      "        \"val\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        }\n",
      "    },\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"instance_groups\": [\n",
      "        \"homogeneousCluster\"\n",
      "    ],\n",
      "    \"instance_groups_dict\": {\n",
      "        \"homogeneousCluster\": {\n",
      "            \"instance_group_name\": \"homogeneousCluster\",\n",
      "            \"instance_type\": \"ml.g4dn.xlarge\",\n",
      "            \"hosts\": [\n",
      "                \"algo-1\"\n",
      "            ]\n",
      "        }\n",
      "    },\n",
      "    \"is_hetero\": false,\n",
      "    \"is_master\": true,\n",
      "    \"is_modelparallel_enabled\": null,\n",
      "    \"is_smddpmprun_installed\": true,\n",
      "    \"job_name\": \"pytorch-training-2024-07-12-12-10-09-779\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://sagemaker-ap-southeast-2-730335271353/pytorch-training-2024-07-12-12-10-09-779/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"evals_main\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 4,\n",
      "    \"num_gpus\": 1,\n",
      "    \"num_neurons\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"current_instance_type\": \"ml.g4dn.xlarge\",\n",
      "        \"current_group_name\": \"homogeneousCluster\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"instance_groups\": [\n",
      "            {\n",
      "                \"instance_group_name\": \"homogeneousCluster\",\n",
      "                \"instance_type\": \"ml.g4dn.xlarge\",\n",
      "                \"hosts\": [\n",
      "                    \"algo-1\"\n",
      "                ]\n",
      "            }\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"evals_main.py\"\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34mEnvironment variables:\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={\"devices\":[\"cuda:0\"],\"fname\":\"configs/evals/vitl16_in1k.yaml\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=evals_main.py\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.g4dn.xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.xlarge\"}],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={\"pretrained_model\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"val\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[\"pretrained_model\",\"train\",\"val\"]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_TYPE=ml.g4dn.xlarge\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP=homogeneousCluster\u001b[0m\n",
      "\u001b[34mSM_CURRENT_INSTANCE_GROUP_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS=[\"homogeneousCluster\"]\u001b[0m\n",
      "\u001b[34mSM_INSTANCE_GROUPS_DICT={\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.xlarge\"}}\u001b[0m\n",
      "\u001b[34mSM_DISTRIBUTION_INSTANCE_GROUPS=[]\u001b[0m\n",
      "\u001b[34mSM_IS_HETERO=false\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=evals_main\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=4\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=1\u001b[0m\n",
      "\u001b[34mSM_NUM_NEURONS=0\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://sagemaker-ap-southeast-2-730335271353/pytorch-training-2024-07-12-12-10-09-779/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"pretrained_model\":\"/opt/ml/input/data/pretrained_model\",\"train\":\"/opt/ml/input/data/train\",\"val\":\"/opt/ml/input/data/val\"},\"current_host\":\"algo-1\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\"],\"current_instance_type\":\"ml.g4dn.xlarge\",\"distribution_hosts\":[],\"distribution_instance_groups\":[],\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"devices\":[\"cuda:0\"],\"fname\":\"configs/evals/vitl16_in1k.yaml\"},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"pretrained_model\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"val\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.xlarge\"}},\"is_hetero\":false,\"is_master\":true,\"is_modelparallel_enabled\":null,\"is_smddpmprun_installed\":true,\"job_name\":\"pytorch-training-2024-07-12-12-10-09-779\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-ap-southeast-2-730335271353/pytorch-training-2024-07-12-12-10-09-779/source/sourcedir.tar.gz\",\"module_name\":\"evals_main\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":1,\"num_neurons\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.g4dn.xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.g4dn.xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"evals_main.py\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[\"--devices\",\"['cuda:0']\",\"--fname\",\"configs/evals/vitl16_in1k.yaml\"]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_PRETRAINED_MODEL=/opt/ml/input/data/pretrained_model\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_TRAIN=/opt/ml/input/data/train\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_VAL=/opt/ml/input/data/val\u001b[0m\n",
      "\u001b[34mSM_HP_DEVICES=[\"cuda:0\"]\u001b[0m\n",
      "\u001b[34mSM_HP_FNAME=configs/evals/vitl16_in1k.yaml\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python39.zip:/opt/conda/lib/python3.9:/opt/conda/lib/python3.9/lib-dynload:/opt/conda/lib/python3.9/site-packages\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python3.9 -m evals_main --devices ['cuda:0'] --fname configs/evals/vitl16_in1k.yaml\u001b[0m\n",
      "\u001b[34m2024-07-12 12:21:42,995 sagemaker-training-toolkit INFO     Exceptions not imported for SageMaker TF as Tensorflow is not installed.\u001b[0m\n",
      "\u001b[34mINFO:root:called-params configs/evals/vitl16_in1k.yaml\u001b[0m\n",
      "\u001b[34mINFO:root:loaded params...\u001b[0m\n",
      "\u001b[34m{   'data'\u001b[0m\n",
      "\u001b[34m:\u001b[0m\n",
      "\u001b[34m{   'dataset_name': 'ImageNet',\n",
      "                'image_folder': 'data/',\n",
      "                'num_classes': 1,\n",
      "                'resolution': 224,\n",
      "                'root_path': '/opt/ml/input/'},\n",
      "    'eval_name': 'image_classification_frozen',\n",
      "    'nodes': 8,\n",
      "    'optimization':\u001b[0m\n",
      "\u001b[34m{   'batch_size': 16,\n",
      "                        'final_lr': 0.0,\n",
      "                        'lr': 0.001,\n",
      "                        'num_epochs': 20,\n",
      "                        'start_lr': 0.001,\n",
      "                        'use_bfloat16': True,\n",
      "                        'warmup': 0.0,\n",
      "                        'weight_decay': 0.001},\n",
      "    'pretrain': {   'checkpoint': '/opt/ml/input/data/pretrained_model/vitl16.pth.tar',\n",
      "                    'checkpoint_key': 'target_encoder',\n",
      "                    'clip_duration': None,\n",
      "                    'folder': '/opt/ml/model/',\n",
      "                    'frames_per_clip': 16,\n",
      "                    'model_name': 'vit_large',\n",
      "                    'patch_size': 16,\n",
      "                    'tight_silu': False,\n",
      "                    'tubelet_size': 2,\n",
      "                    'uniform_power': True,\n",
      "                    'use_sdpa': True,\n",
      "                    'use_silu': False,\n",
      "                    'write_tag': 'jepa'},\n",
      "    'resume_checkpoint': False,\n",
      "    'tag': 'in1k-16f',\n",
      "    'tasks_per_node': 8}\u001b[0m\n",
      "\u001b[34mINFO:root:Running... (rank: 0/1)\u001b[0m\n",
      "\u001b[34mINFO:root:Running evaluation: image_classification_frozen\u001b[0m\n",
      "\u001b[34mINFO:root:Initialized (rank/world-size) 0/1\u001b[0m\n",
      "\u001b[34mINFO:root:Loading pretrained model from /opt/ml/input/data/pretrained_model/vitl16.pth.tar\u001b[0m\n",
      "\u001b[34mVisionTransformer(\n",
      "  (patch_embed): PatchEmbed3D(\n",
      "    (proj): Conv3d(3, 1024, kernel_size=(2, 16, 16), stride=(2, 16, 16))\n",
      "  )\n",
      "  (blocks): ModuleList(\n",
      "    (0-23): 24 x Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=True)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): MLP(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\u001b[0m\n",
      "\u001b[34m)\u001b[0m\n",
      "\u001b[34mINFO:root:loaded pretrained model with msg: <All keys matched successfully>\u001b[0m\n",
      "\u001b[34mINFO:root:loaded pretrained encoder from epoch: 300\n",
      " path: /opt/ml/input/data/pretrained_model/vitl16.pth.tar\u001b[0m\n",
      "\u001b[34mINFO:root:implementing auto-agument strategy\u001b[0m\n",
      "\u001b[34mINFO:root:data-path /opt/ml/input/data/train/\u001b[0m\n",
      "\u001b[34mINFO:root:Initialized ImageFolder\u001b[0m\n",
      "\u001b[34mINFO:root:ImageFolder dataset created\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/torch/utils/data/dataloader.py:558: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 4, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
      "  warnings.warn(_create_warning_msg(\u001b[0m\n",
      "\u001b[34mINFO:root:ImageFolder unsupervised data loader created\u001b[0m\n",
      "\u001b[34mINFO:root:data-path /opt/ml/input/data/val/\u001b[0m\n",
      "\u001b[34mINFO:root:Initialized ImageFolder\u001b[0m\n",
      "\u001b[34mINFO:root:ImageFolder dataset created\u001b[0m\n",
      "\u001b[34mINFO:root:ImageFolder unsupervised data loader created\u001b[0m\n",
      "\u001b[34mINFO:root:Dataloader created... iterations per epoch: 1\u001b[0m\n",
      "\u001b[34mINFO:root:Using AdamW\u001b[0m\n",
      "\u001b[34mNCCL version 2.20.5+cuda12.4\u001b[0m\n",
      "\u001b[34malgo-1:89:100 [0] ofi_init:1304 NCCL WARN NET/OFI Only EFA provider is supported\u001b[0m\n",
      "\u001b[34malgo-1:89:100 [0] ofi_init:1355 NCCL WARN NET/OFI aws-ofi-nccl initialization failed\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 1\u001b[0m\n",
      "\u001b[34m/opt/conda/lib/python3.9/site-packages/torch/backends/cuda/__init__.py:342: FutureWarning: torch.backends.cuda.sdp_kernel() is deprecated. In the future, this context manager will be removed. Please see, torch.nn.attention.sdpa_kernel() for the new context manager, with updated signature.\n",
      "  warnings.warn(\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.36e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.36e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    1] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 2\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    2] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 3\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    3] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 4\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    4] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 5\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    5] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 6\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    6] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 7\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    7] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 8\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    8] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 9\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    9] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 10\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[   10] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 11\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[   11] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 12\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[   12] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 13\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[   13] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 14\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[   14] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 15\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[   15] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 16\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[   16] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 17\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[   17] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 18\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[   18] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 19\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[   19] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34mINFO:root:Epoch 20\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[    0] 100.000% (loss: 0.000) [mem: 1.47e+03]\u001b[0m\n",
      "\u001b[34mINFO:root:[   20] train: 100.000% test: 100.000%\u001b[0m\n",
      "\u001b[34m2024-07-12 12:29:39,424 sagemaker-training-toolkit INFO     Waiting for the process to finish and give a return code.\u001b[0m\n",
      "\u001b[34m2024-07-12 12:29:39,424 sagemaker-training-toolkit INFO     Done waiting for a return code. Received 0 from exiting process.\u001b[0m\n",
      "\u001b[34m2024-07-12 12:29:39,424 sagemaker-training-toolkit INFO     Reporting training SUCCESS\u001b[0m\n",
      "\n",
      "2024-07-12 12:30:34 Uploading - Uploading generated training model\n",
      "2024-07-12 12:30:34 Completed - Training job completed\n",
      "Training seconds: 967\n",
      "Billable seconds: 967\n"
     ]
    }
   ],
   "source": [
    "%cd ~/content/jepa\n",
    "\n",
    "import sagemaker\n",
    "from sagemaker.pytorch import PyTorch, TrainingCompilerConfig\n",
    "from sagemaker import get_execution_role\n",
    "from sagemaker.inputs import TrainingInput\n",
    "\n",
    "role = sagemaker.get_execution_role()\n",
    "hyperparameters={\n",
    "    \"fname\": \"configs/evals/vitl16_in1k.yaml\",\n",
    "    \"devices\": ['cuda:0'],\n",
    "}\n",
    "\n",
    "pytorch_estimator=PyTorch(\n",
    "    entry_point='evals_main.py',\n",
    "    role=role,\n",
    "    source_dir='/home/sagemaker-user/content/jepa', # Optional. Add this if need to install additional packages.\n",
    "    instance_count=1,\n",
    "    instance_type='ml.g4dn.xlarge', # ml.m5.xlarge, ml.g4dn.xlarge(現状はGPUのある ml.g4dn.xlarge でないと上手くいかない、https://github.com/facebookresearch/jepa/issues/55 の対応すれば ml.m5.xlarge でも上手く行く)\n",
    "    framework_version='1.13.1',\n",
    "    py_version='py39',\n",
    "    hyperparameters=hyperparameters,\n",
    "    disable_profiler=True,\n",
    "    debugger_hook_config=False\n",
    ")\n",
    "\n",
    "# この引数を足すと以下エラーが発生する。500GBでやろうとしたが125GBまでしか大きくできない旨のエラーが出た。\n",
    "# ClientError: An error occurred (ValidationException) when calling the CreateTrainingJob operation: Invalid VolumeSizeInGB: 500 GB. The requested instance type ml.g4dn.xlarge includes local instance storage with a fixed total size of 125 GB. Please reduce VolumeSizeInGB in the request or choose a different instance type.\n",
    "# 直そうと思ったが、作業を中断してしまったので直していない\n",
    "#    volume_size=500,\n",
    "\n",
    "\n",
    "# この引数を足すと以下エラーが発生する。jepaで分散トレーニングをしないように修正したのを戻したがやはり下記エラーが発生したので、そのまま下記引数は足さない\n",
    "# [エラー内容]AttributeError: module 'torch' has no attribute '_six'\n",
    "#    compiler_config=TrainingCompilerConfig(),\n",
    "\n",
    "pytorch_estimator.fit(\n",
    "    inputs={\n",
    "        \"pretrained_model\": TrainingInput(s3_data=\"s3://jepa-vitl16\"),\n",
    "        \"train\": TrainingInput(s3_data=\"s3://jepa-images/train\"),\n",
    "        \"val\": TrainingInput(s3_data=\"s3://jepa-images/val\"),\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 613491,
     "status": "ok",
     "timestamp": 1717846203197,
     "user": {
      "displayName": "Music AI",
      "userId": "14623001796099089649"
     },
     "user_tz": -540
    },
    "id": "BIjvSV8BXWNJ",
    "outputId": "0180c7ec-f1e5-4672-b504-b549a4cdcaa1"
   },
   "outputs": [],
   "source": [
    "%cd ~/content/jepa\n",
    "!python -m app.main --fname configs/pretrain/vitl16.yaml --devices cuda:0 cuda:1 cuda:2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2570283,
     "status": "ok",
     "timestamp": 1718242675848,
     "user": {
      "displayName": "深川梓",
      "userId": "06822277665795432790"
     },
     "user_tz": -540
    },
    "id": "lgY2kD2Vcjle",
    "outputId": "71d5fecd-e55f-46c5-f4ea-d37d85ca7f36"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/sagemaker-user/content/jepa\n",
      "INFO:root:called-params configs/evals/vitl16_in1k.yaml\n",
      "INFO:root:loaded params...\n",
      "{   'data': {   'dataset_name': 'ImageNet',\n",
      "                'image_folder': 'imagenet_full_size/061417/',\n",
      "                'num_classes': 1,\n",
      "                'resolution': 224,\n",
      "                'root_path': '/MyDrive/jepa_images/'},\n",
      "    'eval_name': 'image_classification_frozen',\n",
      "    'nodes': 8,\n",
      "    'optimization': {   'batch_size': 16,\n",
      "                        'final_lr': 0.0,\n",
      "                        'lr': 0.001,\n",
      "                        'num_epochs': 20,\n",
      "                        'start_lr': 0.001,\n",
      "                        'use_bfloat16': True,\n",
      "                        'warmup': 0.0,\n",
      "                        'weight_decay': 0.001},\n",
      "    'pretrain': {   'checkpoint': 'vitl16.pth.tar',\n",
      "                    'checkpoint_key': 'target_encoder',\n",
      "                    'clip_duration': None,\n",
      "                    'folder': '/MyDrive/',\n",
      "                    'frames_per_clip': 16,\n",
      "                    'model_name': 'vit_large',\n",
      "                    'patch_size': 16,\n",
      "                    'tight_silu': False,\n",
      "                    'tubelet_size': 2,\n",
      "                    'uniform_power': True,\n",
      "                    'use_sdpa': True,\n",
      "                    'use_silu': False,\n",
      "                    'write_tag': 'jepa'},\n",
      "    'resume_checkpoint': False,\n",
      "    'tag': 'in1k-16f',\n",
      "    'tasks_per_node': 8}\n",
      "INFO:root:Rank: 0. Distributed training not available Distributed package doesn't have NCCL built in\n",
      "INFO:root:Running... (rank: 0/1)\n",
      "INFO:root:Running evaluation: image_classification_frozen\n",
      "INFO:root:SLURM vars not set (distributed training not available)\n",
      "INFO:root:Initialized (rank/world-size) 0/1\n",
      "Process Process-1:\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.10/multiprocessing/process.py\", line 314, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/opt/conda/lib/python3.10/multiprocessing/process.py\", line 108, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/sagemaker-user/content/jepa/evals_main.py\", line 56, in process_main\n",
      "    eval_main(params['eval_name'], args_eval=params)\n",
      "  File \"/home/sagemaker-user/content/jepa/evals/scaffold.py\", line 22, in main\n",
      "    return importlib.import_module(f'evals.{eval_name}.eval').main(\n",
      "  File \"/home/sagemaker-user/content/jepa/evals/image_classification_frozen/eval.py\", line 130, in main\n",
      "    os.makedirs(folder, exist_ok=True)\n",
      "  File \"/opt/conda/lib/python3.10/os.py\", line 215, in makedirs\n",
      "    makedirs(head, exist_ok=exist_ok)\n",
      "  File \"/opt/conda/lib/python3.10/os.py\", line 215, in makedirs\n",
      "    makedirs(head, exist_ok=exist_ok)\n",
      "  File \"/opt/conda/lib/python3.10/os.py\", line 225, in makedirs\n",
      "    mkdir(name, mode)\n",
      "PermissionError: [Errno 13] Permission denied: '/MyDrive'\n"
     ]
    }
   ],
   "source": [
    "%cd ~/content/jepa\n",
    "!python -m evals_main --fname configs/evals/vitl16_in1k.yaml --devices cuda:0 #cuda:1 cuda:2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 76229,
     "status": "ok",
     "timestamp": 1719915559424,
     "user": {
      "displayName": "深川梓",
      "userId": "06822277665795432790"
     },
     "user_tz": -540
    },
    "id": "WzcMxHbFlMBd",
    "outputId": "8336b43a-d6f0-4034-940f-17a5825c67bb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary contents: OrderedDict([('module.backbone.pos_embed', tensor([[[0.0000, 0.0000, 0.0000,  ..., 1.0000, 1.0000, 1.0000],\n",
      "         [0.0000, 0.0000, 0.0000,  ..., 1.0000, 1.0000, 1.0000],\n",
      "         [0.0000, 0.0000, 0.0000,  ..., 1.0000, 1.0000, 1.0000],\n",
      "         ...,\n",
      "         [0.6570, 0.3427, 0.0019,  ..., 0.9999, 0.9999, 1.0000],\n",
      "         [0.6570, 0.3427, 0.0019,  ..., 0.9999, 0.9999, 0.9999],\n",
      "         [0.6570, 0.3427, 0.0019,  ..., 0.9999, 0.9999, 0.9999]]])), ('module.backbone.patch_embed.proj.weight', tensor([[[[[ 1.6566e-03, -2.3490e-03, -1.1865e-03,  ..., -8.6622e-04,\n",
      "            -2.3996e-04, -1.1941e-02],\n",
      "           [ 5.3274e-03,  5.2559e-04,  8.7542e-04,  ...,  6.2355e-04,\n",
      "            -1.8926e-04, -9.8496e-03],\n",
      "           [ 3.3134e-03, -5.0810e-04, -1.3705e-04,  ..., -2.2038e-03,\n",
      "            -1.4955e-03, -1.1411e-02],\n",
      "           ...,\n",
      "           [ 9.2960e-03,  1.0699e-03,  6.2105e-04,  ...,  2.1257e-04,\n",
      "             1.0443e-03, -6.9656e-03],\n",
      "           [ 7.1181e-03, -1.4097e-03, -9.7223e-05,  ..., -7.4368e-04,\n",
      "             1.1718e-03, -6.2686e-03],\n",
      "           [ 6.4004e-03, -3.0046e-03, -4.3801e-03,  ..., -2.5585e-03,\n",
      "            -2.0144e-03, -1.1302e-02]],\n",
      "\n",
      "          [[ 3.0277e-03, -3.4963e-03, -1.2342e-03,  ...,  8.3184e-04,\n",
      "             5.5818e-04, -1.1728e-02],\n",
      "           [ 3.5347e-03, -1.4218e-03, -8.6534e-05,  ...,  6.5607e-04,\n",
      "             1.4459e-03, -6.9788e-03],\n",
      "           [ 6.0452e-03,  1.1188e-03,  1.2723e-03,  ..., -5.1208e-05,\n",
      "             9.4896e-04, -7.8701e-03],\n",
      "           ...,\n",
      "           [ 9.4463e-03,  1.0315e-03,  5.7988e-05,  ...,  3.5060e-03,\n",
      "             3.8953e-03,  3.3004e-04],\n",
      "           [ 7.2433e-03, -8.4504e-05,  1.9993e-03,  ...,  2.7743e-03,\n",
      "             3.7699e-03,  1.9042e-03],\n",
      "           [ 6.0982e-03, -1.6640e-03, -1.0262e-04,  ...,  6.0889e-04,\n",
      "             1.4855e-03, -1.7849e-03]]],\n",
      "\n",
      "\n",
      "         [[[ 3.9333e-03, -1.8109e-03, -1.6949e-03,  ..., -3.8637e-03,\n",
      "            -9.3404e-04, -1.3132e-02],\n",
      "           [ 6.4531e-03,  8.3289e-04,  5.1008e-04,  ..., -1.3950e-03,\n",
      "             3.3656e-04, -9.4059e-03],\n",
      "           [ 3.3524e-03, -5.3534e-04, -8.8356e-04,  ..., -4.6434e-03,\n",
      "            -1.2368e-03, -1.0865e-02],\n",
      "           ...,\n",
      "           [ 1.1321e-02,  3.1455e-03,  2.0503e-03,  ..., -2.2712e-04,\n",
      "             2.2682e-03, -4.2057e-03],\n",
      "           [ 9.7748e-03,  1.6598e-03,  1.7523e-03,  ...,  6.6156e-04,\n",
      "             3.5784e-03, -2.4133e-03],\n",
      "           [ 7.2409e-03, -9.9404e-04, -3.6643e-03,  ..., -2.4527e-03,\n",
      "            -1.1547e-03, -9.0705e-03]],\n",
      "\n",
      "          [[ 1.9365e-03, -6.2220e-03, -4.3317e-03,  ..., -3.5391e-03,\n",
      "            -3.7336e-03, -1.8857e-02],\n",
      "           [ 1.5582e-03, -4.6519e-03, -3.6530e-03,  ..., -2.2403e-03,\n",
      "            -1.0848e-03, -1.2467e-02],\n",
      "           [ 2.5594e-03, -3.3093e-03, -3.2716e-03,  ..., -4.1797e-03,\n",
      "            -2.6432e-03, -1.3976e-02],\n",
      "           ...,\n",
      "           [ 7.8758e-03,  8.1577e-04, -6.9682e-04,  ...,  1.4448e-03,\n",
      "             2.5841e-03, -4.7111e-03],\n",
      "           [ 6.1443e-03, -5.1832e-04,  5.2663e-04,  ...,  8.1964e-04,\n",
      "             1.9776e-03, -4.7779e-03],\n",
      "           [ 2.0965e-04, -5.7839e-03, -5.7497e-03,  ..., -6.6348e-03,\n",
      "            -6.3710e-03, -1.4198e-02]]],\n",
      "\n",
      "\n",
      "         [[[ 7.8511e-03,  1.6869e-03,  1.6198e-03,  ...,  8.8101e-04,\n",
      "             4.6763e-03, -4.1807e-04],\n",
      "           [ 6.4980e-03,  7.0131e-04,  1.7010e-04,  ..., -1.7033e-04,\n",
      "             1.5647e-03, -1.7103e-03],\n",
      "           [ 1.3273e-03, -2.4576e-03, -2.4421e-03,  ..., -2.9225e-03,\n",
      "            -3.4220e-04, -3.9983e-03],\n",
      "           ...,\n",
      "           [ 6.0947e-03,  3.7059e-04,  1.1621e-03,  ...,  2.6633e-04,\n",
      "             2.2822e-03,  1.8492e-03],\n",
      "           [ 5.1474e-03, -1.0825e-03,  7.0446e-04,  ...,  1.2525e-03,\n",
      "             3.8491e-03,  4.3659e-03],\n",
      "           [ 5.3876e-03, -1.7068e-03, -2.1074e-03,  ...,  1.3836e-03,\n",
      "             3.4190e-03,  1.3824e-03]],\n",
      "\n",
      "          [[ 1.0881e-02,  2.4959e-03,  4.2327e-03,  ...,  2.3340e-03,\n",
      "             4.4079e-03, -2.7424e-03],\n",
      "           [ 4.8921e-03, -1.4236e-03, -3.6580e-04,  ..., -6.0049e-04,\n",
      "             1.6131e-03, -1.9521e-03],\n",
      "           [ 2.6093e-03, -3.0377e-03, -2.4996e-03,  ..., -2.6985e-03,\n",
      "            -9.3610e-04, -4.4625e-03],\n",
      "           ...,\n",
      "           [ 4.0551e-03,  1.7464e-04, -1.3551e-03,  ...,  1.6922e-03,\n",
      "             2.5717e-03,  2.1242e-03],\n",
      "           [ 2.8092e-03, -1.5177e-03, -5.1516e-05,  ...,  3.3282e-04,\n",
      "             1.9815e-03,  2.1136e-03],\n",
      "           [ 1.4683e-03, -2.8882e-03, -2.5060e-03,  ..., -3.7446e-03,\n",
      "            -1.9851e-03, -3.3158e-03]]]],\n",
      "\n",
      "\n",
      "\n",
      "        [[[[-2.1840e-02, -1.0310e-02, -9.1853e-03,  ..., -1.5513e-02,\n",
      "            -1.8293e-02, -1.7725e-02],\n",
      "           [-4.5133e-03,  3.2135e-03,  2.1190e-03,  ..., -1.5262e-03,\n",
      "            -5.2281e-03, -1.5993e-03],\n",
      "           [-3.0356e-03,  3.6860e-03,  3.7038e-03,  ..., -2.6894e-04,\n",
      "            -4.1833e-03, -7.3396e-04],\n",
      "           ...,\n",
      "           [-3.7276e-03,  1.4901e-03,  4.0761e-04,  ..., -2.7003e-04,\n",
      "             8.8071e-04,  1.2259e-02],\n",
      "           [ 3.8001e-04,  2.3578e-03,  1.8852e-03,  ...,  1.1379e-03,\n",
      "             9.2210e-04,  1.4173e-02],\n",
      "           [ 3.8349e-03,  8.2104e-03,  6.5829e-03,  ...,  6.9067e-03,\n",
      "             7.4384e-03,  2.2297e-02]],\n",
      "\n",
      "          [[-2.7258e-02, -1.2217e-02, -9.9769e-03,  ..., -2.0331e-02,\n",
      "            -2.3088e-02, -1.9274e-02],\n",
      "           [-6.6892e-03,  3.9848e-03,  3.9564e-03,  ..., -5.3444e-03,\n",
      "            -8.0429e-03, -1.8554e-03],\n",
      "           [-2.9670e-03,  5.6312e-03,  6.6793e-03,  ..., -2.2025e-03,\n",
      "            -4.8821e-03, -6.5392e-04],\n",
      "           ...,\n",
      "           [-5.7857e-03,  2.4369e-03,  1.3939e-03,  ...,  1.7180e-03,\n",
      "             3.7156e-03,  1.3772e-02],\n",
      "           [-4.7044e-03,  2.2997e-03,  2.0776e-03,  ...,  9.8733e-04,\n",
      "            -7.6556e-05,  1.2152e-02],\n",
      "           [-2.6160e-03,  5.5811e-03,  4.4180e-03,  ...,  3.3577e-03,\n",
      "             3.7223e-03,  1.9503e-02]]],\n",
      "\n",
      "\n",
      "         [[[-1.2137e-02, -4.3183e-03, -3.1332e-03,  ..., -7.8165e-03,\n",
      "            -7.2085e-03,  4.1195e-03],\n",
      "           [-1.4464e-03,  4.4305e-03,  3.8086e-03,  ..., -2.5553e-05,\n",
      "            -9.5981e-04,  1.1217e-02],\n",
      "           [-3.2033e-03,  3.1108e-03,  5.2283e-03,  ..., -3.8498e-04,\n",
      "            -3.2433e-03,  9.1681e-03],\n",
      "           ...,\n",
      "           [-7.7838e-03,  1.5447e-03,  7.4415e-04,  ..., -1.0706e-03,\n",
      "             7.8462e-04,  1.6119e-02],\n",
      "           [-5.6343e-03,  3.0742e-04,  4.4236e-04,  ..., -7.4619e-04,\n",
      "             4.3488e-04,  1.7696e-02],\n",
      "           [-8.9220e-03, -8.0972e-04, -2.0978e-03,  ...,  2.4615e-03,\n",
      "             4.5514e-03,  2.3588e-02]],\n",
      "\n",
      "          [[-1.7647e-02, -8.2361e-03, -5.0578e-03,  ..., -1.1688e-02,\n",
      "            -1.1649e-02,  5.7269e-03],\n",
      "           [-4.0631e-03,  3.4089e-03,  4.3070e-03,  ..., -3.4634e-03,\n",
      "            -3.6305e-03,  1.4133e-02],\n",
      "           [-1.7003e-03,  4.8371e-03,  6.8327e-03,  ..., -1.0587e-03,\n",
      "            -3.0117e-03,  1.2038e-02],\n",
      "           ...,\n",
      "           [-6.5605e-03,  4.1622e-03,  3.9526e-03,  ...,  4.1938e-03,\n",
      "             6.9550e-03,  2.2261e-02],\n",
      "           [-6.6010e-03,  2.8608e-03,  3.4408e-03,  ...,  4.1830e-03,\n",
      "             4.5916e-03,  2.2159e-02],\n",
      "           [-9.0716e-03,  1.5312e-03,  9.9921e-04,  ...,  6.2507e-03,\n",
      "             8.6912e-03,  2.9105e-02]]],\n",
      "\n",
      "\n",
      "         [[[ 1.1622e-03,  5.7518e-03,  6.8815e-03,  ...,  3.5774e-03,\n",
      "             4.9241e-03,  1.3922e-02],\n",
      "           [ 3.1426e-03,  5.9924e-03,  4.7534e-03,  ...,  2.4091e-03,\n",
      "             2.2248e-03,  1.1999e-02],\n",
      "           [ 1.4125e-04,  3.3019e-03,  4.7425e-03,  ...,  1.3099e-03,\n",
      "            -7.9429e-04,  8.3341e-03],\n",
      "           ...,\n",
      "           [-1.3169e-02, -4.0759e-03, -4.2194e-03,  ..., -5.2477e-03,\n",
      "            -4.4371e-03,  2.7835e-03],\n",
      "           [-1.3201e-02, -6.2909e-03, -4.5901e-03,  ..., -5.1797e-03,\n",
      "            -5.9710e-03,  2.3553e-03],\n",
      "           [-2.3737e-02, -1.3941e-02, -1.3348e-02,  ..., -9.9977e-03,\n",
      "            -9.6337e-03, -1.0576e-03]],\n",
      "\n",
      "          [[ 5.4227e-03,  1.0301e-02,  1.1869e-02,  ...,  1.8967e-03,\n",
      "             2.3715e-03,  1.5224e-02],\n",
      "           [ 7.8634e-03,  1.0068e-02,  9.9478e-03,  ..., -5.9179e-04,\n",
      "            -2.8376e-04,  1.2960e-02],\n",
      "           [ 6.4705e-03,  9.1111e-03,  9.6960e-03,  ...,  1.8124e-04,\n",
      "            -1.6827e-03,  8.4983e-03],\n",
      "           ...,\n",
      "           [-6.9519e-03,  2.8552e-03, -4.3226e-05,  ..., -1.0815e-03,\n",
      "            -3.8018e-05,  5.0722e-03],\n",
      "           [-8.2004e-03,  6.7994e-04,  2.7012e-05,  ..., -1.7260e-03,\n",
      "            -3.6193e-03,  2.7977e-03],\n",
      "           [-1.7104e-02, -6.1817e-03, -7.7489e-03,  ..., -7.9252e-03,\n",
      "            -8.0847e-03,  7.2158e-04]]]],\n",
      "\n",
      "\n",
      "\n",
      "        [[[[-6.4568e-03,  3.4493e-03,  4.2716e-03,  ...,  4.2184e-03,\n",
      "             5.6113e-03,  7.1699e-03],\n",
      "           [-7.6484e-03,  1.4268e-03, -2.5951e-04,  ...,  4.2690e-03,\n",
      "             2.3560e-03,  2.9543e-03],\n",
      "           [-8.9546e-03, -2.7942e-03, -3.3101e-03,  ...,  2.6571e-03,\n",
      "             8.9757e-04,  6.4347e-05],\n",
      "           ...,\n",
      "           [-8.7720e-03,  1.0909e-03,  1.8023e-03,  ..., -1.3021e-03,\n",
      "             1.6644e-04, -3.3859e-03],\n",
      "           [-8.2794e-03, -8.0985e-04,  2.4385e-03,  ..., -2.7834e-03,\n",
      "            -1.3864e-03, -3.8647e-03],\n",
      "           [-3.0991e-03,  1.8365e-03,  4.7940e-03,  ...,  1.6581e-03,\n",
      "             2.2299e-03, -1.2315e-03]],\n",
      "\n",
      "          [[-5.6074e-03,  6.8723e-03,  7.6385e-03,  ...,  5.4123e-03,\n",
      "             6.0323e-03,  4.2851e-03],\n",
      "           [-1.0016e-02,  6.6284e-04, -1.3600e-04,  ...,  3.3554e-03,\n",
      "             2.9788e-03,  2.1975e-03],\n",
      "           [-8.7672e-03, -1.4449e-03, -3.4610e-03,  ...,  1.8066e-04,\n",
      "             6.2487e-04,  4.3128e-04],\n",
      "           ...,\n",
      "           [-7.8786e-03,  2.1464e-03,  6.7173e-04,  ..., -3.5726e-03,\n",
      "            -3.7698e-03, -8.6479e-03],\n",
      "           [-4.6919e-03,  4.3743e-03,  4.6620e-03,  ..., -2.7534e-03,\n",
      "            -5.1908e-03, -9.2706e-03],\n",
      "           [-8.4884e-04,  6.5891e-03,  5.4272e-03,  ..., -5.2947e-04,\n",
      "            -3.8823e-03, -1.0611e-02]]],\n",
      "\n",
      "\n",
      "         [[[-6.2562e-03,  1.6610e-03,  5.2566e-04,  ..., -2.2291e-03,\n",
      "            -2.2658e-04, -1.0330e-03],\n",
      "           [-5.1128e-03,  2.5183e-03, -7.4120e-04,  ...,  3.4298e-03,\n",
      "             2.8859e-03,  1.4093e-03],\n",
      "           [-4.9462e-03, -2.2461e-04, -1.8458e-03,  ...,  3.2491e-03,\n",
      "             3.0213e-03,  1.0467e-03],\n",
      "           ...,\n",
      "           [-5.9870e-03,  2.1082e-03,  1.5296e-03,  ..., -3.2070e-03,\n",
      "            -2.1757e-03, -3.8602e-03],\n",
      "           [-5.8451e-03, -5.8018e-04,  8.1813e-04,  ..., -3.8908e-03,\n",
      "            -2.4785e-03, -3.8393e-03],\n",
      "           [-1.8099e-03,  9.3183e-04,  1.6120e-03,  ..., -1.3978e-03,\n",
      "            -1.3625e-03, -2.5234e-03]],\n",
      "\n",
      "          [[-9.2659e-03,  1.7948e-03,  9.9011e-04,  ..., -2.6264e-03,\n",
      "            -1.9025e-03, -6.1920e-03],\n",
      "           [-9.6042e-03,  8.9455e-04, -1.3413e-03,  ...,  2.2225e-03,\n",
      "             1.8998e-03,  2.1157e-04],\n",
      "           [-6.4091e-03,  1.1039e-03, -2.0961e-03,  ...,  1.2455e-03,\n",
      "             2.4367e-03,  2.2183e-03],\n",
      "           ...,\n",
      "           [-7.7700e-03,  2.8403e-03,  2.0389e-03,  ..., -3.5228e-03,\n",
      "            -2.6519e-03, -3.9674e-03],\n",
      "           [-5.0278e-03,  3.3858e-03,  4.5936e-03,  ..., -1.0305e-03,\n",
      "            -2.1346e-03, -3.1436e-03],\n",
      "           [-2.9218e-03,  3.9659e-03,  3.5551e-03,  ...,  9.6373e-04,\n",
      "            -1.1318e-03, -4.5760e-03]]],\n",
      "\n",
      "\n",
      "         [[[-8.5802e-03, -1.1531e-03, -1.2329e-04,  ..., -4.0254e-03,\n",
      "            -4.1328e-03, -1.0531e-02],\n",
      "           [-4.8942e-03,  2.2882e-03,  4.3198e-04,  ...,  3.6776e-03,\n",
      "             1.2853e-03, -4.9421e-03],\n",
      "           [-3.7814e-03,  4.4463e-04,  3.8432e-05,  ...,  5.2082e-03,\n",
      "             3.5260e-03, -2.6611e-03],\n",
      "           ...,\n",
      "           [-3.9737e-03,  2.2931e-03,  2.2281e-03,  ..., -1.3693e-03,\n",
      "            -1.9217e-03, -5.3780e-03],\n",
      "           [-3.7116e-03, -1.4776e-05,  1.9881e-03,  ..., -2.7201e-03,\n",
      "            -2.0315e-03, -5.5060e-03],\n",
      "           [-5.5884e-04,  2.4619e-04,  1.4303e-03,  ..., -1.1132e-03,\n",
      "            -2.3548e-03, -6.5282e-03]],\n",
      "\n",
      "          [[-1.4974e-02, -3.4739e-03, -4.9971e-04,  ..., -5.6895e-03,\n",
      "            -5.8135e-03, -1.4550e-02],\n",
      "           [-1.0918e-02, -1.4641e-03, -5.9517e-04,  ...,  2.1419e-03,\n",
      "             6.7033e-04, -4.4983e-03],\n",
      "           [-7.8116e-03, -1.1105e-03, -1.7961e-03,  ...,  2.0301e-03,\n",
      "             2.3329e-03, -1.2546e-03],\n",
      "           ...,\n",
      "           [-7.0750e-03,  2.7664e-03,  1.8408e-03,  ..., -2.0635e-04,\n",
      "            -5.4184e-04, -3.7546e-03],\n",
      "           [-4.8297e-03,  2.2170e-03,  4.2202e-03,  ...,  1.5393e-05,\n",
      "            -1.6877e-03, -4.2415e-03],\n",
      "           [-4.6126e-03,  1.7281e-03,  1.2784e-03,  ..., -1.0618e-04,\n",
      "            -2.1499e-03, -7.4620e-03]]]],\n",
      "\n",
      "\n",
      "\n",
      "        ...,\n",
      "\n",
      "\n",
      "\n",
      "        [[[[ 1.1359e-02, -1.5307e-02, -1.1054e-02,  ..., -7.1714e-03,\n",
      "             4.5788e-03,  3.8548e-02],\n",
      "           [-1.3680e-04, -1.8765e-02, -1.2700e-02,  ..., -8.4481e-03,\n",
      "            -2.6863e-03,  1.9847e-02],\n",
      "           [ 2.5479e-03, -8.5409e-03, -3.0918e-03,  ..., -2.4554e-03,\n",
      "            -3.9390e-03,  7.8891e-03],\n",
      "           ...,\n",
      "           [ 1.5126e-02, -2.6776e-02, -2.3389e-02,  ..., -1.5212e-03,\n",
      "             3.0672e-03,  1.0159e-02],\n",
      "           [ 5.7684e-03, -2.8332e-02, -2.7023e-02,  ..., -8.6041e-03,\n",
      "            -5.9910e-03, -4.0764e-03],\n",
      "           [ 3.8264e-02,  3.6523e-03,  5.1851e-03,  ...,  1.5281e-02,\n",
      "             1.7890e-02,  2.0320e-02]],\n",
      "\n",
      "          [[ 7.4527e-02,  9.3950e-03, -1.7888e-02,  ...,  1.3136e-02,\n",
      "             2.2827e-02,  3.9677e-02],\n",
      "           [ 4.3862e-02, -3.5978e-03, -2.3636e-02,  ..., -9.2028e-04,\n",
      "             1.3216e-03,  1.4473e-03],\n",
      "           [ 3.0011e-02,  6.9411e-03, -4.8337e-03,  ..., -6.4079e-03,\n",
      "            -1.1696e-02, -1.9087e-02],\n",
      "           ...,\n",
      "           [ 2.7149e-02, -1.7609e-02, -1.9963e-02,  ..., -1.2359e-02,\n",
      "            -5.7618e-03,  1.4954e-02],\n",
      "           [ 1.0030e-02, -2.5024e-02, -2.4441e-02,  ..., -1.9133e-02,\n",
      "            -1.5767e-02, -4.5980e-04],\n",
      "           [ 4.7746e-02,  2.3633e-02,  2.4425e-02,  ...,  2.8907e-02,\n",
      "             3.7396e-02,  5.2027e-02]]],\n",
      "\n",
      "\n",
      "         [[[ 1.6248e-02, -2.6784e-03,  1.4554e-03,  ..., -6.5480e-03,\n",
      "             1.1169e-03,  2.6148e-02],\n",
      "           [-4.2355e-03, -1.5490e-02, -9.5670e-03,  ..., -5.4810e-03,\n",
      "            -5.6577e-03,  1.1527e-02],\n",
      "           [-1.0563e-02, -1.2960e-02, -5.4515e-03,  ...,  1.6646e-03,\n",
      "            -5.2455e-03,  1.6638e-03],\n",
      "           ...,\n",
      "           [ 1.2558e-02, -1.0066e-02, -2.2484e-03,  ...,  2.8997e-03,\n",
      "             5.6086e-03,  7.8756e-03],\n",
      "           [-6.8506e-03, -1.8440e-02, -1.0723e-02,  ..., -4.2495e-03,\n",
      "            -2.8611e-03, -8.4898e-03],\n",
      "           [ 3.8873e-03, -4.2500e-03,  8.6001e-03,  ...,  1.9569e-02,\n",
      "             1.9430e-02,  9.4557e-03]],\n",
      "\n",
      "          [[ 5.9421e-02,  1.6658e-02, -4.8172e-03,  ...,  2.0246e-02,\n",
      "             2.8242e-02,  4.1349e-02],\n",
      "           [ 1.7765e-02, -7.5968e-03, -2.1735e-02,  ...,  7.4203e-03,\n",
      "             6.4105e-03,  5.9863e-03],\n",
      "           [-7.8975e-03, -5.7420e-03, -8.1593e-03,  ...,  7.4904e-03,\n",
      "            -5.1773e-04, -9.8686e-03],\n",
      "           ...,\n",
      "           [-3.5872e-03, -9.2474e-03,  2.4884e-03,  ..., -2.3681e-03,\n",
      "             9.5796e-04,  1.7018e-02],\n",
      "           [-4.0991e-02, -3.6033e-02, -1.8015e-02,  ..., -1.4219e-02,\n",
      "            -1.2974e-02, -4.3943e-03],\n",
      "           [-3.8730e-02, -1.7988e-02,  2.4166e-03,  ...,  2.8626e-02,\n",
      "             3.5867e-02,  3.9413e-02]]],\n",
      "\n",
      "\n",
      "         [[[ 9.0331e-03, -2.0729e-03,  7.2528e-03,  ..., -9.9200e-03,\n",
      "            -1.0596e-02,  8.7780e-04],\n",
      "           [-8.5089e-03, -1.3709e-02, -4.1685e-03,  ..., -1.6464e-03,\n",
      "            -4.5926e-03, -5.2013e-04],\n",
      "           [-1.4893e-02, -9.3009e-03,  1.0143e-03,  ...,  9.0517e-03,\n",
      "             1.7703e-03, -4.1789e-03],\n",
      "           ...,\n",
      "           [ 1.3047e-02,  2.0930e-03,  1.2549e-02,  ...,  9.7706e-03,\n",
      "             9.4636e-03, -3.1664e-03],\n",
      "           [-1.1050e-02, -7.4871e-03,  6.3210e-04,  ..., -4.9019e-04,\n",
      "            -2.1274e-03, -2.3798e-02],\n",
      "           [-2.3065e-02, -1.4474e-02,  6.4888e-04,  ..., -1.7101e-03,\n",
      "            -5.9928e-03, -3.2720e-02]],\n",
      "\n",
      "          [[ 2.7672e-02,  3.4122e-03, -6.2054e-03,  ...,  7.3945e-03,\n",
      "             8.1845e-03,  7.2732e-03],\n",
      "           [-6.5464e-03, -1.2667e-02, -1.7991e-02,  ...,  7.0967e-03,\n",
      "             4.0214e-03, -1.1869e-02],\n",
      "           [-3.0196e-02, -8.2158e-03, -4.6819e-03,  ...,  1.4333e-02,\n",
      "             4.5052e-03, -1.8165e-02],\n",
      "           ...,\n",
      "           [-4.9800e-03,  6.3622e-03,  2.0115e-02,  ..., -2.5122e-03,\n",
      "            -8.5156e-03, -1.6088e-02],\n",
      "           [-4.7404e-02, -2.1270e-02, -3.9956e-03,  ..., -1.6191e-02,\n",
      "            -2.4144e-02, -4.3234e-02],\n",
      "           [-7.3874e-02, -3.2393e-02, -7.3596e-03,  ..., -6.7751e-03,\n",
      "            -1.2680e-02, -3.6134e-02]]]],\n",
      "\n",
      "\n",
      "\n",
      "        [[[[-3.2745e-02,  4.3010e-03,  2.0125e-02,  ..., -5.6401e-03,\n",
      "            -1.4653e-02, -2.8919e-02],\n",
      "           [-2.3962e-02,  8.9175e-03,  2.3419e-02,  ...,  9.8595e-03,\n",
      "             4.4707e-03, -8.3696e-03],\n",
      "           [-2.0571e-02,  1.0091e-02,  2.4327e-02,  ...,  6.5546e-03,\n",
      "             4.5507e-03,  2.0242e-03],\n",
      "           ...,\n",
      "           [-1.7121e-02, -1.9688e-02, -8.2728e-03,  ..., -5.3899e-03,\n",
      "             8.6326e-04,  3.1473e-02],\n",
      "           [-3.1481e-02, -2.7069e-02, -1.3522e-02,  ...,  4.7846e-03,\n",
      "             1.1515e-02,  4.4153e-02],\n",
      "           [-2.7278e-02, -1.0917e-02,  6.6322e-03,  ..., -3.8987e-03,\n",
      "            -8.0236e-03,  1.3771e-02]],\n",
      "\n",
      "          [[-5.5262e-02, -2.4934e-02, -1.7649e-02,  ...,  1.8304e-02,\n",
      "             9.5023e-04, -9.8591e-04],\n",
      "           [-4.6085e-02, -1.7792e-02, -9.4929e-03,  ..., -2.0834e-02,\n",
      "            -2.7801e-02, -2.2656e-02],\n",
      "           [-1.6115e-02,  7.8335e-03,  1.8600e-02,  ..., -5.1311e-03,\n",
      "            -1.6798e-02, -7.4860e-03],\n",
      "           ...,\n",
      "           [ 9.6698e-03,  3.7354e-02,  2.3456e-02,  ..., -3.6139e-03,\n",
      "            -2.3048e-02, -1.3817e-02],\n",
      "           [-3.8912e-02,  1.4799e-03, -1.8574e-03,  ...,  2.5973e-02,\n",
      "             5.4744e-03,  8.8645e-03],\n",
      "           [-7.0004e-02, -3.4860e-04,  2.1320e-02,  ..., -1.2720e-02,\n",
      "            -5.6828e-02, -8.0625e-02]]],\n",
      "\n",
      "\n",
      "         [[[-6.7155e-03,  9.1656e-03,  1.8305e-02,  ..., -4.5398e-03,\n",
      "            -1.2337e-03,  5.7674e-03],\n",
      "           [-1.3759e-02,  1.2663e-03,  1.4623e-02,  ...,  6.9202e-03,\n",
      "             1.1269e-02,  1.8520e-02],\n",
      "           [-1.4839e-02,  1.0755e-03,  1.5805e-02,  ...,  4.1445e-03,\n",
      "             9.9513e-03,  2.5785e-02],\n",
      "           ...,\n",
      "           [ 2.1642e-02, -9.1742e-03, -1.0298e-02,  ..., -1.0176e-02,\n",
      "            -6.5267e-03,  2.7685e-02],\n",
      "           [ 7.5950e-03, -1.7673e-02, -1.8834e-02,  ...,  3.3202e-04,\n",
      "             5.4221e-03,  4.1175e-02],\n",
      "           [ 1.5883e-02, -4.0746e-04, -1.8612e-03,  ..., -1.1525e-02,\n",
      "            -1.6390e-02,  1.1172e-02]],\n",
      "\n",
      "          [[-1.4141e-02, -7.3836e-03, -5.3821e-03,  ...,  2.9578e-02,\n",
      "             2.4233e-02,  4.5882e-02],\n",
      "           [-3.1071e-02, -2.1931e-02, -1.4336e-02,  ..., -2.0778e-02,\n",
      "            -1.6700e-02,  1.1542e-02],\n",
      "           [-1.2871e-02, -5.8648e-03,  5.7997e-03,  ..., -7.1084e-03,\n",
      "            -1.0523e-02,  2.0482e-02],\n",
      "           ...,\n",
      "           [ 4.3732e-02,  3.9170e-02,  1.0820e-02,  ...,  1.5099e-04,\n",
      "            -2.0231e-02, -7.3529e-03],\n",
      "           [-8.9531e-03, -1.3580e-03, -1.9824e-02,  ...,  3.5745e-02,\n",
      "             1.4701e-02,  2.2066e-02],\n",
      "           [-3.9593e-02, -3.8132e-03, -7.0566e-04,  ..., -3.4843e-03,\n",
      "            -4.6063e-02, -6.2590e-02]]],\n",
      "\n",
      "\n",
      "         [[[ 3.7390e-02,  3.0025e-02,  3.2556e-02,  ..., -5.4585e-03,\n",
      "            -8.0432e-03, -1.9795e-02],\n",
      "           [ 6.3344e-03, -1.0970e-04,  6.9666e-03,  ...,  1.1141e-02,\n",
      "             1.2567e-02,  7.5960e-05],\n",
      "           [-6.0149e-03, -1.0631e-02, -2.3613e-03,  ...,  5.0847e-03,\n",
      "             7.8438e-03,  3.6487e-03],\n",
      "           ...,\n",
      "           [ 2.5015e-02, -9.4144e-03, -1.1331e-02,  ..., -2.1670e-03,\n",
      "            -3.7836e-03,  1.1984e-02],\n",
      "           [ 6.8261e-03, -2.2246e-02, -2.3433e-02,  ...,  6.1813e-03,\n",
      "             5.7872e-03,  2.6400e-02],\n",
      "           [ 3.5536e-03, -1.6595e-02, -1.8490e-02,  ..., -1.3706e-04,\n",
      "            -9.4461e-03,  4.7371e-03]],\n",
      "\n",
      "          [[ 5.9819e-02,  4.0980e-02,  3.4804e-02,  ...,  1.0136e-02,\n",
      "            -6.7042e-03, -1.2844e-02],\n",
      "           [ 8.6719e-03, -4.7123e-03, -1.6979e-03,  ..., -2.4330e-02,\n",
      "            -3.0036e-02, -3.0666e-02],\n",
      "           [ 5.9193e-03, -6.0498e-03,  6.8297e-04,  ..., -1.2424e-02,\n",
      "            -2.3481e-02, -2.0437e-02],\n",
      "           ...,\n",
      "           [ 2.9659e-02,  2.6941e-02,  2.9632e-03,  ...,  8.0120e-03,\n",
      "            -1.3568e-02, -1.4277e-02],\n",
      "           [-2.3256e-02, -1.5085e-02, -3.2233e-02,  ...,  4.4778e-02,\n",
      "             2.5377e-02,  2.2131e-02],\n",
      "           [-6.1187e-02, -2.7356e-02, -2.5540e-02,  ...,  2.2716e-02,\n",
      "            -1.8819e-02, -4.2655e-02]]]],\n",
      "\n",
      "\n",
      "\n",
      "        [[[[-4.4175e-02, -3.0375e-02, -1.7998e-02,  ...,  2.2911e-03,\n",
      "             4.6033e-03,  3.0973e-02],\n",
      "           [-1.5905e-02, -1.2564e-02, -5.0489e-03,  ..., -4.2434e-03,\n",
      "            -2.6008e-03,  1.9722e-02],\n",
      "           [-6.3521e-03, -4.0911e-03, -1.2887e-04,  ..., -8.9927e-03,\n",
      "            -5.7172e-03,  8.4477e-03],\n",
      "           ...,\n",
      "           [-1.7441e-03, -8.0828e-03, -6.9113e-03,  ..., -5.0599e-04,\n",
      "            -1.1811e-02, -8.7751e-03],\n",
      "           [-3.1469e-03, -1.6512e-03,  6.0092e-04,  ...,  3.6428e-04,\n",
      "            -1.4259e-02, -1.6863e-02],\n",
      "           [-1.1790e-02, -1.0694e-02, -1.2134e-02,  ..., -4.2154e-03,\n",
      "            -1.1587e-02, -1.6952e-02]],\n",
      "\n",
      "          [[-5.3010e-02, -3.5188e-02, -1.7007e-02,  ...,  3.9218e-02,\n",
      "             3.6542e-02,  4.0299e-02],\n",
      "           [-1.4247e-02, -5.5609e-03,  1.1426e-02,  ...,  4.6791e-02,\n",
      "             3.3162e-02,  2.7748e-02],\n",
      "           [-1.7933e-02, -1.4149e-02,  5.3289e-03,  ...,  8.1033e-04,\n",
      "            -6.4202e-03, -1.3056e-02],\n",
      "           ...,\n",
      "           [ 9.4701e-03, -7.6917e-03, -5.2160e-04,  ..., -1.7835e-03,\n",
      "             2.4619e-04,  1.6679e-02],\n",
      "           [ 4.7648e-03, -1.6380e-03,  7.7995e-03,  ...,  4.4829e-03,\n",
      "            -2.7592e-03,  3.5809e-03],\n",
      "           [ 6.8120e-03, -6.4213e-03, -1.0612e-02,  ...,  2.9326e-03,\n",
      "            -4.4980e-03,  1.9738e-03]]],\n",
      "\n",
      "\n",
      "         [[[ 1.6480e-02,  1.8586e-04, -2.0793e-03,  ..., -9.2832e-03,\n",
      "            -1.1162e-02, -1.7704e-03],\n",
      "           [ 2.0605e-02, -8.2641e-04, -2.0550e-03,  ...,  7.7574e-04,\n",
      "            -1.3529e-03,  3.7987e-03],\n",
      "           [ 1.2884e-02, -2.8725e-03, -2.5126e-03,  ..., -1.5800e-03,\n",
      "            -1.6368e-03, -2.8260e-03],\n",
      "           ...,\n",
      "           [ 2.4708e-03, -9.0674e-03, -8.6137e-03,  ...,  1.0718e-02,\n",
      "            -1.4298e-03, -3.6486e-03],\n",
      "           [ 3.6190e-03, -2.2343e-03, -5.7190e-03,  ...,  7.8425e-03,\n",
      "            -5.5346e-03, -1.0040e-02],\n",
      "           [-3.6375e-03, -1.1999e-02, -2.1573e-02,  ..., -1.6991e-03,\n",
      "            -5.7005e-03, -1.0621e-02]],\n",
      "\n",
      "          [[ 1.1271e-02, -3.5781e-03, -7.3345e-04,  ...,  1.2119e-02,\n",
      "            -1.2831e-03, -2.9581e-02],\n",
      "           [ 2.1634e-02,  4.1918e-03,  1.4154e-02,  ...,  4.4784e-02,\n",
      "             1.9971e-02, -1.9290e-02],\n",
      "           [-5.6105e-03, -1.9922e-02, -9.2302e-04,  ...,  5.7533e-03,\n",
      "            -1.1578e-02, -5.0451e-02],\n",
      "           ...,\n",
      "           [ 1.6211e-02, -7.5042e-03,  1.9081e-04,  ...,  9.7375e-03,\n",
      "             7.5248e-03,  5.0597e-03],\n",
      "           [ 1.7321e-02,  2.9578e-03,  9.0430e-03,  ...,  1.3060e-02,\n",
      "             1.3769e-03, -7.3998e-03],\n",
      "           [ 2.5755e-02,  9.9244e-04, -1.1633e-02,  ...,  4.3569e-03,\n",
      "            -5.0655e-03, -1.1969e-02]]],\n",
      "\n",
      "\n",
      "         [[[ 3.4696e-02,  2.7682e-02,  2.1698e-02,  ...,  1.6325e-02,\n",
      "             1.9848e-02,  4.6865e-02],\n",
      "           [ 8.6077e-03, -5.7728e-04, -4.9442e-03,  ...,  3.6503e-03,\n",
      "             8.6670e-03,  3.2572e-02],\n",
      "           [-1.0091e-02, -1.0028e-02, -1.0820e-02,  ..., -2.1662e-03,\n",
      "             4.0087e-03,  2.0010e-02],\n",
      "           ...,\n",
      "           [-1.5508e-02, -6.0864e-03, -4.6016e-03,  ...,  6.1978e-03,\n",
      "            -6.1220e-04,  1.0321e-02],\n",
      "           [-1.3728e-02, -1.3114e-03, -2.5226e-03,  ..., -7.8156e-05,\n",
      "            -6.3445e-03,  1.9612e-03],\n",
      "           [-1.2387e-02, -5.8632e-03, -1.2567e-02,  ..., -6.4450e-03,\n",
      "            -1.6137e-03,  5.6248e-03]],\n",
      "\n",
      "          [[ 4.4222e-02,  3.5191e-02,  2.8158e-02,  ...,  2.8834e-02,\n",
      "             2.1267e-02,  5.4875e-03],\n",
      "           [ 1.8041e-02,  1.0187e-02,  1.1014e-02,  ...,  3.8776e-02,\n",
      "             2.1408e-02, -3.6986e-03],\n",
      "           [-2.1992e-02, -1.9482e-02, -7.9045e-03,  ...,  3.3012e-04,\n",
      "            -1.2791e-02, -3.9795e-02],\n",
      "           ...,\n",
      "           [ 1.2555e-02,  9.5198e-03,  1.6645e-02,  ...,  8.7359e-03,\n",
      "            -2.0590e-03,  2.1645e-03],\n",
      "           [ 1.6487e-02,  1.6052e-02,  2.1033e-02,  ...,  3.9806e-03,\n",
      "            -1.0366e-02, -1.1161e-02],\n",
      "           [ 4.2044e-02,  2.8314e-02,  1.5705e-02,  ..., -2.3382e-03,\n",
      "            -1.1885e-02, -1.1243e-02]]]]])), ('module.backbone.patch_embed.proj.bias', tensor([ 0.1006,  0.0790,  0.0679,  ...,  0.1730, -0.1150, -0.1844])), ('module.backbone.blocks.0.norm1.weight', tensor([0.1243, 0.0011, 0.0372,  ..., 0.1686, 0.3993, 0.5332])), ('module.backbone.blocks.0.norm1.bias', tensor([ 0.0045,  0.0002,  0.0057,  ..., -0.1715, -0.2044, -0.2177])), ('module.backbone.blocks.0.attn.qkv.weight', tensor([[-0.0239,  0.0018, -0.0141,  ..., -0.0003, -0.0152, -0.0102],\n",
      "        [ 0.0014, -0.0009,  0.0006,  ...,  0.0032,  0.0051,  0.0047],\n",
      "        [ 0.0092,  0.0003,  0.0040,  ...,  0.0043,  0.0037, -0.0003],\n",
      "        ...,\n",
      "        [ 0.0108,  0.0028, -0.0044,  ..., -0.0364, -0.0064,  0.0599],\n",
      "        [-0.0058, -0.0009,  0.0058,  ..., -0.0032,  0.0025, -0.0129],\n",
      "        [ 0.0031,  0.0041,  0.0036,  ..., -0.0440,  0.0303, -0.0171]])), ('module.backbone.blocks.0.attn.qkv.bias', tensor([-0.4266, -0.1599, -0.3442,  ..., -0.0042, -0.0073, -0.0186])), ('module.backbone.blocks.0.attn.proj.weight', tensor([[-0.0006,  0.0275,  0.0122,  ...,  0.0039,  0.0072,  0.0033],\n",
      "        [ 0.0081, -0.0012,  0.0154,  ...,  0.0040, -0.0005, -0.0022],\n",
      "        [ 0.0055, -0.0034,  0.0195,  ...,  0.0063, -0.0003, -0.0018],\n",
      "        ...,\n",
      "        [ 0.0034,  0.0132, -0.0015,  ...,  0.0269,  0.0022,  0.0420],\n",
      "        [ 0.0233,  0.0016,  0.0056,  ...,  0.0070, -0.0028, -0.0281],\n",
      "        [ 0.0083, -0.0093, -0.0263,  ..., -0.0672,  0.0241,  0.0300]])), ('module.backbone.blocks.0.attn.proj.bias', tensor([ 0.1236,  0.1094,  0.0695,  ..., -0.0404, -0.1339, -0.0435])), ('module.backbone.blocks.0.norm2.weight', tensor([0.5820, 0.6190, 0.3092,  ..., 0.7605, 1.4875, 1.7843])), ('module.backbone.blocks.0.norm2.bias', tensor([-0.0019, -0.1101, -0.0655,  ..., -0.0797,  0.1322,  0.0425])), ('module.backbone.blocks.0.mlp.fc1.weight', tensor([[ 0.0035, -0.0027,  0.0026,  ...,  0.0134, -0.0022,  0.0256],\n",
      "        [-0.0116, -0.0011,  0.0022,  ...,  0.0039,  0.0007, -0.0049],\n",
      "        [-0.0153,  0.0062, -0.0015,  ...,  0.0037, -0.0170,  0.0046],\n",
      "        ...,\n",
      "        [ 0.0053, -0.0002, -0.0058,  ..., -0.0010,  0.0323, -0.0017],\n",
      "        [-0.0190,  0.0210, -0.0084,  ...,  0.0128, -0.0017,  0.0022],\n",
      "        [ 0.0076,  0.0015,  0.0025,  ..., -0.0132,  0.0251,  0.0207]])), ('module.backbone.blocks.0.mlp.fc1.bias', tensor([ 0.0996, -0.3235, -0.4333,  ...,  0.2177,  0.1871,  0.1776])), ('module.backbone.blocks.0.mlp.fc2.weight', tensor([[-0.0036, -0.0079, -0.0052,  ...,  0.0081, -0.0132, -0.0040],\n",
      "        [ 0.0047,  0.0105,  0.0068,  ...,  0.0016,  0.0240,  0.0029],\n",
      "        [ 0.0004,  0.0035,  0.0054,  ..., -0.0045, -0.0103, -0.0012],\n",
      "        ...,\n",
      "        [-0.0047, -0.0070,  0.0159,  ...,  0.0120, -0.0144, -0.0059],\n",
      "        [ 0.0099,  0.0068, -0.0072,  ...,  0.0229, -0.0105,  0.0148],\n",
      "        [ 0.0355, -0.0053,  0.0127,  ..., -0.0027,  0.0085,  0.0074]])), ('module.backbone.blocks.0.mlp.fc2.bias', tensor([ 0.0641,  0.0536, -0.0872,  ...,  0.0246, -0.0535,  0.0736])), ('module.backbone.blocks.1.norm1.weight', tensor([0.2771, 0.0726, 0.3090,  ..., 0.3962, 0.5127, 0.4205])), ('module.backbone.blocks.1.norm1.bias', tensor([ 0.0423, -0.0369,  0.0189,  ...,  0.0341,  0.0552,  0.0827])), ('module.backbone.blocks.1.attn.qkv.weight', tensor([[-0.0147, -0.0054, -0.0028,  ..., -0.0185, -0.0177, -0.0229],\n",
      "        [ 0.0190,  0.0071,  0.0025,  ...,  0.0076, -0.0124,  0.0028],\n",
      "        [ 0.0080,  0.0056,  0.0047,  ..., -0.0062, -0.0144, -0.0010],\n",
      "        ...,\n",
      "        [ 0.0110,  0.0143,  0.0056,  ...,  0.0009, -0.0062, -0.0107],\n",
      "        [-0.0014, -0.0060, -0.0081,  ...,  0.0164,  0.0072, -0.0186],\n",
      "        [-0.0133, -0.0044, -0.0072,  ..., -0.0145, -0.0025,  0.0127]])), ('module.backbone.blocks.1.attn.qkv.bias', tensor([-0.6793,  0.3230,  0.3712,  ..., -0.0139, -0.0071, -0.0012])), ('module.backbone.blocks.1.attn.proj.weight', tensor([[-2.5709e-03, -5.6698e-03,  6.4382e-03,  ...,  4.9765e-05,\n",
      "          1.0310e-03, -2.1387e-03],\n",
      "        [ 1.0437e-02, -1.5403e-03, -6.6214e-03,  ..., -6.7161e-04,\n",
      "          2.3625e-03,  5.7145e-04],\n",
      "        [ 1.3157e-04,  9.9570e-03,  7.7672e-05,  ..., -5.4828e-03,\n",
      "          9.1308e-03,  8.9672e-03],\n",
      "        ...,\n",
      "        [ 1.7657e-02,  1.6556e-02,  1.8078e-02,  ...,  6.4267e-04,\n",
      "         -2.3735e-02,  2.1142e-02],\n",
      "        [-2.2308e-02, -1.1036e-02,  1.3257e-02,  ...,  1.3619e-02,\n",
      "         -1.5436e-03,  1.2809e-02],\n",
      "        [ 1.9163e-02, -1.1788e-02,  1.8983e-02,  ...,  1.4662e-02,\n",
      "          8.8473e-03, -1.6293e-04]])), ('module.backbone.blocks.1.attn.proj.bias', tensor([ 0.0690,  0.0568, -0.0630,  ...,  0.0042, -0.0344,  0.0634])), ('module.backbone.blocks.1.norm2.weight', tensor([0.1961, 0.3423, 0.0225,  ..., 0.4402, 0.5549, 0.7055])), ('module.backbone.blocks.1.norm2.bias', tensor([-0.0025, -0.0003, -0.0877,  ...,  0.0249,  0.0134,  0.0654])), ('module.backbone.blocks.1.mlp.fc1.weight', tensor([[ 0.0130,  0.0027,  0.0192,  ...,  0.0065,  0.0028,  0.0128],\n",
      "        [ 0.0087, -0.0062,  0.0149,  ..., -0.0186,  0.0148,  0.0060],\n",
      "        [ 0.0005, -0.0003,  0.0095,  ...,  0.0092,  0.0058, -0.0086],\n",
      "        ...,\n",
      "        [ 0.0007, -0.0064,  0.0079,  ...,  0.0034, -0.0033, -0.0108],\n",
      "        [-0.0058, -0.0027, -0.0063,  ..., -0.0130,  0.0178,  0.0007],\n",
      "        [ 0.0003, -0.0033,  0.0109,  ...,  0.0110,  0.0049, -0.0110]])), ('module.backbone.blocks.1.mlp.fc1.bias', tensor([-0.2282, -0.2127, -0.2997,  ..., -0.3342, -0.1283, -0.3021])), ('module.backbone.blocks.1.mlp.fc2.weight', tensor([[ 0.0176, -0.0032,  0.0046,  ..., -0.0003, -0.0038,  0.0055],\n",
      "        [ 0.0070,  0.0132,  0.0041,  ..., -0.0046,  0.0016,  0.0021],\n",
      "        [ 0.0014,  0.0099,  0.0012,  ...,  0.0040, -0.0021, -0.0005],\n",
      "        ...,\n",
      "        [ 0.0337, -0.0022,  0.0074,  ..., -0.0032, -0.0016,  0.0063],\n",
      "        [ 0.0177,  0.0014,  0.0004,  ..., -0.0030,  0.0039,  0.0023],\n",
      "        [ 0.0030,  0.0044,  0.0017,  ...,  0.0004, -0.0118, -0.0004]])), ('module.backbone.blocks.1.mlp.fc2.bias', tensor([ 0.0357,  0.0088, -0.0209,  ..., -0.0025,  0.0201,  0.0619])), ('module.backbone.blocks.2.norm1.weight', tensor([0.8812, 0.6683, 1.1499,  ..., 0.6100, 0.6947, 0.6345])), ('module.backbone.blocks.2.norm1.bias', tensor([ 0.0531, -0.1279,  0.1018,  ..., -0.0324,  0.0126,  0.0077])), ('module.backbone.blocks.2.attn.qkv.weight', tensor([[-0.0020, -0.0020,  0.0082,  ..., -0.0254,  0.0083, -0.0072],\n",
      "        [-0.0025, -0.0077,  0.0124,  ...,  0.0179, -0.0032,  0.0127],\n",
      "        [-0.0106, -0.0071, -0.0130,  ..., -0.0183,  0.0059, -0.0226],\n",
      "        ...,\n",
      "        [ 0.0091,  0.0011, -0.0410,  ..., -0.0061,  0.0075,  0.0175],\n",
      "        [ 0.0071,  0.0069,  0.0221,  ..., -0.0099, -0.0043,  0.0043],\n",
      "        [-0.0067, -0.0063, -0.0140,  ..., -0.0038,  0.0056,  0.0145]])), ('module.backbone.blocks.2.attn.qkv.bias', tensor([-0.5150, -0.1678,  0.0071,  ...,  0.0239,  0.0052,  0.0042])), ('module.backbone.blocks.2.attn.proj.weight', tensor([[-0.0066,  0.0058,  0.0102,  ..., -0.0070, -0.0030,  0.0045],\n",
      "        [ 0.0030, -0.0100,  0.0040,  ...,  0.0020,  0.0089,  0.0083],\n",
      "        [-0.0120, -0.0006,  0.0044,  ...,  0.0216, -0.0085,  0.0042],\n",
      "        ...,\n",
      "        [ 0.0067,  0.0001, -0.0235,  ...,  0.0025,  0.0182, -0.0070],\n",
      "        [-0.0156, -0.0074, -0.0129,  ...,  0.0015,  0.0095, -0.0116],\n",
      "        [ 0.0165,  0.0160, -0.0109,  ...,  0.0030, -0.0049, -0.0262]])), ('module.backbone.blocks.2.attn.proj.bias', tensor([ 0.0212,  0.0452, -0.0446,  ...,  0.0036,  0.0341,  0.0469])), ('module.backbone.blocks.2.norm2.weight', tensor([0.7082, 0.7826, 0.5900,  ..., 0.7561, 1.0094, 0.9491])), ('module.backbone.blocks.2.norm2.bias', tensor([-0.0062, -0.0441, -0.0956,  ...,  0.0036, -0.0351, -0.0131])), ('module.backbone.blocks.2.mlp.fc1.weight', tensor([[-6.5913e-03, -1.1715e-02, -2.0507e-03,  ...,  1.4932e-02,\n",
      "         -3.4064e-02,  1.7015e-02],\n",
      "        [ 1.5557e-02, -1.6670e-02, -3.2599e-03,  ...,  5.2605e-03,\n",
      "         -1.9165e-02,  2.6947e-03],\n",
      "        [-4.0728e-03, -4.5028e-03,  1.7908e-02,  ..., -8.6352e-03,\n",
      "          7.5763e-04, -9.0153e-03],\n",
      "        ...,\n",
      "        [-2.8229e-03, -1.2314e-02,  1.4126e-02,  ..., -1.1748e-02,\n",
      "          4.9039e-03,  5.1814e-03],\n",
      "        [ 5.3139e-03, -1.0816e-02,  5.8992e-03,  ..., -5.0271e-03,\n",
      "         -9.8033e-03, -5.3763e-03],\n",
      "        [ 4.5654e-03,  5.1645e-03, -8.6651e-05,  ..., -5.2469e-03,\n",
      "         -3.7056e-03,  8.5371e-03]])), ('module.backbone.blocks.2.mlp.fc1.bias', tensor([-0.1454, -0.0198, -0.2655,  ..., -0.1240, -0.1413, -0.0802])), ('module.backbone.blocks.2.mlp.fc2.weight', tensor([[ 0.0137,  0.0225, -0.0026,  ..., -0.0009,  0.0065,  0.0047],\n",
      "        [ 0.0180, -0.0156,  0.0089,  ..., -0.0078, -0.0036, -0.0025],\n",
      "        [ 0.0026, -0.0084,  0.0130,  ..., -0.0045, -0.0051, -0.0046],\n",
      "        ...,\n",
      "        [-0.0259,  0.0010,  0.0037,  ...,  0.0055,  0.0112,  0.0108],\n",
      "        [ 0.0152,  0.0075, -0.0011,  ..., -0.0022,  0.0310, -0.0037],\n",
      "        [ 0.0082, -0.0019,  0.0004,  ...,  0.0031, -0.0106, -0.0016]])), ('module.backbone.blocks.2.mlp.fc2.bias', tensor([ 0.0110,  0.0267,  0.0006,  ..., -0.0105,  0.0484,  0.0388])), ('module.backbone.blocks.3.norm1.weight', tensor([0.8641, 0.9104, 1.3143,  ..., 0.6709, 0.8959, 0.8146])), ('module.backbone.blocks.3.norm1.bias', tensor([ 0.0536, -0.0820,  0.1364,  ..., -0.0445,  0.0124,  0.0485])), ('module.backbone.blocks.3.attn.qkv.weight', tensor([[-0.0166, -0.0042, -0.0054,  ...,  0.0010,  0.0352,  0.0087],\n",
      "        [-0.0223, -0.0078,  0.0034,  ...,  0.0065,  0.0167,  0.0072],\n",
      "        [ 0.0046,  0.0101,  0.0017,  ..., -0.0257, -0.0423, -0.0099],\n",
      "        ...,\n",
      "        [ 0.0004,  0.0005,  0.0008,  ..., -0.0189,  0.0048, -0.0062],\n",
      "        [ 0.0043,  0.0082, -0.0071,  ...,  0.0200,  0.0125, -0.0194],\n",
      "        [ 0.0076,  0.0032,  0.0017,  ...,  0.0106, -0.0019,  0.0096]])), ('module.backbone.blocks.3.attn.qkv.bias', tensor([-0.1545, -0.1416,  0.2614,  ..., -0.0060, -0.0460, -0.0273])), ('module.backbone.blocks.3.attn.proj.weight', tensor([[ 3.5050e-03,  6.6373e-03, -2.3483e-02,  ..., -5.6558e-03,\n",
      "          6.6697e-03,  1.0169e-03],\n",
      "        [-1.6379e-03,  3.6270e-03, -4.9517e-03,  ...,  1.1088e-02,\n",
      "         -1.0308e-02, -6.1122e-03],\n",
      "        [ 1.6931e-02,  2.9925e-03,  7.8401e-03,  ..., -5.8266e-03,\n",
      "          7.3287e-03,  8.2169e-04],\n",
      "        ...,\n",
      "        [-1.3763e-02, -4.0481e-03,  3.4315e-03,  ...,  5.9629e-03,\n",
      "         -1.4742e-02,  2.7181e-04],\n",
      "        [-1.8954e-02, -1.0003e-02,  2.7168e-02,  ...,  3.4164e-03,\n",
      "         -1.2421e-02,  1.2676e-05],\n",
      "        [ 4.2922e-04, -9.9936e-03, -2.4039e-02,  ...,  1.3306e-02,\n",
      "          2.6548e-03, -8.9338e-04]])), ('module.backbone.blocks.3.attn.proj.bias', tensor([ 0.0068,  0.0638, -0.0620,  ...,  0.0264,  0.0605, -0.0015])), ('module.backbone.blocks.3.norm2.weight', tensor([0.9302, 1.1130, 1.0086,  ..., 0.8506, 1.2796, 1.1700])), ('module.backbone.blocks.3.norm2.bias', tensor([ 0.0161,  0.0236, -0.1147,  ...,  0.0236,  0.0127,  0.0154])), ('module.backbone.blocks.3.mlp.fc1.weight', tensor([[-1.5643e-02, -4.9243e-03,  1.5904e-02,  ..., -2.7883e-02,\n",
      "          1.1617e-02,  2.6634e-02],\n",
      "        [-9.6673e-03, -1.0326e-02,  2.0304e-02,  ..., -7.3520e-03,\n",
      "          5.9833e-03, -5.9503e-03],\n",
      "        [-7.6621e-03, -7.2658e-03,  1.5710e-02,  ..., -1.2384e-03,\n",
      "          6.7930e-04,  6.8924e-05],\n",
      "        ...,\n",
      "        [ 4.7274e-03, -2.9778e-03,  1.7514e-02,  ..., -4.9730e-03,\n",
      "          6.3129e-03,  4.4663e-03],\n",
      "        [-2.4689e-03,  3.1030e-02,  1.4372e-02,  ..., -2.8089e-02,\n",
      "         -3.3419e-03,  1.5235e-02],\n",
      "        [ 2.0060e-02,  2.5087e-02,  9.6320e-03,  ..., -6.7825e-03,\n",
      "         -9.2600e-03, -3.5435e-03]])), ('module.backbone.blocks.3.mlp.fc1.bias', tensor([-0.1357, -0.2863, -0.1910,  ..., -0.3095, -0.0662, -0.0768])), ('module.backbone.blocks.3.mlp.fc2.weight', tensor([[ 1.8050e-02,  1.6337e-03, -4.3256e-03,  ...,  6.7838e-04,\n",
      "          1.9097e-03,  9.7881e-04],\n",
      "        [-1.9089e-02,  1.0568e-03,  3.2453e-06,  ..., -4.4295e-03,\n",
      "         -5.5294e-03,  3.0788e-03],\n",
      "        [-2.0924e-03,  1.4790e-02,  4.8409e-03,  ..., -6.1862e-03,\n",
      "          6.9618e-03,  1.1534e-02],\n",
      "        ...,\n",
      "        [ 2.5195e-02,  2.4536e-03,  1.2225e-03,  ..., -8.0491e-04,\n",
      "          2.6749e-02, -8.5551e-03],\n",
      "        [-8.0456e-03,  1.2883e-03, -5.4868e-03,  ..., -6.4024e-03,\n",
      "          1.1479e-02, -1.1602e-02],\n",
      "        [ 2.7010e-02, -5.1959e-03,  1.9330e-04,  ..., -5.4917e-03,\n",
      "          9.7287e-03,  4.7382e-03]])), ('module.backbone.blocks.3.mlp.fc2.bias', tensor([-0.0223,  0.0246,  0.0022,  ..., -0.0068,  0.0552,  0.0020])), ('module.backbone.blocks.4.norm1.weight', tensor([1.0127, 1.0977, 1.5737,  ..., 0.6938, 1.0040, 0.9079])), ('module.backbone.blocks.4.norm1.bias', tensor([-0.0061,  0.0359,  0.2018,  ..., -0.0076,  0.0385,  0.0213])), ('module.backbone.blocks.4.attn.qkv.weight', tensor([[-0.0222, -0.0314, -0.0469,  ..., -0.0144,  0.0202,  0.0100],\n",
      "        [ 0.0138, -0.0355, -0.0468,  ...,  0.0105,  0.0183,  0.0036],\n",
      "        [-0.0218, -0.0182, -0.0283,  ..., -0.0101, -0.0044, -0.0190],\n",
      "        ...,\n",
      "        [-0.0018, -0.0137,  0.0026,  ...,  0.0005,  0.0144, -0.0004],\n",
      "        [ 0.0024, -0.0017, -0.0051,  ...,  0.0196, -0.0108, -0.0450],\n",
      "        [-0.0094,  0.0096, -0.0015,  ..., -0.0214, -0.0024, -0.0075]])), ('module.backbone.blocks.4.attn.qkv.bias', tensor([-0.0784,  0.5206, -0.1351,  ..., -0.0067, -0.0383,  0.0177])), ('module.backbone.blocks.4.attn.proj.weight', tensor([[-0.0030,  0.0058,  0.0003,  ..., -0.0036, -0.0004,  0.0059],\n",
      "        [ 0.0041, -0.0053,  0.0114,  ...,  0.0026,  0.0039, -0.0112],\n",
      "        [-0.0113, -0.0070, -0.0106,  ..., -0.0030,  0.0048,  0.0082],\n",
      "        ...,\n",
      "        [ 0.0181,  0.0262, -0.0056,  ..., -0.0046, -0.0146,  0.0348],\n",
      "        [-0.0072,  0.0215,  0.0051,  ..., -0.0087,  0.0194, -0.0028],\n",
      "        [ 0.0021,  0.0188,  0.0037,  ..., -0.0054,  0.0272,  0.0132]])), ('module.backbone.blocks.4.attn.proj.bias', tensor([-0.0074,  0.0265, -0.0955,  ...,  0.0172,  0.0422, -0.0258])), ('module.backbone.blocks.4.norm2.weight', tensor([0.5268, 1.2042, 1.7183,  ..., 0.9100, 1.4300, 1.4079])), ('module.backbone.blocks.4.norm2.bias', tensor([-0.0463,  0.0300, -0.1632,  ...,  0.0451,  0.0358, -0.0352])), ('module.backbone.blocks.4.mlp.fc1.weight', tensor([[-0.0031,  0.0061, -0.0047,  ...,  0.0194, -0.0041,  0.0146],\n",
      "        [ 0.0023, -0.0130,  0.0075,  ...,  0.0154,  0.0040, -0.0055],\n",
      "        [-0.0017,  0.0001,  0.0142,  ...,  0.0035, -0.0045,  0.0080],\n",
      "        ...,\n",
      "        [ 0.0104, -0.0052,  0.0174,  ..., -0.0186, -0.0160, -0.0027],\n",
      "        [-0.0012,  0.0028,  0.0022,  ...,  0.0222,  0.0070, -0.0130],\n",
      "        [-0.0081, -0.0060,  0.0141,  ..., -0.0152,  0.0027,  0.0125]])), ('module.backbone.blocks.4.mlp.fc1.bias', tensor([-0.1412, -0.1064, -0.2890,  ..., -0.3036, -0.0972, -0.1045])), ('module.backbone.blocks.4.mlp.fc2.weight', tensor([[ 0.0082,  0.0050, -0.0045,  ..., -0.0126,  0.0007, -0.0027],\n",
      "        [ 0.0110, -0.0019,  0.0032,  ...,  0.0045,  0.0150, -0.0102],\n",
      "        [ 0.0047, -0.0055,  0.0061,  ..., -0.0173, -0.0081,  0.0064],\n",
      "        ...,\n",
      "        [ 0.0142,  0.0009,  0.0013,  ..., -0.0026, -0.0127,  0.0137],\n",
      "        [ 0.0145, -0.0012, -0.0037,  ...,  0.0090, -0.0101, -0.0027],\n",
      "        [ 0.0146, -0.0211,  0.0038,  ...,  0.0174, -0.0063, -0.0036]])), ('module.backbone.blocks.4.mlp.fc2.bias', tensor([-0.0102,  0.0135, -0.0013,  ..., -0.0168,  0.0249,  0.0133])), ('module.backbone.blocks.5.norm1.weight', tensor([1.0423, 1.4515, 1.9280,  ..., 0.7542, 1.1890, 1.1589])), ('module.backbone.blocks.5.norm1.bias', tensor([ 0.0087,  0.0537,  0.0807,  ..., -0.0342,  0.0060,  0.0368])), ('module.backbone.blocks.5.attn.qkv.weight', tensor([[ 5.7996e-03,  1.1647e-02, -9.1662e-03,  ...,  2.2268e-02,\n",
      "          2.0380e-03, -5.9582e-03],\n",
      "        [-4.3955e-02, -4.3230e-03,  7.4728e-03,  ...,  5.4224e-03,\n",
      "         -1.6252e-02, -5.0437e-03],\n",
      "        [ 1.4529e-02,  2.7869e-03,  4.9928e-05,  ...,  7.1944e-03,\n",
      "         -1.1372e-02, -5.7278e-03],\n",
      "        ...,\n",
      "        [ 1.6814e-03, -2.3333e-04, -4.4135e-04,  ...,  4.4487e-03,\n",
      "          1.1081e-02, -6.2197e-03],\n",
      "        [-3.3202e-03, -1.7554e-04,  7.4135e-05,  ..., -7.8342e-04,\n",
      "         -3.2450e-03,  1.6506e-02],\n",
      "        [-9.0751e-05,  1.1984e-02, -8.0939e-03,  ...,  2.2669e-02,\n",
      "          2.2119e-02, -2.3487e-02]])), ('module.backbone.blocks.5.attn.qkv.bias', tensor([-0.1214, -0.0230,  0.0265,  ..., -0.0121, -0.0037,  0.0052])), ('module.backbone.blocks.5.attn.proj.weight', tensor([[-0.0111, -0.0072,  0.0012,  ...,  0.0065, -0.0003, -0.0161],\n",
      "        [ 0.0021, -0.0008,  0.0019,  ...,  0.0015, -0.0088,  0.0084],\n",
      "        [-0.0014, -0.0016, -0.0006,  ..., -0.0032, -0.0090, -0.0096],\n",
      "        ...,\n",
      "        [-0.0038,  0.0375,  0.0018,  ...,  0.0050,  0.0011, -0.0103],\n",
      "        [-0.0018, -0.0184,  0.0075,  ..., -0.0435, -0.0123,  0.0091],\n",
      "        [ 0.0239, -0.0069,  0.0072,  ..., -0.0080, -0.0211, -0.0107]])), ('module.backbone.blocks.5.attn.proj.bias', tensor([ 0.0099, -0.0073, -0.0305,  ..., -0.0217,  0.0317, -0.0294])), ('module.backbone.blocks.5.norm2.weight', tensor([0.1245, 0.5136, 0.1565,  ..., 0.9271, 1.3135, 1.3827])), ('module.backbone.blocks.5.norm2.bias', tensor([ 0.0109, -0.0068, -0.1510,  ...,  0.0320,  0.0180, -0.0387])), ('module.backbone.blocks.5.mlp.fc1.weight', tensor([[-0.1250, -0.0985, -0.1177,  ..., -0.0081, -0.0064,  0.0405],\n",
      "        [-0.0239, -0.0175,  0.0066,  ..., -0.0118, -0.0098,  0.0234],\n",
      "        [ 0.0121,  0.0026, -0.0092,  ..., -0.0038, -0.0200,  0.0282],\n",
      "        ...,\n",
      "        [ 0.0226, -0.0112,  0.0176,  ..., -0.0109, -0.0120,  0.0199],\n",
      "        [-0.0116, -0.0320, -0.0045,  ..., -0.0015, -0.0311,  0.0142],\n",
      "        [-0.0099, -0.0087,  0.0027,  ..., -0.0124, -0.0105, -0.0018]])), ('module.backbone.blocks.5.mlp.fc1.bias', tensor([-0.1217, -0.1398, -0.1888,  ..., -0.1713, -0.0666, -0.2080])), ('module.backbone.blocks.5.mlp.fc2.weight', tensor([[ 0.0609, -0.0034, -0.0011,  ...,  0.0072,  0.0045,  0.0081],\n",
      "        [ 0.0768,  0.0163, -0.0076,  ..., -0.0020, -0.0096,  0.0056],\n",
      "        [ 0.0917,  0.0009, -0.0030,  ..., -0.0045, -0.0039,  0.0014],\n",
      "        ...,\n",
      "        [-0.0126,  0.0066,  0.0196,  ...,  0.0171, -0.0057, -0.0008],\n",
      "        [-0.0280,  0.0224,  0.0037,  ...,  0.0203,  0.0040, -0.0112],\n",
      "        [-0.0543, -0.0081,  0.0091,  ..., -0.0069, -0.0020,  0.0042]])), ('module.backbone.blocks.5.mlp.fc2.bias', tensor([-0.0024, -0.0076,  0.0016,  ..., -0.0355,  0.0130, -0.0003])), ('module.backbone.blocks.6.norm1.weight', tensor([0.9162, 0.9250, 1.3339,  ..., 0.8365, 1.1071, 1.0991])), ('module.backbone.blocks.6.norm1.bias', tensor([ 0.0360,  0.1303,  0.1140,  ..., -0.0080,  0.0145,  0.0184])), ('module.backbone.blocks.6.attn.qkv.weight', tensor([[-0.0086, -0.0303, -0.0337,  ..., -0.0098,  0.0153, -0.0138],\n",
      "        [ 0.0071,  0.0181, -0.0012,  ..., -0.0064,  0.0199, -0.0231],\n",
      "        [-0.0473, -0.0015,  0.0176,  ..., -0.0120,  0.0192,  0.0045],\n",
      "        ...,\n",
      "        [-0.0120,  0.0070,  0.0001,  ..., -0.0031,  0.0148, -0.0038],\n",
      "        [ 0.0114, -0.0031, -0.0025,  ..., -0.0088, -0.0105, -0.0008],\n",
      "        [ 0.0152,  0.0041, -0.0089,  ...,  0.0050,  0.0077,  0.0081]])), ('module.backbone.blocks.6.attn.qkv.bias', tensor([-0.0968, -0.4263,  0.1936,  ..., -0.0193,  0.0090, -0.0200])), ('module.backbone.blocks.6.attn.proj.weight', tensor([[ 0.0120, -0.0042, -0.0041,  ...,  0.0309, -0.0017, -0.0008],\n",
      "        [-0.0069,  0.0017, -0.0101,  ...,  0.0040,  0.0118,  0.0065],\n",
      "        [ 0.0025, -0.0098,  0.0081,  ...,  0.0091, -0.0172,  0.0028],\n",
      "        ...,\n",
      "        [ 0.0179,  0.0081,  0.0017,  ...,  0.0037,  0.0033, -0.0107],\n",
      "        [ 0.0150, -0.0031,  0.0026,  ..., -0.0002, -0.0191,  0.0042],\n",
      "        [-0.0068,  0.0066, -0.0085,  ...,  0.0045,  0.0052, -0.0091]])), ('module.backbone.blocks.6.attn.proj.bias', tensor([ 0.0132, -0.0217, -0.0088,  ..., -0.0082,  0.0064, -0.0104])), ('module.backbone.blocks.6.norm2.weight', tensor([0.0678, 0.5369, 0.0802,  ..., 1.1871, 1.5102, 1.4843])), ('module.backbone.blocks.6.norm2.bias', tensor([-0.0022, -0.0270, -0.1262,  ...,  0.0342, -0.0357,  0.0114])), ('module.backbone.blocks.6.mlp.fc1.weight', tensor([[-0.0103, -0.0074,  0.0096,  ...,  0.0062,  0.0099, -0.0270],\n",
      "        [ 0.0041,  0.0074,  0.0229,  ...,  0.0009, -0.0117, -0.0131],\n",
      "        [-0.0024,  0.0058, -0.0042,  ..., -0.0263,  0.0255,  0.0011],\n",
      "        ...,\n",
      "        [-0.0057,  0.0080,  0.0081,  ..., -0.0081,  0.0044,  0.0069],\n",
      "        [-0.0238, -0.0089,  0.0034,  ..., -0.0142,  0.0025, -0.0086],\n",
      "        [ 0.0180,  0.0011,  0.0083,  ..., -0.0129, -0.0062,  0.0209]])), ('module.backbone.blocks.6.mlp.fc1.bias', tensor([-0.1102, -0.1908, -0.1672,  ..., -0.0897, -0.1923, -0.0680])), ('module.backbone.blocks.6.mlp.fc2.weight', tensor([[ 0.0166, -0.0051, -0.0195,  ...,  0.0034, -0.0024,  0.0027],\n",
      "        [ 0.0237, -0.0095, -0.0191,  ...,  0.0080,  0.0012,  0.0130],\n",
      "        [ 0.0094, -0.0031, -0.0080,  ...,  0.0015, -0.0047, -0.0026],\n",
      "        ...,\n",
      "        [ 0.0034, -0.0002, -0.0041,  ..., -0.0131,  0.0071,  0.0132],\n",
      "        [-0.0061, -0.0191, -0.0123,  ..., -0.0191,  0.0017, -0.0168],\n",
      "        [ 0.0293, -0.0081,  0.0035,  ...,  0.0143,  0.0022, -0.0005]])), ('module.backbone.blocks.6.mlp.fc2.bias', tensor([ 0.0088, -0.0127,  0.0086,  ..., -0.0150,  0.0210,  0.0046])), ('module.backbone.blocks.7.norm1.weight', tensor([0.8421, 0.7654, 1.0426,  ..., 0.9234, 1.1944, 1.0628])), ('module.backbone.blocks.7.norm1.bias', tensor([-0.0321,  0.0435,  0.0797,  ...,  0.0263,  0.0353,  0.0268])), ('module.backbone.blocks.7.attn.qkv.weight', tensor([[ 1.8016e-03,  2.4113e-02,  2.7691e-03,  ..., -2.1507e-03,\n",
      "          3.3672e-02, -5.5428e-03],\n",
      "        [-1.4568e-02,  2.1432e-02, -1.4596e-02,  ..., -4.8697e-02,\n",
      "         -5.6907e-03, -8.9120e-03],\n",
      "        [ 2.4295e-02,  2.6125e-02, -5.7954e-03,  ..., -6.6273e-03,\n",
      "          1.5428e-02, -1.8513e-02],\n",
      "        ...,\n",
      "        [ 9.9475e-03,  4.3836e-03,  2.6596e-05,  ..., -6.1150e-03,\n",
      "          6.1469e-03, -2.4534e-03],\n",
      "        [ 6.6238e-03, -9.5548e-03, -5.0044e-03,  ..., -1.8924e-02,\n",
      "         -3.7133e-02, -2.6821e-03],\n",
      "        [ 1.0015e-02,  3.0955e-03,  1.7671e-02,  ...,  2.6777e-03,\n",
      "          1.2046e-02,  6.6109e-03]])), ('module.backbone.blocks.7.attn.qkv.bias', tensor([-0.0038, -0.1068, -0.2020,  ..., -0.0032, -0.0084, -0.0047])), ('module.backbone.blocks.7.attn.proj.weight', tensor([[-0.0040, -0.0018, -0.0038,  ...,  0.0004, -0.0170,  0.0029],\n",
      "        [ 0.0216, -0.0147,  0.0067,  ..., -0.0015,  0.0237, -0.0133],\n",
      "        [ 0.0109, -0.0158, -0.0011,  ..., -0.0370,  0.0101, -0.0282],\n",
      "        ...,\n",
      "        [-0.0061,  0.0068, -0.0017,  ..., -0.0042, -0.0135,  0.0029],\n",
      "        [ 0.0007,  0.0084,  0.0265,  ...,  0.0174,  0.0182, -0.0118],\n",
      "        [-0.0298, -0.0043,  0.0147,  ..., -0.0022,  0.0088,  0.0007]])), ('module.backbone.blocks.7.attn.proj.bias', tensor([ 0.0148, -0.0171,  0.0030,  ..., -0.0442,  0.0222, -0.0100])), ('module.backbone.blocks.7.norm2.weight', tensor([0.2295, 0.6557, 0.0847,  ..., 0.9972, 1.3552, 1.2186])), ('module.backbone.blocks.7.norm2.bias', tensor([ 0.0022, -0.0494, -0.1535,  ...,  0.0291, -0.0377,  0.0010])), ('module.backbone.blocks.7.mlp.fc1.weight', tensor([[ 0.0025,  0.0187,  0.0046,  ...,  0.0065,  0.0067,  0.0012],\n",
      "        [-0.0067, -0.0098,  0.0067,  ...,  0.0235,  0.0012, -0.0092],\n",
      "        [ 0.0024,  0.0175,  0.0113,  ...,  0.0099, -0.0039, -0.0075],\n",
      "        ...,\n",
      "        [-0.0009, -0.0125,  0.0056,  ..., -0.0038, -0.0170,  0.0340],\n",
      "        [ 0.0053,  0.0138,  0.0210,  ...,  0.0015,  0.0124,  0.0089],\n",
      "        [ 0.0092, -0.0147,  0.0093,  ..., -0.0226,  0.0123, -0.0002]])), ('module.backbone.blocks.7.mlp.fc1.bias', tensor([-0.2616, -0.1309, -0.2342,  ..., -0.0601, -0.1839, -0.1329])), ('module.backbone.blocks.7.mlp.fc2.weight', tensor([[ 0.0032, -0.0044, -0.0073,  ...,  0.0118, -0.0120, -0.0198],\n",
      "        [ 0.0170, -0.0024,  0.0087,  ...,  0.0036, -0.0029, -0.0154],\n",
      "        [ 0.0050, -0.0066, -0.0072,  ..., -0.0031, -0.0017, -0.0011],\n",
      "        ...,\n",
      "        [-0.0047, -0.0007,  0.0144,  ...,  0.0087,  0.0106,  0.0003],\n",
      "        [ 0.0322, -0.0014, -0.0048,  ...,  0.0073, -0.0125, -0.0220],\n",
      "        [-0.0020,  0.0099, -0.0090,  ..., -0.0197,  0.0019,  0.0016]])), ('module.backbone.blocks.7.mlp.fc2.bias', tensor([ 0.0059, -0.0101,  0.0096,  ..., -0.0374,  0.0555, -0.0069])), ('module.backbone.blocks.8.norm1.weight', tensor([0.9656, 0.8027, 1.0407,  ..., 0.8485, 1.1502, 0.9717])), ('module.backbone.blocks.8.norm1.bias', tensor([-0.0618,  0.0429,  0.1141,  ...,  0.0187,  0.0320,  0.0260])), ('module.backbone.blocks.8.attn.qkv.weight', tensor([[ 0.0006,  0.0075,  0.0099,  ...,  0.0068,  0.0192,  0.0142],\n",
      "        [ 0.0236, -0.0270, -0.0027,  ..., -0.0107,  0.0363, -0.0227],\n",
      "        [-0.0015,  0.0185, -0.0054,  ...,  0.0190,  0.0168, -0.0053],\n",
      "        ...,\n",
      "        [-0.0004, -0.0029, -0.0076,  ...,  0.0069, -0.0067,  0.0263],\n",
      "        [ 0.0019, -0.0060, -0.0006,  ...,  0.0186,  0.0068, -0.0075],\n",
      "        [-0.0122, -0.0085, -0.0169,  ..., -0.0126, -0.0185, -0.0172]])), ('module.backbone.blocks.8.attn.qkv.bias', tensor([ 0.0884,  0.0848, -0.6187,  ..., -0.0045,  0.0021,  0.0040])), ('module.backbone.blocks.8.attn.proj.weight', tensor([[-0.0094, -0.0034,  0.0027,  ...,  0.0130, -0.0117,  0.0042],\n",
      "        [-0.0156,  0.0076, -0.0073,  ...,  0.0065,  0.0132,  0.0047],\n",
      "        [-0.0026,  0.0153, -0.0006,  ...,  0.0101, -0.0011,  0.0223],\n",
      "        ...,\n",
      "        [ 0.0182,  0.0004,  0.0182,  ..., -0.0090, -0.0307,  0.0201],\n",
      "        [ 0.0166, -0.0042,  0.0027,  ...,  0.0003,  0.0023,  0.0113],\n",
      "        [-0.0157,  0.0117, -0.0083,  ..., -0.0141,  0.0176,  0.0159]])), ('module.backbone.blocks.8.attn.proj.bias', tensor([ 0.0141, -0.0169,  0.0004,  ..., -0.0374,  0.0600, -0.0283])), ('module.backbone.blocks.8.norm2.weight', tensor([0.3612, 0.7661, 0.1568,  ..., 0.9451, 1.1691, 1.1171])), ('module.backbone.blocks.8.norm2.bias', tensor([ 0.0275, -0.0256, -0.1890,  ...,  0.0089, -0.0259, -0.0390])), ('module.backbone.blocks.8.mlp.fc1.weight', tensor([[ 0.0079, -0.0247, -0.0116,  ..., -0.0171,  0.0029, -0.0243],\n",
      "        [-0.0220, -0.0102, -0.0050,  ...,  0.0263, -0.0021,  0.0046],\n",
      "        [-0.0195, -0.0171,  0.0066,  ...,  0.0178,  0.0132, -0.0493],\n",
      "        ...,\n",
      "        [ 0.0192,  0.0137,  0.0065,  ..., -0.0210,  0.0034, -0.0012],\n",
      "        [ 0.0002,  0.0044,  0.0040,  ...,  0.0193,  0.0142, -0.0223],\n",
      "        [-0.0358,  0.0183, -0.0015,  ...,  0.0194,  0.0047,  0.0090]])), ('module.backbone.blocks.8.mlp.fc1.bias', tensor([-0.1621, -0.1756, -0.1638,  ..., -0.0280, -0.0799, -0.0982])), ('module.backbone.blocks.8.mlp.fc2.weight', tensor([[ 0.0093,  0.0044,  0.0008,  ...,  0.0054, -0.0054,  0.0081],\n",
      "        [ 0.0049, -0.0131, -0.0003,  ...,  0.0117, -0.0070, -0.0200],\n",
      "        [ 0.0037, -0.0110, -0.0087,  ..., -0.0086,  0.0091,  0.0112],\n",
      "        ...,\n",
      "        [ 0.0052,  0.0149,  0.0045,  ...,  0.0241, -0.0106,  0.0169],\n",
      "        [ 0.0049, -0.0057,  0.0021,  ...,  0.0053, -0.0086, -0.0116],\n",
      "        [-0.0016,  0.0019,  0.0030,  ...,  0.0105, -0.0070, -0.0114]])), ('module.backbone.blocks.8.mlp.fc2.bias', tensor([-0.0002, -0.0052, -0.0001,  ..., -0.0359,  0.0673, -0.0043])), ('module.backbone.blocks.9.norm1.weight', tensor([0.9171, 0.7872, 0.9255,  ..., 0.9125, 1.0698, 1.0332])), ('module.backbone.blocks.9.norm1.bias', tensor([-0.0304,  0.0436,  0.1101,  ..., -0.0337,  0.0534,  0.0528])), ('module.backbone.blocks.9.attn.qkv.weight', tensor([[-0.0179, -0.0057, -0.0180,  ..., -0.0004,  0.0071,  0.0142],\n",
      "        [-0.0094,  0.0065,  0.0080,  ..., -0.0025, -0.0171,  0.0054],\n",
      "        [-0.0058,  0.0178, -0.0056,  ...,  0.0130,  0.0240, -0.0246],\n",
      "        ...,\n",
      "        [ 0.0006,  0.0060,  0.0030,  ..., -0.0015, -0.0014,  0.0025],\n",
      "        [ 0.0028,  0.0164,  0.0046,  ..., -0.0184,  0.0228, -0.0168],\n",
      "        [ 0.0005, -0.0085,  0.0004,  ..., -0.0111, -0.0128, -0.0106]])), ('module.backbone.blocks.9.attn.qkv.bias', tensor([-0.1170, -0.0152, -0.0889,  ...,  0.0030,  0.0129, -0.0213])), ('module.backbone.blocks.9.attn.proj.weight', tensor([[ 0.0078,  0.0047, -0.0063,  ..., -0.0056, -0.0134, -0.0025],\n",
      "        [ 0.0051,  0.0038,  0.0091,  ..., -0.0097, -0.0145,  0.0096],\n",
      "        [-0.0090,  0.0039, -0.0102,  ..., -0.0018,  0.0013,  0.0134],\n",
      "        ...,\n",
      "        [-0.0206, -0.0279,  0.0018,  ...,  0.0134, -0.0060,  0.0087],\n",
      "        [-0.0109,  0.0129, -0.0017,  ...,  0.0044, -0.0216, -0.0067],\n",
      "        [-0.0050,  0.0152, -0.0158,  ...,  0.0027,  0.0181, -0.0123]])), ('module.backbone.blocks.9.attn.proj.bias', tensor([ 0.0150, -0.0071, -0.0159,  ..., -0.0193,  0.0506, -0.0272])), ('module.backbone.blocks.9.norm2.weight', tensor([0.6837, 0.9233, 0.7491,  ..., 1.1021, 1.2182, 1.0696])), ('module.backbone.blocks.9.norm2.bias', tensor([ 0.0644,  0.0126, -0.1461,  ..., -0.0203,  0.0045,  0.0229])), ('module.backbone.blocks.9.mlp.fc1.weight', tensor([[ 1.1667e-02,  1.2885e-03, -3.5566e-03,  ..., -1.8800e-02,\n",
      "         -5.9043e-03, -9.7528e-03],\n",
      "        [-1.3831e-02,  2.1122e-03,  2.3890e-03,  ..., -1.1187e-02,\n",
      "          1.1997e-02, -1.0249e-02],\n",
      "        [ 1.3502e-02,  1.4598e-02,  2.7352e-03,  ...,  9.4158e-04,\n",
      "         -2.3889e-03,  5.2877e-05],\n",
      "        ...,\n",
      "        [ 4.0489e-04, -2.0536e-02,  5.4138e-03,  ..., -1.0867e-02,\n",
      "          2.1561e-02, -1.1225e-03],\n",
      "        [-1.6719e-02,  1.0769e-03, -1.2487e-02,  ...,  1.7925e-02,\n",
      "          5.7090e-03, -2.5455e-02],\n",
      "        [-1.9351e-02, -1.8085e-02,  1.3288e-02,  ...,  1.6911e-03,\n",
      "          1.4117e-02, -2.3137e-03]])), ('module.backbone.blocks.9.mlp.fc1.bias', tensor([-0.0484, -0.0510, -0.0828,  ..., -0.0517, -0.1041, -0.1034])), ('module.backbone.blocks.9.mlp.fc2.weight', tensor([[-0.0168, -0.0024, -0.0110,  ...,  0.0101,  0.0054,  0.0015],\n",
      "        [-0.0112,  0.0059,  0.0024,  ...,  0.0088,  0.0093, -0.0117],\n",
      "        [ 0.0110,  0.0150, -0.0043,  ...,  0.0023,  0.0087,  0.0126],\n",
      "        ...,\n",
      "        [ 0.0147, -0.0081, -0.0203,  ...,  0.0111, -0.0206,  0.0133],\n",
      "        [ 0.0146, -0.0109,  0.0014,  ..., -0.0297, -0.0216,  0.0028],\n",
      "        [ 0.0044, -0.0141, -0.0080,  ...,  0.0029,  0.0033,  0.0056]])), ('module.backbone.blocks.9.mlp.fc2.bias', tensor([-0.0020, -0.0084,  0.0144,  ..., -0.0146,  0.0426, -0.0198])), ('module.backbone.blocks.10.norm1.weight', tensor([1.0047, 0.9604, 1.0489,  ..., 0.9354, 1.1717, 1.0540])), ('module.backbone.blocks.10.norm1.bias', tensor([-0.0785,  0.0014,  0.0801,  ...,  0.0109,  0.0559,  0.0166])), ('module.backbone.blocks.10.attn.qkv.weight', tensor([[ 0.0007, -0.0380, -0.0040,  ...,  0.0095, -0.0005, -0.0271],\n",
      "        [ 0.0380, -0.0040,  0.0001,  ...,  0.0471, -0.0131,  0.0254],\n",
      "        [-0.0087, -0.0233,  0.0162,  ...,  0.0267,  0.0022,  0.0138],\n",
      "        ...,\n",
      "        [ 0.0166,  0.0090, -0.0025,  ..., -0.0099,  0.0110,  0.0077],\n",
      "        [-0.0023, -0.0013,  0.0039,  ..., -0.0002,  0.0068, -0.0090],\n",
      "        [-0.0069, -0.0081, -0.0008,  ..., -0.0026, -0.0343,  0.0198]])), ('module.backbone.blocks.10.attn.qkv.bias', tensor([ 0.0606,  0.0480,  0.0180,  ..., -0.0175,  0.0122,  0.0029])), ('module.backbone.blocks.10.attn.proj.weight', tensor([[ 6.8867e-03, -1.8347e-03, -2.0338e-04,  ..., -5.0077e-03,\n",
      "          1.7198e-02, -2.4491e-04],\n",
      "        [-6.9242e-03,  4.3826e-03, -1.8603e-02,  ..., -2.4515e-02,\n",
      "         -5.0282e-03, -2.1227e-02],\n",
      "        [-9.8681e-03,  2.1397e-02, -7.4755e-06,  ...,  4.0834e-03,\n",
      "         -1.2954e-02,  2.6357e-02],\n",
      "        ...,\n",
      "        [ 1.1706e-02, -1.3926e-02, -3.7544e-02,  ...,  4.4794e-03,\n",
      "         -3.7540e-03,  7.6398e-03],\n",
      "        [-1.5817e-02, -2.4791e-02, -1.6663e-02,  ..., -1.5987e-02,\n",
      "         -2.7143e-03, -4.2722e-03],\n",
      "        [ 3.8333e-03, -8.6208e-03, -6.1166e-03,  ..., -4.6414e-02,\n",
      "          4.4176e-03,  1.8141e-03]])), ('module.backbone.blocks.10.attn.proj.bias', tensor([ 0.0230,  0.0017, -0.0047,  ..., -0.0400,  0.0524, -0.0294])), ('module.backbone.blocks.10.norm2.weight', tensor([0.7800, 0.9221, 0.8366,  ..., 1.0625, 1.2019, 1.0537])), ('module.backbone.blocks.10.norm2.bias', tensor([ 0.1580, -0.0318, -0.0851,  ...,  0.0263, -0.0264, -0.0376])), ('module.backbone.blocks.10.mlp.fc1.weight', tensor([[ 0.0024, -0.0035, -0.0031,  ...,  0.0078, -0.0037,  0.0042],\n",
      "        [ 0.0009,  0.0171, -0.0191,  ..., -0.0023, -0.0021,  0.0018],\n",
      "        [-0.0016,  0.0178,  0.0026,  ..., -0.0424, -0.0136, -0.0309],\n",
      "        ...,\n",
      "        [-0.0021,  0.0065, -0.0009,  ..., -0.0094, -0.0054,  0.0042],\n",
      "        [-0.0109,  0.0096,  0.0005,  ...,  0.0032, -0.0226,  0.0106],\n",
      "        [-0.0044, -0.0031, -0.0144,  ...,  0.0154,  0.0222,  0.0139]])), ('module.backbone.blocks.10.mlp.fc1.bias', tensor([-0.1864, -0.1215, -0.1461,  ..., -0.0455, -0.2065, -0.1801])), ('module.backbone.blocks.10.mlp.fc2.weight', tensor([[ 0.0098,  0.0085,  0.0019,  ...,  0.0016,  0.0042, -0.0039],\n",
      "        [ 0.0083, -0.0048, -0.0077,  ..., -0.0110, -0.0031, -0.0047],\n",
      "        [-0.0202,  0.0050, -0.0064,  ...,  0.0036,  0.0102, -0.0110],\n",
      "        ...,\n",
      "        [ 0.0026,  0.0229, -0.0070,  ...,  0.0138, -0.0017,  0.0075],\n",
      "        [ 0.0085,  0.0001, -0.0278,  ..., -0.0097,  0.0189,  0.0066],\n",
      "        [-0.0005,  0.0141, -0.0044,  ..., -0.0094, -0.0038,  0.0006]])), ('module.backbone.blocks.10.mlp.fc2.bias', tensor([-0.0058,  0.0016,  0.0036,  ..., -0.0293,  0.0791, -0.0200])), ('module.backbone.blocks.11.norm1.weight', tensor([1.0546, 0.9330, 1.0730,  ..., 1.0057, 1.1986, 1.0562])), ('module.backbone.blocks.11.norm1.bias', tensor([-0.0378, -0.0148,  0.0408,  ..., -0.0310,  0.0046,  0.0374])), ('module.backbone.blocks.11.attn.qkv.weight', tensor([[ 0.0295, -0.0364, -0.0209,  ...,  0.0093, -0.0201, -0.0160],\n",
      "        [ 0.0032, -0.0081,  0.0029,  ..., -0.0085,  0.0185,  0.0173],\n",
      "        [-0.0190,  0.0146, -0.0100,  ..., -0.0157, -0.0086, -0.0228],\n",
      "        ...,\n",
      "        [-0.0053, -0.0030,  0.0062,  ...,  0.0150, -0.0160,  0.0087],\n",
      "        [-0.0115,  0.0125, -0.0158,  ...,  0.0019,  0.0215, -0.0240],\n",
      "        [-0.0004,  0.0418, -0.0015,  ...,  0.0143, -0.0377,  0.0066]])), ('module.backbone.blocks.11.attn.qkv.bias', tensor([-0.0888, -0.0724, -0.0941,  ...,  0.0001, -0.0238,  0.0145])), ('module.backbone.blocks.11.attn.proj.weight', tensor([[-0.0019,  0.0124,  0.0111,  ...,  0.0082,  0.0057, -0.0016],\n",
      "        [-0.0005,  0.0037, -0.0193,  ...,  0.0113, -0.0208, -0.0163],\n",
      "        [-0.0027,  0.0061,  0.0084,  ...,  0.0022,  0.0136, -0.0020],\n",
      "        ...,\n",
      "        [ 0.0283,  0.0157,  0.0098,  ..., -0.0136,  0.0017, -0.0133],\n",
      "        [-0.0052, -0.0111,  0.0113,  ...,  0.0169, -0.0269, -0.0033],\n",
      "        [ 0.0113,  0.0133,  0.0176,  ..., -0.0102, -0.0257,  0.0156]])), ('module.backbone.blocks.11.attn.proj.bias', tensor([ 0.0213,  0.0089,  0.0015,  ..., -0.0102,  0.0887, -0.0191])), ('module.backbone.blocks.11.norm2.weight', tensor([0.7623, 0.8458, 0.6329,  ..., 1.0762, 1.1876, 1.0615])), ('module.backbone.blocks.11.norm2.bias', tensor([ 0.1497, -0.0950, -0.1805,  ..., -0.0087, -0.0884, -0.0069])), ('module.backbone.blocks.11.mlp.fc1.weight', tensor([[-0.0171, -0.0061, -0.0039,  ...,  0.0128,  0.0134, -0.0112],\n",
      "        [-0.0143,  0.0052,  0.0095,  ...,  0.0211,  0.0007, -0.0040],\n",
      "        [-0.0269, -0.0146,  0.0172,  ...,  0.0008,  0.0167,  0.0413],\n",
      "        ...,\n",
      "        [-0.0157, -0.0055,  0.0052,  ..., -0.0178, -0.0163,  0.0342],\n",
      "        [ 0.0079,  0.0605, -0.0010,  ..., -0.0143,  0.0143, -0.0036],\n",
      "        [-0.0107, -0.0091, -0.0098,  ...,  0.0227,  0.0117,  0.0077]])), ('module.backbone.blocks.11.mlp.fc1.bias', tensor([-0.0345, -0.1059, -0.2261,  ..., -0.1687, -0.2685, -0.0479])), ('module.backbone.blocks.11.mlp.fc2.weight', tensor([[ 1.5669e-02, -8.9435e-03,  1.1030e-02,  ..., -2.5976e-03,\n",
      "          2.1207e-02,  2.7551e-03],\n",
      "        [-6.4386e-04, -1.0983e-02, -1.5692e-02,  ...,  1.4777e-02,\n",
      "         -3.7844e-03,  1.2217e-03],\n",
      "        [-1.5701e-05,  3.3798e-03,  2.4076e-03,  ...,  1.5749e-02,\n",
      "          1.6410e-02,  3.9563e-03],\n",
      "        ...,\n",
      "        [-9.0431e-03, -6.5803e-03,  1.1375e-02,  ...,  6.4219e-03,\n",
      "         -1.0955e-02, -8.3048e-03],\n",
      "        [-1.3645e-02, -4.1997e-03,  3.6004e-02,  ...,  1.3771e-02,\n",
      "          6.5738e-03,  5.2565e-03],\n",
      "        [ 4.5930e-03, -7.9024e-03,  3.5604e-03,  ...,  7.9574e-03,\n",
      "          6.0408e-03, -6.1092e-03]])), ('module.backbone.blocks.11.mlp.fc2.bias', tensor([ 0.0011,  0.0087, -0.0020,  ..., -0.0042,  0.1047, -0.0167])), ('module.backbone.blocks.12.norm1.weight', tensor([1.1120, 0.9363, 1.0578,  ..., 1.0353, 1.1449, 1.1027])), ('module.backbone.blocks.12.norm1.bias', tensor([-0.0434,  0.0414,  0.1372,  ..., -0.0321,  0.0764, -0.0171])), ('module.backbone.blocks.12.attn.qkv.weight', tensor([[-0.0104, -0.0140, -0.0031,  ..., -0.0364,  0.0032,  0.0140],\n",
      "        [ 0.0006,  0.0250, -0.0263,  ...,  0.0019, -0.0425, -0.0159],\n",
      "        [-0.0223, -0.0013,  0.0269,  ...,  0.0158, -0.0001,  0.0382],\n",
      "        ...,\n",
      "        [-0.0084, -0.0202, -0.0071,  ..., -0.0248,  0.0190, -0.0157],\n",
      "        [ 0.0014,  0.0092,  0.0072,  ...,  0.0020, -0.0211,  0.0029],\n",
      "        [ 0.0145,  0.0070,  0.0007,  ..., -0.0063, -0.0033,  0.0040]])), ('module.backbone.blocks.12.attn.qkv.bias', tensor([-0.2654,  0.1043, -0.2318,  ...,  0.0062, -0.0063,  0.0279])), ('module.backbone.blocks.12.attn.proj.weight', tensor([[-0.0014, -0.0068, -0.0014,  ...,  0.0095, -0.0013,  0.0168],\n",
      "        [ 0.0012,  0.0243,  0.0153,  ..., -0.0071,  0.0102,  0.0021],\n",
      "        [-0.0227,  0.0231,  0.0068,  ...,  0.0103, -0.0141, -0.0038],\n",
      "        ...,\n",
      "        [-0.0146,  0.0152,  0.0371,  ...,  0.0278,  0.0446, -0.0312],\n",
      "        [-0.0195, -0.0238, -0.0131,  ...,  0.0042,  0.0128, -0.0007],\n",
      "        [-0.0231,  0.0061,  0.0158,  ...,  0.0021,  0.0121, -0.0138]])), ('module.backbone.blocks.12.attn.proj.bias', tensor([ 0.0273,  0.0043, -0.0269,  ...,  0.0333,  0.0881, -0.0190])), ('module.backbone.blocks.12.norm2.weight', tensor([0.9173, 0.9347, 0.8763,  ..., 1.1500, 1.2691, 1.1452])), ('module.backbone.blocks.12.norm2.bias', tensor([ 0.1617, -0.0386, -0.0526,  ..., -0.0753,  0.0297, -0.0862])), ('module.backbone.blocks.12.mlp.fc1.weight', tensor([[ 0.0048,  0.0267, -0.0166,  ...,  0.0060, -0.0116, -0.0041],\n",
      "        [ 0.0008,  0.0006, -0.0156,  ...,  0.0475,  0.0443,  0.0114],\n",
      "        [ 0.0113, -0.0140, -0.0006,  ...,  0.0109,  0.0026, -0.0052],\n",
      "        ...,\n",
      "        [-0.0095, -0.0096,  0.0103,  ...,  0.0212,  0.0207, -0.0088],\n",
      "        [ 0.0152,  0.0092, -0.0182,  ...,  0.0305,  0.0253, -0.0256],\n",
      "        [ 0.0082, -0.0021,  0.0110,  ...,  0.0080,  0.0002,  0.0083]])), ('module.backbone.blocks.12.mlp.fc1.bias', tensor([-0.2481, -0.1522, -0.1297,  ..., -0.0496, -0.1703,  0.0467])), ('module.backbone.blocks.12.mlp.fc2.weight', tensor([[-0.0089, -0.0031, -0.0088,  ...,  0.0116,  0.0046, -0.0093],\n",
      "        [ 0.0122,  0.0174, -0.0037,  ...,  0.0030, -0.0039,  0.0010],\n",
      "        [ 0.0082,  0.0127,  0.0215,  ..., -0.0109,  0.0004, -0.0081],\n",
      "        ...,\n",
      "        [-0.0124,  0.0126,  0.0068,  ..., -0.0279,  0.0067, -0.0062],\n",
      "        [-0.0007,  0.0069,  0.0007,  ..., -0.0139,  0.0031,  0.0043],\n",
      "        [-0.0109,  0.0086,  0.0052,  ...,  0.0050,  0.0003, -0.0081]])), ('module.backbone.blocks.12.mlp.fc2.bias', tensor([ 0.0032,  0.0042, -0.0243,  ...,  0.0206,  0.0705, -0.0182])), ('module.backbone.blocks.13.norm1.weight', tensor([1.1920, 0.9739, 1.1245,  ..., 1.0944, 1.1870, 1.1469])), ('module.backbone.blocks.13.norm1.bias', tensor([-0.0024,  0.0579,  0.0461,  ..., -0.0307,  0.0109,  0.0120])), ('module.backbone.blocks.13.attn.qkv.weight', tensor([[ 0.0287, -0.0062, -0.0037,  ...,  0.0239,  0.0032,  0.0304],\n",
      "        [ 0.0365,  0.0115,  0.0107,  ...,  0.0288,  0.0198, -0.0134],\n",
      "        [ 0.0095,  0.0003,  0.0055,  ...,  0.0001,  0.0014,  0.0130],\n",
      "        ...,\n",
      "        [ 0.0077,  0.0145,  0.0064,  ..., -0.0044,  0.0058,  0.0132],\n",
      "        [ 0.0212,  0.0058, -0.0014,  ..., -0.0159,  0.0048,  0.0183],\n",
      "        [ 0.0102,  0.0005, -0.0048,  ..., -0.0148, -0.0144, -0.0037]])), ('module.backbone.blocks.13.attn.qkv.bias', tensor([ 0.1162,  0.0116,  0.1295,  ..., -0.0292,  0.0187, -0.0503])), ('module.backbone.blocks.13.attn.proj.weight', tensor([[ 0.0206,  0.0240,  0.0173,  ..., -0.0087,  0.0015, -0.0043],\n",
      "        [ 0.0348,  0.0044, -0.0039,  ..., -0.0031,  0.0051, -0.0156],\n",
      "        [ 0.0427,  0.0030,  0.0309,  ..., -0.0018, -0.0117, -0.0279],\n",
      "        ...,\n",
      "        [-0.0093, -0.0120, -0.0173,  ...,  0.0127,  0.0104, -0.0256],\n",
      "        [ 0.0158, -0.0193, -0.0023,  ...,  0.0261, -0.0062, -0.0085],\n",
      "        [-0.0172, -0.0263, -0.0283,  ...,  0.0118,  0.0093,  0.0095]])), ('module.backbone.blocks.13.attn.proj.bias', tensor([ 0.0217, -0.0133, -0.0229,  ...,  0.0599,  0.0780, -0.0107])), ('module.backbone.blocks.13.norm2.weight', tensor([0.7891, 1.0304, 0.9850,  ..., 1.2364, 1.3937, 1.2477])), ('module.backbone.blocks.13.norm2.bias', tensor([ 0.1845, -0.0863, -0.1490,  ..., -0.0881, -0.0139, -0.0976])), ('module.backbone.blocks.13.mlp.fc1.weight', tensor([[-0.0231,  0.0101,  0.0169,  ..., -0.0136, -0.0041,  0.0022],\n",
      "        [ 0.0012,  0.0151, -0.0049,  ..., -0.0184, -0.0030,  0.0099],\n",
      "        [-0.0066,  0.0014,  0.0201,  ..., -0.0091,  0.0148, -0.0232],\n",
      "        ...,\n",
      "        [-0.0231, -0.0026,  0.0001,  ..., -0.0196, -0.0074,  0.0034],\n",
      "        [-0.0136,  0.0017, -0.0019,  ..., -0.0082,  0.0037,  0.0205],\n",
      "        [-0.0056, -0.0089,  0.0044,  ..., -0.0192, -0.0335,  0.0117]])), ('module.backbone.blocks.13.mlp.fc1.bias', tensor([-0.1435, -0.1265, -0.0065,  ..., -0.0337, -0.0813, -0.0251])), ('module.backbone.blocks.13.mlp.fc2.weight', tensor([[ 0.0129,  0.0045,  0.0024,  ...,  0.0186,  0.0054,  0.0016],\n",
      "        [-0.0167, -0.0018,  0.0013,  ..., -0.0097, -0.0102,  0.0141],\n",
      "        [-0.0096,  0.0144, -0.0256,  ...,  0.0060, -0.0138, -0.0063],\n",
      "        ...,\n",
      "        [-0.0042,  0.0007,  0.0168,  ...,  0.0220, -0.0033,  0.0175],\n",
      "        [-0.0236, -0.0034, -0.0146,  ..., -0.0019,  0.0161,  0.0322],\n",
      "        [-0.0033,  0.0024,  0.0195,  ...,  0.0050,  0.0146, -0.0048]])), ('module.backbone.blocks.13.mlp.fc2.bias', tensor([ 0.0093,  0.0014, -0.0039,  ...,  0.0581,  0.0519, -0.0123])), ('module.backbone.blocks.14.norm1.weight', tensor([1.0882, 0.9502, 1.0390,  ..., 1.1464, 1.3077, 1.2551])), ('module.backbone.blocks.14.norm1.bias', tensor([-0.0619,  0.0286,  0.0275,  ...,  0.0808,  0.0375,  0.0325])), ('module.backbone.blocks.14.attn.qkv.weight', tensor([[-2.1092e-02, -1.4143e-02, -6.8879e-03,  ...,  2.0748e-03,\n",
      "          1.7621e-02, -9.4712e-03],\n",
      "        [-5.3125e-03, -6.3927e-03,  1.9769e-02,  ...,  2.7914e-02,\n",
      "         -3.2580e-02,  2.6360e-02],\n",
      "        [ 3.1084e-02,  2.4542e-02, -2.5978e-02,  ..., -2.0251e-02,\n",
      "          3.4109e-03, -1.6155e-02],\n",
      "        ...,\n",
      "        [-6.6314e-05,  2.7743e-02, -1.8137e-02,  ..., -8.4442e-03,\n",
      "         -1.4984e-02, -1.9340e-02],\n",
      "        [ 1.5384e-02, -5.5256e-04, -1.2809e-02,  ..., -3.4244e-03,\n",
      "          8.7975e-04, -5.3077e-03],\n",
      "        [-1.4380e-02, -3.4425e-02,  1.3010e-02,  ...,  4.5831e-03,\n",
      "          2.2355e-02,  2.6751e-02]])), ('module.backbone.blocks.14.attn.qkv.bias', tensor([ 0.1351,  0.0188,  0.1056,  ..., -0.0710,  0.0142,  0.0310])), ('module.backbone.blocks.14.attn.proj.weight', tensor([[ 0.0023, -0.0340,  0.0025,  ...,  0.0078, -0.0050,  0.0415],\n",
      "        [ 0.0071, -0.0075,  0.0268,  ...,  0.0070,  0.0143,  0.0028],\n",
      "        [ 0.0162, -0.0058, -0.0087,  ...,  0.0037,  0.0028, -0.0179],\n",
      "        ...,\n",
      "        [-0.0218,  0.0206, -0.0043,  ...,  0.0175,  0.0078, -0.0111],\n",
      "        [-0.0165, -0.0008, -0.0287,  ...,  0.0324,  0.0117, -0.0016],\n",
      "        [-0.0124, -0.0274, -0.0177,  ..., -0.0081, -0.0020, -0.0107]])), ('module.backbone.blocks.14.attn.proj.bias', tensor([ 0.0463, -0.0001, -0.0110,  ...,  0.0420,  0.0533, -0.0219])), ('module.backbone.blocks.14.norm2.weight', tensor([0.8940, 1.0676, 1.0012,  ..., 1.3141, 1.3917, 1.2817])), ('module.backbone.blocks.14.norm2.bias', tensor([ 0.3172, -0.0768, -0.1609,  ...,  0.0736,  0.0498, -0.0627])), ('module.backbone.blocks.14.mlp.fc1.weight', tensor([[ 1.5714e-04,  6.8878e-03,  1.4470e-02,  ...,  1.2519e-04,\n",
      "         -1.2263e-02,  3.5513e-02],\n",
      "        [-4.2762e-02,  1.9295e-03,  2.3811e-02,  ..., -3.9235e-02,\n",
      "         -5.9839e-03, -4.9723e-05],\n",
      "        [ 2.3887e-02, -1.3846e-02, -1.3809e-02,  ..., -2.5361e-02,\n",
      "         -1.5108e-02, -1.1024e-02],\n",
      "        ...,\n",
      "        [ 6.7624e-03,  8.1451e-03, -8.1024e-03,  ..., -4.2840e-03,\n",
      "         -4.1153e-02,  8.3431e-03],\n",
      "        [-4.0350e-03, -1.5415e-02,  3.8157e-02,  ..., -1.3487e-02,\n",
      "         -2.5555e-02, -3.0690e-03],\n",
      "        [-1.3575e-02, -1.2267e-02,  7.6788e-03,  ...,  1.6112e-02,\n",
      "         -2.5344e-02,  9.5259e-03]])), ('module.backbone.blocks.14.mlp.fc1.bias', tensor([-0.2674, -0.0637, -0.0961,  ..., -0.2470, -0.1668, -0.1447])), ('module.backbone.blocks.14.mlp.fc2.weight', tensor([[-0.0098,  0.0028, -0.0040,  ...,  0.0002,  0.0119,  0.0027],\n",
      "        [ 0.0098,  0.0008,  0.0033,  ...,  0.0093,  0.0083,  0.0012],\n",
      "        [ 0.0086, -0.0107, -0.0207,  ...,  0.0023,  0.0055, -0.0129],\n",
      "        ...,\n",
      "        [ 0.0263,  0.0166, -0.0100,  ..., -0.0397,  0.0004,  0.0073],\n",
      "        [-0.0003,  0.0029, -0.0204,  ..., -0.0015,  0.0040, -0.0074],\n",
      "        [-0.0112, -0.0084, -0.0189,  ...,  0.0194,  0.0385,  0.0136]])), ('module.backbone.blocks.14.mlp.fc2.bias', tensor([ 0.0011, -0.0002,  0.0193,  ...,  0.0148,  0.0267, -0.0094])), ('module.backbone.blocks.15.norm1.weight', tensor([0.9993, 0.9441, 0.9990,  ..., 1.1520, 1.2488, 1.2364])), ('module.backbone.blocks.15.norm1.bias', tensor([-0.0770, -0.0322,  0.0628,  ...,  0.0234,  0.0128,  0.0378])), ('module.backbone.blocks.15.attn.qkv.weight', tensor([[ 0.0323, -0.0210,  0.0006,  ...,  0.0203,  0.0413,  0.0184],\n",
      "        [ 0.0003,  0.0237, -0.0035,  ...,  0.0131, -0.0104,  0.0188],\n",
      "        [ 0.0168, -0.0163, -0.0218,  ..., -0.0213,  0.0154, -0.0238],\n",
      "        ...,\n",
      "        [-0.0011,  0.0162, -0.0121,  ...,  0.0027, -0.0315,  0.0082],\n",
      "        [-0.0177, -0.0080,  0.0278,  ...,  0.0090, -0.0164, -0.0197],\n",
      "        [-0.0090, -0.0049,  0.0244,  ...,  0.0064,  0.0273,  0.0170]])), ('module.backbone.blocks.15.attn.qkv.bias', tensor([-0.0913,  0.2236, -0.2863,  ..., -0.0169, -0.0005,  0.0087])), ('module.backbone.blocks.15.attn.proj.weight', tensor([[ 0.0119,  0.0080,  0.0197,  ..., -0.0047, -0.0030, -0.0132],\n",
      "        [-0.0087,  0.0020,  0.0009,  ...,  0.0017, -0.0107,  0.0195],\n",
      "        [ 0.0069,  0.0084, -0.0155,  ...,  0.0055, -0.0083, -0.0204],\n",
      "        ...,\n",
      "        [ 0.0096,  0.0110, -0.0063,  ...,  0.0244,  0.0265,  0.0067],\n",
      "        [-0.0042,  0.0197,  0.0105,  ...,  0.0304, -0.0215,  0.0080],\n",
      "        [ 0.0073, -0.0089,  0.0170,  ...,  0.0222,  0.0067, -0.0035]])), ('module.backbone.blocks.15.attn.proj.bias', tensor([ 0.0143,  0.0054, -0.0148,  ...,  0.0036,  0.0285, -0.0119])), ('module.backbone.blocks.15.norm2.weight', tensor([0.8652, 1.0831, 1.0124,  ..., 1.4089, 1.4663, 1.3485])), ('module.backbone.blocks.15.norm2.bias', tensor([ 0.2126, -0.2042, -0.0832,  ..., -0.0053, -0.0126, -0.1075])), ('module.backbone.blocks.15.mlp.fc1.weight', tensor([[-0.0223,  0.0132, -0.0063,  ..., -0.0062,  0.0354,  0.0191],\n",
      "        [-0.0060,  0.0116, -0.0061,  ...,  0.0279, -0.0055,  0.0185],\n",
      "        [-0.0088, -0.0091, -0.0073,  ...,  0.0116, -0.0173,  0.0201],\n",
      "        ...,\n",
      "        [-0.0031, -0.0060,  0.0270,  ..., -0.0145, -0.0161, -0.0016],\n",
      "        [-0.0032, -0.0071,  0.0124,  ..., -0.0218,  0.0062,  0.0154],\n",
      "        [ 0.0129,  0.0194, -0.0222,  ..., -0.0039,  0.0371,  0.0032]])), ('module.backbone.blocks.15.mlp.fc1.bias', tensor([-0.1711, -0.0535, -0.1277,  ..., -0.0560, -0.0490, -0.1028])), ('module.backbone.blocks.15.mlp.fc2.weight', tensor([[-0.0132,  0.0182,  0.0003,  ...,  0.0003, -0.0088, -0.0020],\n",
      "        [-0.0056,  0.0026,  0.0182,  ..., -0.0077,  0.0034,  0.0065],\n",
      "        [-0.0132,  0.0075,  0.0093,  ..., -0.0212, -0.0116,  0.0092],\n",
      "        ...,\n",
      "        [-0.0107, -0.0301, -0.0115,  ..., -0.0022,  0.0401, -0.0024],\n",
      "        [ 0.0146, -0.0012,  0.0160,  ..., -0.0159, -0.0035,  0.0395],\n",
      "        [ 0.0132, -0.0127, -0.0111,  ..., -0.0179, -0.0325, -0.0098]])), ('module.backbone.blocks.15.mlp.fc2.bias', tensor([-0.0077, -0.0024, -0.0124,  ...,  0.0019,  0.0214, -0.0107])), ('module.backbone.blocks.16.norm1.weight', tensor([0.9755, 0.9811, 1.0653,  ..., 1.2790, 1.3584, 1.3106])), ('module.backbone.blocks.16.norm1.bias', tensor([-0.0560, -0.0076,  0.0196,  ...,  0.0221,  0.0221,  0.0326])), ('module.backbone.blocks.16.attn.qkv.weight', tensor([[-0.0178, -0.0059,  0.0031,  ...,  0.0105, -0.0162,  0.0097],\n",
      "        [ 0.0015, -0.0230, -0.0302,  ...,  0.0522, -0.0111, -0.0002],\n",
      "        [ 0.0064,  0.0298,  0.0431,  ..., -0.0201,  0.0305,  0.0123],\n",
      "        ...,\n",
      "        [ 0.0017,  0.0085, -0.0178,  ..., -0.0263,  0.0287,  0.0222],\n",
      "        [ 0.0162,  0.0127,  0.0105,  ...,  0.0176,  0.0053,  0.0108],\n",
      "        [ 0.0027, -0.0031, -0.0055,  ...,  0.0076, -0.0182, -0.0193]])), ('module.backbone.blocks.16.attn.qkv.bias', tensor([ 0.2424, -0.2382, -0.4108,  ...,  0.0248,  0.0250, -0.0159])), ('module.backbone.blocks.16.attn.proj.weight', tensor([[-0.0105,  0.0006, -0.0109,  ...,  0.0276, -0.0181,  0.0159],\n",
      "        [ 0.0006,  0.0156,  0.0221,  ..., -0.0447, -0.0232,  0.0198],\n",
      "        [ 0.0185, -0.0111,  0.0128,  ..., -0.0015,  0.0040,  0.0162],\n",
      "        ...,\n",
      "        [ 0.0276, -0.0081,  0.0294,  ..., -0.0013, -0.0007, -0.0087],\n",
      "        [-0.0160, -0.0136, -0.0188,  ..., -0.0102, -0.0117,  0.0391],\n",
      "        [ 0.0022, -0.0080, -0.0036,  ...,  0.0027,  0.0275,  0.0115]])), ('module.backbone.blocks.16.attn.proj.bias', tensor([ 0.0088,  0.0158, -0.0097,  ..., -0.0105,  0.0342,  0.0084])), ('module.backbone.blocks.16.norm2.weight', tensor([0.9333, 1.2046, 1.2120,  ..., 1.5055, 1.5467, 1.4646])), ('module.backbone.blocks.16.norm2.bias', tensor([ 0.2230, -0.1128, -0.1384,  ...,  0.0664,  0.0242, -0.0736])), ('module.backbone.blocks.16.mlp.fc1.weight', tensor([[ 0.0098,  0.0333, -0.0077,  ...,  0.0292,  0.0102,  0.0195],\n",
      "        [-0.0135,  0.0061,  0.0040,  ..., -0.0236, -0.0101, -0.0057],\n",
      "        [-0.0021, -0.0084,  0.0185,  ...,  0.0050, -0.0250,  0.0050],\n",
      "        ...,\n",
      "        [-0.0046, -0.0066, -0.0189,  ..., -0.0270,  0.0242, -0.0133],\n",
      "        [-0.0092, -0.0092,  0.0110,  ...,  0.0300,  0.0022,  0.0178],\n",
      "        [ 0.0173,  0.0062,  0.0273,  ...,  0.0006,  0.0145, -0.0043]])), ('module.backbone.blocks.16.mlp.fc1.bias', tensor([-0.1434, -0.0639, -0.1237,  ..., -0.0694, -0.2410, -0.1796])), ('module.backbone.blocks.16.mlp.fc2.weight', tensor([[-3.8977e-05,  1.1504e-02, -4.7869e-03,  ...,  3.6222e-03,\n",
      "          3.1551e-02, -4.3829e-03],\n",
      "        [ 2.2332e-02, -1.4450e-02, -9.2046e-03,  ..., -4.6651e-03,\n",
      "         -5.4163e-02,  3.6657e-03],\n",
      "        [-1.3854e-02,  9.6589e-03, -7.7508e-03,  ...,  3.4442e-02,\n",
      "          2.3156e-02,  4.0944e-03],\n",
      "        ...,\n",
      "        [-1.0444e-02,  8.5846e-03, -6.8628e-04,  ...,  1.8654e-02,\n",
      "          1.4769e-02, -5.2859e-02],\n",
      "        [ 1.3801e-03,  8.3123e-03, -1.2833e-02,  ..., -1.7121e-02,\n",
      "         -8.2326e-03, -9.3902e-03],\n",
      "        [ 1.5958e-03,  1.2791e-02,  4.0704e-03,  ...,  8.1277e-03,\n",
      "          1.9197e-03,  6.0377e-03]])), ('module.backbone.blocks.16.mlp.fc2.bias', tensor([-0.0156,  0.0180,  0.0033,  ..., -0.0026,  0.0192, -0.0012])), ('module.backbone.blocks.17.norm1.weight', tensor([1.0916, 1.0351, 1.1505,  ..., 1.3217, 1.3989, 1.4783])), ('module.backbone.blocks.17.norm1.bias', tensor([-0.0534,  0.0121, -0.0144,  ...,  0.0256,  0.0194,  0.0063])), ('module.backbone.blocks.17.attn.qkv.weight', tensor([[ 0.0005,  0.0002,  0.0127,  ...,  0.0027,  0.0114, -0.0240],\n",
      "        [ 0.0053, -0.0111,  0.0049,  ..., -0.0020, -0.0026, -0.0076],\n",
      "        [ 0.0227,  0.0052,  0.0138,  ..., -0.0051, -0.0009,  0.0146],\n",
      "        ...,\n",
      "        [ 0.0035,  0.0037, -0.0009,  ...,  0.0165,  0.0272, -0.0271],\n",
      "        [-0.0006, -0.0065, -0.0068,  ...,  0.0016, -0.0403, -0.0026],\n",
      "        [ 0.0129, -0.0103, -0.0120,  ..., -0.0089, -0.0027,  0.0115]])), ('module.backbone.blocks.17.attn.qkv.bias', tensor([ 0.0306,  0.0245, -0.0050,  ...,  0.0307, -0.0004, -0.0240])), ('module.backbone.blocks.17.attn.proj.weight', tensor([[ 8.5549e-03,  8.1224e-03,  9.4893e-05,  ...,  1.1818e-02,\n",
      "          3.3635e-03,  2.6470e-02],\n",
      "        [-3.8726e-03, -1.8086e-03,  2.3339e-02,  ...,  1.3768e-02,\n",
      "          8.2142e-03, -1.6320e-02],\n",
      "        [-7.0436e-03, -1.9556e-02,  2.6610e-02,  ...,  4.9619e-04,\n",
      "          1.0083e-02, -2.1860e-02],\n",
      "        ...,\n",
      "        [-7.6754e-03, -2.5495e-02, -1.0892e-02,  ..., -3.7540e-03,\n",
      "         -2.8263e-02,  1.6416e-02],\n",
      "        [ 2.9362e-02,  2.0377e-02, -2.3375e-02,  ..., -2.5725e-02,\n",
      "          3.4433e-03,  1.2662e-02],\n",
      "        [-1.4876e-02, -7.4654e-03, -1.7030e-02,  ...,  1.4807e-02,\n",
      "          2.5523e-02,  1.8460e-02]])), ('module.backbone.blocks.17.attn.proj.bias', tensor([ 0.0018,  0.0140, -0.0049,  ..., -0.0115,  0.0082,  0.0321])), ('module.backbone.blocks.17.norm2.weight', tensor([1.0997, 1.3239, 1.2566,  ..., 1.5837, 1.6885, 1.5405])), ('module.backbone.blocks.17.norm2.bias', tensor([ 0.2276, -0.0930, -0.1211,  ...,  0.0557,  0.0119, -0.1069])), ('module.backbone.blocks.17.mlp.fc1.weight', tensor([[-0.0056,  0.0044,  0.0133,  ..., -0.0079,  0.0158,  0.0099],\n",
      "        [ 0.0025,  0.0366,  0.0154,  ..., -0.0206,  0.0240,  0.0339],\n",
      "        [ 0.0028, -0.0246, -0.0175,  ...,  0.0118,  0.0013,  0.0077],\n",
      "        ...,\n",
      "        [-0.0131, -0.0103,  0.0237,  ..., -0.0025,  0.0189,  0.0210],\n",
      "        [-0.0107, -0.0012,  0.0164,  ...,  0.0137, -0.0102, -0.0215],\n",
      "        [ 0.0039,  0.0143, -0.0057,  ...,  0.0421, -0.0198,  0.0310]])), ('module.backbone.blocks.17.mlp.fc1.bias', tensor([-0.2104, -0.1509, -0.1196,  ..., -0.2045, -0.2142, -0.0820])), ('module.backbone.blocks.17.mlp.fc2.weight', tensor([[ 0.0178,  0.0071, -0.0065,  ...,  0.0270,  0.0068,  0.0095],\n",
      "        [-0.0012,  0.0033, -0.0105,  ..., -0.0129,  0.0186, -0.0151],\n",
      "        [ 0.0379, -0.0218, -0.0005,  ...,  0.0145,  0.0002,  0.0041],\n",
      "        ...,\n",
      "        [-0.0129, -0.0131,  0.0084,  ...,  0.0426, -0.0221, -0.0232],\n",
      "        [ 0.0086,  0.0044, -0.0190,  ..., -0.0143,  0.0021,  0.0204],\n",
      "        [ 0.0326,  0.0033,  0.0166,  ..., -0.0135,  0.0294, -0.0359]])), ('module.backbone.blocks.17.mlp.fc2.bias', tensor([-0.0125,  0.0182, -0.0018,  ..., -0.0258, -0.0271,  0.0164])), ('module.backbone.blocks.18.norm1.weight', tensor([1.1366, 1.1360, 1.2493,  ..., 1.3604, 1.4082, 1.4249])), ('module.backbone.blocks.18.norm1.bias', tensor([-0.0641,  0.0244,  0.0115,  ..., -0.0175,  0.0058,  0.0585])), ('module.backbone.blocks.18.attn.qkv.weight', tensor([[ 0.0117, -0.0027,  0.0178,  ..., -0.0111,  0.0046, -0.0431],\n",
      "        [ 0.0036, -0.0015, -0.0244,  ..., -0.0152,  0.0006, -0.0180],\n",
      "        [ 0.0261,  0.0111, -0.0004,  ...,  0.0380,  0.0345,  0.0034],\n",
      "        ...,\n",
      "        [-0.0058,  0.0095, -0.0067,  ...,  0.0013, -0.0124,  0.0030],\n",
      "        [-0.0142,  0.0402, -0.0126,  ..., -0.0145,  0.0133,  0.0075],\n",
      "        [ 0.0315,  0.0100,  0.0064,  ..., -0.0013,  0.0194,  0.0112]])), ('module.backbone.blocks.18.attn.qkv.bias', tensor([-0.0182,  0.1747,  0.0833,  ..., -0.0312, -0.0116,  0.0023])), ('module.backbone.blocks.18.attn.proj.weight', tensor([[ 0.0037,  0.0046,  0.0065,  ...,  0.0231, -0.0077, -0.0295],\n",
      "        [ 0.0015,  0.0170, -0.0127,  ...,  0.0073, -0.0147, -0.0201],\n",
      "        [-0.0147,  0.0163, -0.0356,  ..., -0.0006,  0.0442, -0.0259],\n",
      "        ...,\n",
      "        [ 0.0102,  0.0092, -0.0146,  ..., -0.0178,  0.0288,  0.0091],\n",
      "        [ 0.0074,  0.0206, -0.0141,  ..., -0.0196, -0.0055, -0.0151],\n",
      "        [ 0.0086, -0.0240, -0.0008,  ..., -0.0180,  0.0167, -0.0002]])), ('module.backbone.blocks.18.attn.proj.bias', tensor([ 0.0112,  0.0290, -0.0111,  ..., -0.0192,  0.0035,  0.0247])), ('module.backbone.blocks.18.norm2.weight', tensor([1.1122, 1.4777, 1.3462,  ..., 1.6904, 1.8137, 1.6240])), ('module.backbone.blocks.18.norm2.bias', tensor([ 0.2406, -0.1010, -0.1212,  ...,  0.0198, -0.0355, -0.0903])), ('module.backbone.blocks.18.mlp.fc1.weight', tensor([[-0.0027, -0.0145, -0.0131,  ...,  0.0117, -0.0107,  0.0081],\n",
      "        [-0.0097,  0.0114, -0.0103,  ...,  0.0065,  0.0124,  0.0251],\n",
      "        [-0.0199,  0.0468, -0.0083,  ..., -0.0102, -0.0017, -0.0380],\n",
      "        ...,\n",
      "        [ 0.0167,  0.0012,  0.0285,  ..., -0.0025, -0.0243, -0.0033],\n",
      "        [-0.0045, -0.0137,  0.0038,  ..., -0.0263, -0.0053, -0.0145],\n",
      "        [ 0.0094, -0.0032,  0.0164,  ...,  0.0002, -0.0206, -0.0146]])), ('module.backbone.blocks.18.mlp.fc1.bias', tensor([-0.0154, -0.0629, -0.2075,  ..., -0.0164, -0.0702, -0.0123])), ('module.backbone.blocks.18.mlp.fc2.weight', tensor([[ 0.0024,  0.0114, -0.0067,  ..., -0.0163, -0.0152, -0.0021],\n",
      "        [ 0.0246, -0.0159,  0.0098,  ..., -0.0108, -0.0015, -0.0038],\n",
      "        [-0.0031, -0.0014, -0.0042,  ..., -0.0178,  0.0018, -0.0383],\n",
      "        ...,\n",
      "        [ 0.0041,  0.0027, -0.0162,  ...,  0.0145,  0.0015,  0.0037],\n",
      "        [ 0.0083,  0.0130,  0.0057,  ...,  0.0330,  0.0089,  0.0271],\n",
      "        [-0.0113, -0.0096, -0.0306,  ...,  0.0080,  0.0193,  0.0201]])), ('module.backbone.blocks.18.mlp.fc2.bias', tensor([-0.0149,  0.0243,  0.0022,  ..., -0.0073, -0.0339,  0.0265])), ('module.backbone.blocks.19.norm1.weight', tensor([1.1537, 1.1670, 1.1950,  ..., 1.4461, 1.4447, 1.3817])), ('module.backbone.blocks.19.norm1.bias', tensor([-0.0687,  0.0496,  0.0508,  ..., -0.0105,  0.0073,  0.0497])), ('module.backbone.blocks.19.attn.qkv.weight', tensor([[-0.0251,  0.0152, -0.0057,  ..., -0.0184,  0.0101,  0.0024],\n",
      "        [-0.0106, -0.0495,  0.0144,  ..., -0.0102,  0.0283, -0.0003],\n",
      "        [ 0.0164,  0.0322, -0.0122,  ...,  0.0020,  0.0127, -0.0018],\n",
      "        ...,\n",
      "        [ 0.0083,  0.0067,  0.0053,  ..., -0.0012,  0.0033, -0.0028],\n",
      "        [-0.0027,  0.0138,  0.0051,  ...,  0.0321,  0.0026, -0.0123],\n",
      "        [-0.0060,  0.0149,  0.0027,  ..., -0.0221, -0.0170,  0.0003]])), ('module.backbone.blocks.19.attn.qkv.bias', tensor([ 0.2899, -0.0362, -0.0106,  ..., -0.0100,  0.0096, -0.0016])), ('module.backbone.blocks.19.attn.proj.weight', tensor([[ 0.0084,  0.0120, -0.0202,  ...,  0.0080,  0.0069,  0.0352],\n",
      "        [ 0.0087,  0.0171, -0.0092,  ...,  0.0018, -0.0155, -0.0097],\n",
      "        [-0.0083, -0.0054, -0.0008,  ...,  0.0220, -0.0054, -0.0011],\n",
      "        ...,\n",
      "        [-0.0115,  0.0129, -0.0081,  ...,  0.0319, -0.0141,  0.0126],\n",
      "        [ 0.0123,  0.0024,  0.0071,  ..., -0.0106,  0.0066,  0.0324],\n",
      "        [-0.0112,  0.0109,  0.0135,  ..., -0.0155, -0.0022,  0.0045]])), ('module.backbone.blocks.19.attn.proj.bias', tensor([ 0.0028,  0.0136, -0.0199,  ..., -0.0137, -0.0044,  0.0420])), ('module.backbone.blocks.19.norm2.weight', tensor([1.2835, 1.6258, 1.5355,  ..., 1.7116, 1.8512, 1.6087])), ('module.backbone.blocks.19.norm2.bias', tensor([ 0.2112, -0.0840, -0.1747,  ...,  0.0252, -0.0146, -0.0581])), ('module.backbone.blocks.19.mlp.fc1.weight', tensor([[ 0.0082,  0.0110, -0.0122,  ..., -0.0040, -0.0055,  0.0006],\n",
      "        [-0.0139, -0.0047,  0.0126,  ..., -0.0077, -0.0323,  0.0186],\n",
      "        [ 0.0059,  0.0065, -0.0090,  ..., -0.0158, -0.0121,  0.0118],\n",
      "        ...,\n",
      "        [-0.0257,  0.0318, -0.0110,  ...,  0.0322, -0.0240,  0.0246],\n",
      "        [-0.0093, -0.0084, -0.0047,  ..., -0.0131,  0.0135, -0.0184],\n",
      "        [ 0.0104,  0.0284,  0.0206,  ...,  0.0203,  0.0007, -0.0324]])), ('module.backbone.blocks.19.mlp.fc1.bias', tensor([-0.0283, -0.0885, -0.0903,  ..., -0.1519, -0.0095, -0.1986])), ('module.backbone.blocks.19.mlp.fc2.weight', tensor([[ 0.0033,  0.0061,  0.0044,  ..., -0.0157,  0.0051, -0.0225],\n",
      "        [-0.0073, -0.0052, -0.0080,  ...,  0.0137,  0.0038,  0.0127],\n",
      "        [ 0.0060,  0.0115,  0.0050,  ..., -0.0242,  0.0162, -0.0070],\n",
      "        ...,\n",
      "        [ 0.0076, -0.0006, -0.0080,  ...,  0.0147,  0.0105,  0.0157],\n",
      "        [ 0.0043,  0.0097,  0.0065,  ..., -0.0262, -0.0156, -0.0121],\n",
      "        [-0.0065, -0.0013,  0.0009,  ...,  0.0187,  0.0138,  0.0104]])), ('module.backbone.blocks.19.mlp.fc2.bias', tensor([-0.0031,  0.0162,  0.0043,  ..., -0.0151, -0.0341,  0.0200])), ('module.backbone.blocks.20.norm1.weight', tensor([1.0807, 1.1600, 1.2347,  ..., 1.4253, 1.3581, 1.4127])), ('module.backbone.blocks.20.norm1.bias', tensor([-0.0574,  0.0550,  0.0427,  ...,  0.0113,  0.0202,  0.0608])), ('module.backbone.blocks.20.attn.qkv.weight', tensor([[-0.0107,  0.0077, -0.0020,  ..., -0.0063, -0.0106, -0.0185],\n",
      "        [ 0.0185,  0.0134,  0.0090,  ..., -0.0011,  0.0034, -0.0164],\n",
      "        [ 0.0152,  0.0322,  0.0143,  ..., -0.0234,  0.0088,  0.0191],\n",
      "        ...,\n",
      "        [ 0.0134, -0.0166, -0.0039,  ..., -0.0176,  0.0089, -0.0030],\n",
      "        [-0.0014, -0.0225,  0.0228,  ..., -0.0048,  0.0077,  0.0437],\n",
      "        [ 0.0071, -0.0077,  0.0058,  ..., -0.0329,  0.0056,  0.0094]])), ('module.backbone.blocks.20.attn.qkv.bias', tensor([-0.4863, -0.0456, -0.1790,  ...,  0.0099, -0.0051,  0.0429])), ('module.backbone.blocks.20.attn.proj.weight', tensor([[ 0.0069, -0.0453,  0.0192,  ..., -0.0130, -0.0149,  0.0126],\n",
      "        [ 0.0008, -0.0073,  0.0043,  ...,  0.0010, -0.0192,  0.0353],\n",
      "        [ 0.0134,  0.0079,  0.0135,  ...,  0.0177, -0.0012, -0.0101],\n",
      "        ...,\n",
      "        [-0.0295,  0.0289, -0.0011,  ...,  0.0308, -0.0251,  0.0509],\n",
      "        [ 0.0012,  0.0039, -0.0074,  ...,  0.0165, -0.0081, -0.0359],\n",
      "        [-0.0232,  0.0046, -0.0209,  ..., -0.0071,  0.0141, -0.0043]])), ('module.backbone.blocks.20.attn.proj.bias', tensor([-0.0012, -0.0078, -0.0299,  ..., -0.0070, -0.0211,  0.0245])), ('module.backbone.blocks.20.norm2.weight', tensor([1.3918, 1.7558, 1.6700,  ..., 1.7742, 1.9106, 1.5993])), ('module.backbone.blocks.20.norm2.bias', tensor([ 0.2361, -0.0697, -0.2116,  ...,  0.0393,  0.0259, -0.0586])), ('module.backbone.blocks.20.mlp.fc1.weight', tensor([[ 0.0160,  0.0253,  0.0098,  ..., -0.0025,  0.0126,  0.0247],\n",
      "        [-0.0090,  0.0058, -0.0168,  ...,  0.0004, -0.0013, -0.0018],\n",
      "        [-0.0019,  0.0043,  0.0163,  ..., -0.0122, -0.0119,  0.0193],\n",
      "        ...,\n",
      "        [-0.0222,  0.0044, -0.0038,  ..., -0.0005, -0.0087, -0.0047],\n",
      "        [ 0.0099,  0.0125, -0.0033,  ..., -0.0077,  0.0019, -0.0130],\n",
      "        [-0.0031,  0.0165,  0.0009,  ..., -0.0144,  0.0205,  0.0014]])), ('module.backbone.blocks.20.mlp.fc1.bias', tensor([-0.0637, -0.0408, -0.0538,  ..., -0.0587, -0.0492, -0.0800])), ('module.backbone.blocks.20.mlp.fc2.weight', tensor([[-0.0259,  0.0009, -0.0114,  ..., -0.0138, -0.0064,  0.0177],\n",
      "        [-0.0185, -0.0115, -0.0038,  ..., -0.0102, -0.0102,  0.0092],\n",
      "        [-0.0071,  0.0134, -0.0119,  ..., -0.0186,  0.0045, -0.0038],\n",
      "        ...,\n",
      "        [ 0.0037, -0.0075,  0.0064,  ..., -0.0208, -0.0095, -0.0287],\n",
      "        [-0.0080, -0.0091,  0.0010,  ...,  0.0002, -0.0006, -0.0168],\n",
      "        [-0.0162,  0.0117,  0.0075,  ..., -0.0050,  0.0104,  0.0006]])), ('module.backbone.blocks.20.mlp.fc2.bias', tensor([-0.0115, -0.0014, -0.0020,  ..., -0.0088, -0.0353,  0.0257])), ('module.backbone.blocks.21.norm1.weight', tensor([1.1128, 1.0929, 1.1800,  ..., 1.3820, 1.2767, 1.4118])), ('module.backbone.blocks.21.norm1.bias', tensor([-0.0901,  0.0131,  0.0684,  ...,  0.0130,  0.0166,  0.0557])), ('module.backbone.blocks.21.attn.qkv.weight', tensor([[ 0.0328,  0.0017,  0.0154,  ..., -0.0072, -0.0277,  0.0051],\n",
      "        [ 0.0128, -0.0183, -0.0312,  ...,  0.0176,  0.0126, -0.0070],\n",
      "        [-0.0132, -0.0027, -0.0013,  ..., -0.0141,  0.0077, -0.0328],\n",
      "        ...,\n",
      "        [ 0.0129, -0.0072, -0.0038,  ..., -0.0384,  0.0201,  0.0456],\n",
      "        [-0.0069,  0.0053, -0.0064,  ...,  0.0319,  0.0194,  0.0018],\n",
      "        [ 0.0266,  0.0134, -0.0132,  ..., -0.0111,  0.0107,  0.0060]])), ('module.backbone.blocks.21.attn.qkv.bias', tensor([-0.4004, -0.0239, -0.3548,  ..., -0.0235, -0.0107, -0.0255])), ('module.backbone.blocks.21.attn.proj.weight', tensor([[-0.0216, -0.0332,  0.0174,  ..., -0.0161,  0.0029, -0.0039],\n",
      "        [-0.0087,  0.0351, -0.0204,  ..., -0.0169,  0.0092, -0.0143],\n",
      "        [-0.0321,  0.0240, -0.0183,  ..., -0.0082, -0.0106,  0.0001],\n",
      "        ...,\n",
      "        [-0.0449, -0.0094,  0.0182,  ..., -0.0253,  0.0060,  0.0108],\n",
      "        [-0.0170, -0.0118,  0.0053,  ...,  0.0353, -0.0040,  0.0048],\n",
      "        [ 0.0115,  0.0094,  0.0040,  ..., -0.0424, -0.0121, -0.0023]])), ('module.backbone.blocks.21.attn.proj.bias', tensor([ 0.0146, -0.0050, -0.0337,  ...,  0.0011, -0.0221,  0.0182])), ('module.backbone.blocks.21.norm2.weight', tensor([1.6153, 1.7193, 1.7957,  ..., 1.7123, 1.8515, 1.6613])), ('module.backbone.blocks.21.norm2.bias', tensor([ 0.2580, -0.1015, -0.3023,  ...,  0.0533,  0.0274, -0.0471])), ('module.backbone.blocks.21.mlp.fc1.weight', tensor([[-0.0029,  0.0048, -0.0076,  ..., -0.0099, -0.0240,  0.0133],\n",
      "        [-0.0105, -0.0293,  0.0032,  ...,  0.0165, -0.0111,  0.0265],\n",
      "        [-0.0107,  0.0073,  0.0254,  ..., -0.0116, -0.0186, -0.0130],\n",
      "        ...,\n",
      "        [-0.0063, -0.0138,  0.0159,  ...,  0.0081, -0.0188, -0.0149],\n",
      "        [-0.0083, -0.0062, -0.0032,  ..., -0.0212,  0.0068, -0.0223],\n",
      "        [-0.0049,  0.0148, -0.0011,  ..., -0.0055,  0.0374,  0.0106]])), ('module.backbone.blocks.21.mlp.fc1.bias', tensor([-0.0261, -0.1371, -0.1125,  ..., -0.0393, -0.0851, -0.0873])), ('module.backbone.blocks.21.mlp.fc2.weight', tensor([[-4.6795e-03, -8.5243e-03,  6.7794e-03,  ..., -7.5647e-03,\n",
      "          1.3032e-02,  8.2898e-03],\n",
      "        [ 1.5409e-05, -7.0184e-02, -8.4499e-03,  ...,  1.1478e-02,\n",
      "         -1.2707e-03,  9.1301e-03],\n",
      "        [ 5.6879e-03, -1.7050e-03,  9.7161e-03,  ...,  1.4690e-02,\n",
      "          9.7171e-03,  1.8307e-02],\n",
      "        ...,\n",
      "        [ 2.0586e-02,  4.5302e-03, -1.0505e-02,  ..., -4.4028e-03,\n",
      "         -1.9479e-02, -9.4744e-03],\n",
      "        [ 3.4730e-02, -5.6265e-03, -1.9568e-02,  ..., -1.0544e-02,\n",
      "         -2.1788e-04,  6.2672e-03],\n",
      "        [-7.1774e-03,  9.4550e-04,  6.0876e-03,  ..., -9.8822e-03,\n",
      "          2.5417e-02,  4.2177e-03]])), ('module.backbone.blocks.21.mlp.fc2.bias', tensor([-0.0029, -0.0057,  0.0110,  ..., -0.0071, -0.0305,  0.0189])), ('module.backbone.blocks.22.norm1.weight', tensor([1.1407, 1.1467, 1.2274,  ..., 1.3283, 1.2781, 1.3564])), ('module.backbone.blocks.22.norm1.bias', tensor([-0.0834,  0.0334,  0.0597,  ...,  0.0493,  0.0492,  0.0646])), ('module.backbone.blocks.22.attn.qkv.weight', tensor([[ 0.0225, -0.0065,  0.0164,  ...,  0.0272,  0.0008,  0.0187],\n",
      "        [-0.0326, -0.0366, -0.0001,  ...,  0.0078, -0.0057, -0.0006],\n",
      "        [-0.0149, -0.0313,  0.0272,  ...,  0.0114,  0.0007,  0.0192],\n",
      "        ...,\n",
      "        [ 0.0046,  0.0054,  0.0060,  ..., -0.0270, -0.0082, -0.0030],\n",
      "        [-0.0021, -0.0139, -0.0204,  ..., -0.0137, -0.0119, -0.0373],\n",
      "        [-0.0045, -0.0033, -0.0116,  ...,  0.0295, -0.0022,  0.0303]])), ('module.backbone.blocks.22.attn.qkv.bias', tensor([ 0.0864, -0.1129,  0.1427,  ..., -0.0321, -0.0145, -0.0126])), ('module.backbone.blocks.22.attn.proj.weight', tensor([[-0.0268,  0.0313,  0.0122,  ...,  0.0068, -0.0107,  0.0053],\n",
      "        [ 0.0009, -0.0088,  0.0008,  ...,  0.0049, -0.0066,  0.0147],\n",
      "        [ 0.0069,  0.0202,  0.0056,  ..., -0.0216,  0.0050, -0.0162],\n",
      "        ...,\n",
      "        [ 0.0062, -0.0156, -0.0254,  ...,  0.0362,  0.0078, -0.0215],\n",
      "        [-0.0312, -0.0248, -0.0374,  ..., -0.0113, -0.0161, -0.0494],\n",
      "        [ 0.0418, -0.0137,  0.0215,  ...,  0.0030,  0.0046, -0.0089]])), ('module.backbone.blocks.22.attn.proj.bias', tensor([ 0.0162, -0.0108, -0.0222,  ..., -0.0099, -0.0287,  0.0061])), ('module.backbone.blocks.22.norm2.weight', tensor([1.8946, 1.6927, 1.8119,  ..., 1.5745, 1.6501, 1.6067])), ('module.backbone.blocks.22.norm2.bias', tensor([ 0.2009, -0.1529, -0.2779,  ...,  0.0539,  0.0145, -0.0015])), ('module.backbone.blocks.22.mlp.fc1.weight', tensor([[-0.0193, -0.0127,  0.0099,  ..., -0.0015,  0.0022,  0.0043],\n",
      "        [ 0.0001, -0.0053,  0.0024,  ...,  0.0088,  0.0055, -0.0168],\n",
      "        [-0.0058,  0.0056,  0.0010,  ..., -0.0144,  0.0088,  0.0122],\n",
      "        ...,\n",
      "        [-0.0138,  0.0031,  0.0032,  ...,  0.0024,  0.0216, -0.0014],\n",
      "        [ 0.0109, -0.0004, -0.0036,  ...,  0.0015,  0.0025,  0.0005],\n",
      "        [-0.0307, -0.0011,  0.0090,  ...,  0.0181, -0.0033,  0.0055]])), ('module.backbone.blocks.22.mlp.fc1.bias', tensor([-0.0495, -0.0798, -0.1456,  ..., -0.0672, -0.0505, -0.0627])), ('module.backbone.blocks.22.mlp.fc2.weight', tensor([[ 0.0132, -0.0048, -0.0158,  ..., -0.0150,  0.0206,  0.0078],\n",
      "        [ 0.0124,  0.0130, -0.0131,  ..., -0.0147, -0.0087,  0.0082],\n",
      "        [-0.0003,  0.0020, -0.0180,  ..., -0.0042,  0.0207,  0.0176],\n",
      "        ...,\n",
      "        [-0.0067,  0.0066, -0.0081,  ..., -0.0274,  0.0083, -0.0134],\n",
      "        [-0.0131, -0.0154,  0.0221,  ...,  0.0141,  0.0050,  0.0050],\n",
      "        [-0.0103,  0.0052,  0.0013,  ..., -0.0067,  0.0023,  0.0148]])), ('module.backbone.blocks.22.mlp.fc2.bias', tensor([-7.1326e-05,  4.3737e-03,  1.9963e-02,  ..., -1.7858e-02,\n",
      "        -2.5380e-02,  1.3540e-02])), ('module.backbone.blocks.23.norm1.weight', tensor([1.2214, 1.2635, 1.2391,  ..., 1.2712, 1.2411, 1.2689])), ('module.backbone.blocks.23.norm1.bias', tensor([-0.1114,  0.0639,  0.0616,  ...,  0.0682,  0.0811,  0.0479])), ('module.backbone.blocks.23.attn.qkv.weight', tensor([[ 0.0249, -0.0126,  0.0196,  ..., -0.0066, -0.0131, -0.0105],\n",
      "        [ 0.0246,  0.0201,  0.0080,  ...,  0.0253, -0.0257, -0.0129],\n",
      "        [ 0.0091,  0.0177, -0.0208,  ...,  0.0108,  0.0212, -0.0094],\n",
      "        ...,\n",
      "        [-0.0193, -0.0054,  0.0102,  ..., -0.0030, -0.0111,  0.0119],\n",
      "        [-0.0002, -0.0051,  0.0081,  ...,  0.0168,  0.0054,  0.0058],\n",
      "        [ 0.0015,  0.0233,  0.0080,  ..., -0.0179,  0.0077, -0.0213]])), ('module.backbone.blocks.23.attn.qkv.bias', tensor([ 0.0953, -0.0898,  0.0302,  ...,  0.0076, -0.0217,  0.0038])), ('module.backbone.blocks.23.attn.proj.weight', tensor([[ 0.0072,  0.0011,  0.0168,  ...,  0.0052,  0.0126, -0.0023],\n",
      "        [ 0.0063,  0.0003, -0.0069,  ...,  0.0084, -0.0177,  0.0041],\n",
      "        [ 0.0268,  0.0061, -0.0155,  ..., -0.0269,  0.0044, -0.0093],\n",
      "        ...,\n",
      "        [ 0.0254, -0.0232,  0.0089,  ..., -0.0199,  0.0078,  0.0099],\n",
      "        [ 0.0170,  0.0089,  0.0034,  ..., -0.0301, -0.0057, -0.0392],\n",
      "        [-0.0027, -0.0086, -0.0265,  ...,  0.0018, -0.0187, -0.0004]])), ('module.backbone.blocks.23.attn.proj.bias', tensor([ 0.0437, -0.0227, -0.0206,  ..., -0.0403, -0.0595,  0.0147])), ('module.backbone.blocks.23.norm2.weight', tensor([1.8774, 1.5253, 1.6057,  ..., 1.4571, 1.4678, 1.4068])), ('module.backbone.blocks.23.norm2.bias', tensor([-0.0200, -0.1218, -0.0673,  ..., -0.0145, -0.0238, -0.0111])), ('module.backbone.blocks.23.mlp.fc1.weight', tensor([[-0.0171,  0.0010, -0.0141,  ...,  0.0207,  0.0059,  0.0098],\n",
      "        [ 0.0073,  0.0113,  0.0093,  ..., -0.0012,  0.0071,  0.0325],\n",
      "        [ 0.0230,  0.0050,  0.0006,  ...,  0.0259, -0.0077,  0.0014],\n",
      "        ...,\n",
      "        [ 0.0003,  0.0112,  0.0020,  ..., -0.0073,  0.0020,  0.0028],\n",
      "        [ 0.0045,  0.0088, -0.0181,  ...,  0.0163, -0.0125, -0.0281],\n",
      "        [-0.0012,  0.0020,  0.0139,  ..., -0.0228, -0.0103,  0.0046]])), ('module.backbone.blocks.23.mlp.fc1.bias', tensor([-0.0756, -0.0284, -0.1218,  ..., -0.1461, -0.0623, -0.0647])), ('module.backbone.blocks.23.mlp.fc2.weight', tensor([[-0.0134,  0.0029,  0.0081,  ..., -0.0023, -0.0121, -0.0024],\n",
      "        [-0.0204, -0.0279, -0.0044,  ..., -0.0118,  0.0019,  0.0124],\n",
      "        [-0.0070,  0.0004,  0.0026,  ...,  0.0039,  0.0086, -0.0126],\n",
      "        ...,\n",
      "        [-0.0121,  0.0068,  0.0030,  ..., -0.0014,  0.0054,  0.0186],\n",
      "        [ 0.0020,  0.0004, -0.0017,  ...,  0.0031,  0.0005,  0.0093],\n",
      "        [ 0.0028, -0.0143, -0.0006,  ..., -0.0058, -0.0098, -0.0018]])), ('module.backbone.blocks.23.mlp.fc2.bias', tensor([ 0.0877,  0.0231,  0.1154,  ..., -0.0042, -0.0033,  0.0224])), ('module.backbone.norm.weight', tensor([1.8262, 2.0317, 1.8274,  ..., 1.3849, 1.8330, 0.5778])), ('module.backbone.norm.bias', tensor([ 0.0090,  0.0238, -0.0286,  ...,  0.0408,  0.0410, -0.0259]))])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "# モデルの読み込み\n",
    "PATH = '/content/drive/MyDrive/vitl16.pth.tar'  # モデルファイルへのパスを指定\n",
    "my_dict = torch.load(PATH, map_location=torch.device('cpu'))\n",
    "\n",
    "classifier=my_dict['target_encoder']\n",
    "\n",
    "print(f\"Dictionary contents: {classifier}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 295,
     "status": "ok",
     "timestamp": 1720226270367,
     "user": {
      "displayName": "深川梓",
      "userId": "06822277665795432790"
     },
     "user_tz": -540
    },
    "id": "Lerq2Zyn04v1",
    "outputId": "73fcb9bb-29ad-4c4e-9154-439b8d4e8f92"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/sagemaker-user/content/drive/MyDrive/jepa_images\n",
      "Filesystem      Size  Used Avail Use% Mounted on\n",
      "overlay          32G   52K   32G   1% /\n",
      "tmpfs            64M     0   64M   0% /dev\n",
      "tmpfs           1.9G     0  1.9G   0% /sys/fs/cgroup\n",
      "shm             392M   20K  392M   1% /dev/shm\n",
      "/dev/nvme0n1p1  160G   32G  129G  20% /opt/.sagemakerinternal\n",
      "/dev/nvme1n1    500G  8.6G  492G   2% /home/sagemaker-user\n",
      "tmpfs           1.9G     0  1.9G   0% /proc/acpi\n",
      "tmpfs           1.9G     0  1.9G   0% /sys/firmware\n"
     ]
    }
   ],
   "source": [
    "%cd ~/\n",
    "!df -h"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": [
    {
     "file_id": "1HYLXN7dS8Njxh-X51rAQ0coeyqEuyY4j",
     "timestamp": 1718078088400
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}