NEW_EveryDream_Tools_4_colab.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/nawnie/New-Ed-Tools/blob/main/NEW_EveryDream_Tools_4_colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uJfwih4wAVgw"
      },
      "source": [
        "# Please read the documentation here before you start.\n",
        "\n",
        "I suggest reading this doc before you connect to your runtime to avoid using credits or being charged while you figure it out.\n",
        "\n",
        "[link to old readme, new one is a wip](doc/AUTO_CAPTION.md)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title # Install python 3.10  and connect Gdrive\n",
        "#@markdown # This will show a runtime error, it's ok, it's on purpose to restart the kernel to update python.\n",
        "import os\n",
        "import time\n",
        "import sys\n",
        "from IPython.display import clear_output\n",
        "\n",
        "\n",
        "#@markdown Optional connect Gdrive  But strongly recommended\n",
        "#@markdown This will let you put all your training data and checkpoints directly on your drive.  Much faster/easier to continue later, less setup time.\n",
        "\n",
        "#@markdown Creates /content/drive/MyDrive/everydreamlogs/ckpt\n",
        "Mount_to_Gdrive = True #@param{type:\"boolean\"} \n",
        "\n",
        "if Mount_to_Gdrive:\n",
        "  from google.colab import drive\n",
        "  drive.mount('/content/drive')\n",
        "\n",
        "  !mkdir -p /content/drive/MyDrive/everydreamlogs/ckpt\n",
        "\n",
        "# Define a custom function to display a progress bar\n",
        "def display_progress_bar(progress, total, prefix=\"\"):\n",
        "    sys.stdout.write(f\"\\r{prefix}[{'=' * progress}>{' ' * (total - progress - 1)}] {progress + 1}/{total}\")\n",
        "    sys.stdout.flush()\n",
        "\n",
        "total_steps = 10\n",
        "current_step = 0\n",
        "\n",
        "!pip install patool --progress-bar on --quiet\n",
        "current_step += 1\n",
        "display_progress_bar(current_step, total_steps, \"install progress:\")\n",
        "\n",
        "!pip install transformers==4.25.1 --progress-bar on --quiet\n",
        "current_step += 1\n",
        "display_progress_bar(current_step, total_steps, \"install progress:\")\n",
        "\n",
        "\n",
        "!pip install watchdog --progress-bar on --quiet\n",
        "current_step += 1\n",
        "display_progress_bar(current_step, total_steps, \"install progress:\")\n",
        "\n",
        "!pip install matplotlib --progress-bar on --quiet\n",
        "current_step += 1\n",
        "display_progress_bar(current_step, total_steps, \"install progress:\")\n",
        "\n",
        "# Install the alive-package library\n",
        "!pip install alive-progress --progress-bar on --quiet\n",
        "current_step += 1\n",
        "display_progress_bar(current_step, total_steps, \"install progress:\")\n",
        "\n",
        "\n",
        "# Install the tqdm library\n",
        "!pip install tqdm --progress-bar on --quiet\n",
        "current_step += 1\n",
        "display_progress_bar(current_step, total_steps, \"install progress:\")\n",
        "\n",
        "# Download the py310.sh script\n",
        "!wget https://github.com/korakot/kora/releases/download/v0.10/py310.sh -q\n",
        "current_step += 1\n",
        "display_progress_bar(current_step, total_steps, \"install progress:\")\n",
        "\n",
        "# Run the py310.sh script\n",
        "try:\n",
        "    output = os.popen('bash ./py310.sh -b -f -p /usr/local 2>&1').read()\n",
        "    total_lines = len(output.splitlines())\n",
        "    for i, line in enumerate(output.splitlines()):\n",
        "        clear_output(wait=True)\n",
        "        display_progress_bar(i, total_lines, \"install progress:\")\n",
        "except Exception as e:\n",
        "    print(str(e))\n",
        "\n",
        "current_step += 1\n",
        "display_progress_bar(current_step, total_steps, \"install progress:\")\n",
        "\n",
        "# Install the py310 kernel\n",
        "!python -m ipykernel install --name \"py310\" --user > /dev/null 2>&1\n",
        "current_step += 1\n",
        "display_progress_bar(current_step, total_steps, \"install progress:\")\n",
        "\n",
        "# Clear output\n",
        "!rm /content/py310.sh\n",
        "current_step += 1\n",
        "display_progress_bar(current_step, total_steps, \"install progress:\")\n",
        "clear_output()\n",
        "time.sleep(1) #needed to clear is before kill\n",
        "os.kill(os.getpid(), 9)\n",
        "print(\"\\nInstallation completed.\")\n"
      ],
      "metadata": {
        "id": "Z_ZHfnQ52dg9",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RJxfSai-8pkD",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@markdown # Finish Install Dependencies into the new python\n",
        "#@markdown This will take a couple minutes, be patient and watch the output for \"DONE!\"\n",
        "from IPython.display import clear_output\n",
        "import subprocess\n",
        "from tqdm.notebook import tqdm\n",
        "\n",
        "print(\"downloading the required repos to use\")\n",
        "#downloading repos\n",
        "\n",
        "!git clone https://github.com/victorchall/EveryDream.git\n",
        "\n",
        "# Set working directory\n",
        "\n",
        "%cd EveryDream\n",
        "\n",
        "!git clone https://github.com/salesforce/BLIP scripts/BLIP\n",
        "!wget -O caption.py https://raw.githubusercontent.com/nawnie/New-Ed-Tools/main/caption.py\n",
        "!pip install git+https://github.com/huggingface/transformers \n",
        "\n",
        "clear_output()\n",
        "\n",
        "print(\"DONE! now, installing dependcies, this may seem to freeze for a moment at the start do not worry\")\n",
        "\n",
        "packages = [\n",
        "    ('torch==1.13.1+cu117 torchvision==0.14.1+cu117', 'https://download.pytorch.org/whl/cu117'),\n",
        "    'diffusers[torch]==0.13.0',\n",
        "    'pynvml==11.4.1',\n",
        "    'bitsandbytes==0.35.0',\n",
        "    'pandas>=1.3.5'\n",
        "    'aiofiles'\n",
        "    'timm'\n",
        "    'fairscale==0.4.4'\n",
        "    'ftfy==6.1.1',\n",
        "    'aiohttp==3.8.3',\n",
        "    'tensorboard>=2.11.0',\n",
        "    'protobuf==3.20.1',\n",
        "    'wandb==0.13.6',\n",
        "    'pyre-extensions==0.0.23',\n",
        "    'xformers==0.0.16',\n",
        "    'pytorch-lightning==1.6.5',\n",
        "    'OmegaConf==2.2.3',\n",
        "    'numpy==1.23.5',\n",
        "    'colorama',\n",
        "    'keyboard',\n",
        "    'triton',\n",
        "    'lion-pytorch'\n",
        "]\n",
        "\n",
        "for package in tqdm(packages, desc='Installing packages', unit='package'):\n",
        "    if isinstance(package, tuple):\n",
        "        package_name, extra_index_url = package\n",
        "        cmd = f\"pip install -q {package_name} --extra-index-url {extra_index_url}\"\n",
        "    else:\n",
        "        cmd = f\"pip install -q {package}\"\n",
        "        \n",
        "    subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n",
        "\n",
        "!pip install nltk #oddly this does not install correctly with above process\n",
        "\n",
        "clear_output()\n",
        "\n",
        "\n",
        "!python utils/get_yamls.py\n",
        "clear_output()\n",
        "print(\"DONE! installing dependcies make sure we are using python 3.10.x\")\n",
        "!python --version\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Upload your input images or video into the EveryDream/input folder\n",
        "#@markdown Run the following cell to create an upload button, allowing you to upload your images directly to this folder. \n",
        "#@markdown * it is faster to simply right click the input folder in the file browser available on the left toolbar\n",
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()\n",
        "\n",
        "for name, data in uploaded.items():\n",
        "    with open(f\"input/{name}\", \"wb\") as f:\n",
        "        f.write(data)\n",
        "    print(f\"Uploaded file: {name}\")\n",
        "\n"
      ],
      "metadata": {
        "cellView": "form",
        "id": "pvLcqUEobmAV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Blip, Blip-COCO, T5-FLAN, and Git Captioner\n",
        "_____________________________________\n",
        "\n",
        "\"Blip\", \"Blip-COCO\", \"T5-FLAN\", and \"Git Captioner\" are four transformer-based models developed for image captioning.\n",
        "\n",
        "* \"Blip\" is trained on a large and diverse dataset of image captions, and is known for its strong performance on a wide range of image captioning tasks and datasets.\n",
        "\n",
        "* \"Blip-COCO\" is a variant of \"Blip\" that has been fine-tuned on the COCO dataset, a popular benchmark for image captioning. Fine-tuning on COCO can improve a model's performance specifically on this dataset, but may not generalize as well to other datasets or tasks.\n",
        "\n",
        " * The COCO dataset consists of more than 330,000 images with five captions each, and is known for its high-quality captions and diversity of image types. \n",
        "\n",
        "\n",
        "* \"T5-FLAN\" is similar to \"Blip\" in that it is trained on a large and diverse dataset of image captions. However, it uses a different architecture called the \"FLAN\" (Feature-wise Linear Attention) transformer. The FLAN transformer is designed to better capture long-range dependencies and improve the modeling of feature interactions in the encoder, which can lead to better performance on certain tasks.\n",
        "\n",
        "* \"Git Captioner\" is a model that generates image captions using a combination of computer vision and natural language processing techniques. It leverages pre-trained models for object detection and recognition, and then generates captions using a transformer-based language model.\n",
        "\n",
        "When choosing among these models, consider your specific task or dataset needs. If you require a model that can perform well across a wide range of image captioning tasks and datasets, \"Blip\" or \"T5-FLAN\" may be a good choice. If you require strong performance specifically on the COCO dataset, consider fine-tuning \"Blip-COCO\". If you prefer a model that leverages object detection and recognition in addition to language modeling, consider \"Git Captioner\".\n",
        "\n",
        "In addition to image captioning, \"T5-FLAN\" and \"Git Captioner\" have also been applied to tasks such as image generation and visual question answering.\n",
        "\n"
      ],
      "metadata": {
        "id": "NPWIluxMhAm8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Blip2 uses alot of ram, this cell will clear our ram usage\n",
        "#@markdown Run this cell before captioning and if your model crashes while loading.\n",
        "import gc\n",
        "gc.collect()"
      ],
      "metadata": {
        "cellView": "form",
        "id": "23gKI8QGJTyM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4TAICahl-RPn",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Dream Captioner\n",
        "input_folder = \"/content/EveryDream/input\" #@param {type:\"string\"}\n",
        "#@markdown * The location of images to be captoned\n",
        "model_name = \"Salesforce/blip2-opt-2.7b-coco\" #@param [\"microsoft/git-large-r-textcaps\", \"microsoft/git-large-textcaps\", \"microsoft/git-base-textcaps\", \"Salesforce/blip2-opt-2.7b\", \"Salesforce/blip2-opt-2.7b-coco\", \"Salesforce/blip2-flan-t5-xl\"]\n",
        "#@markdown * Select a model from the drop down menu, the ones on this menu have been tested to run in this colab enviroment.\n",
        "model_storage = \"/content/\" #@param {type:\"string\"}\n",
        "#@markdown * Choose where to save these models on the Gdrive\n",
        "token_limit = 24 #@param {type:\"slider\", min:10, max:38, step:1}\n",
        "#@markdown * Token length\n",
        "replace_subject = True #@param {type:\"boolean\"}\n",
        "#@markdown * This will find the first noun and replace it with the name of the folder it was in\n",
        "#@markdown * EXAMPLE: if i caption images in a folder named Peter Griffin the first noun (person place or thing) in that caption will be replaced by \"Peter Griffin\"\n",
        "use_cpu = False #@param {type:\"boolean\"}\n",
        "#@markdown * this probably should not be used with colab but its there to try without a gpu\n",
        "\n",
        "rename = \"\"\n",
        "if replace_subject:\n",
        "  rename = \"--replace_subject\"\n",
        "\n",
        "cpu = \"\"\n",
        "if use_cpu:\n",
        "  cpu = \"--force_cpu\"\n",
        "\n",
        "!python caption.py \\\n",
        "$cpu \\\n",
        "$rename \\\n",
        "--model $model_name \\\n",
        "--data_root \"$input_folder\" \\\n",
        "--Blip_location \"$model_storage\" \\\n",
        "--max_new_tokens $token_limit\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Extract Frames from video\n",
        "\n",
        "#@markdown Here we will use the folder input_vid \n",
        "!mkdir output/vid \n",
        "\n",
        "!python /scripts/extract_video_frames.py \\\n",
        "--vid_dir input \\\n",
        "--out_dir output/vid \\\n",
        "--format png \\\n",
        "--interval 10 "
      ],
      "metadata": {
        "id": "RDuBL4k8Avz-",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Move the images to the input folder for captioning\n",
        "!cp -r output/vid input"
      ],
      "metadata": {
        "id": "Uv8wAHSQAvrm",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Laion Downloader\n",
        "\n",
        "* --laion_dir: directory with laion parquet files, default is ./laion\n",
        "\n",
        "* --search_text: csv of words with AND logic, ex \\\"photo,man,dog\\\"\n",
        "\n",
        "* --out_dir: directory to download files to, ive defaulted this to inputs so they can be captioned \n",
        "\n",
        "* --log_dir: directory for logs, if ommitted will not log, logs may be large!\n",
        "\n",
        "* --column:column to search for matches, defaults is 'TEXT', but you could use 'URL' if you wanted\",\n",
        "\n",
        "* --limit: max number of matching images to download, warning: may be slightly imprecise due to concurrency and http errors, defaults is 100\n",
        "\n",
        "* --min_hw: min height AND width of image to download, default is 512\n",
        "  \n",
        "* --force: forces a full download of all images, even if no search is provided, USE CAUTION!\n",
        "\n",
        "* --parquet_skip: skips the first n parquet files on disk, useful to resume\n",
        "        \n",
        "* --verbose: additional logging of URL and TEXT \n",
        "        \n",
        "* --test: skips downloading, for checking filters, use with \"--verbose\"\n"
      ],
      "metadata": {
        "id": "wY2f2LkPGSVa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!python scripts/download_laion.py \\\n",
        "--laion_dir ./laion \\\n",
        "--search_text \"photo,man,dog\" \\\n",
        "#--out_dir input \\\n",
        "#--log_dir logs \\\n",
        "#--column TEXT \\\n",
        "#--limit 100 \\\n",
        "#--min_hw 512 \\\n",
        "#--force False \\\n",
        "#--parquet_skip 0 \\\n",
        "#--Verbose False \\\n",
        "#--test not \\\n"
      ],
      "metadata": {
        "id": "cxw60TTmEy2C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Here we can take our now captioned images and replace generic terms with our subjects\n",
        "\n",
        "* --find: will search for a word in this case man\n",
        "\n",
        "* --replace: will replace our found word with in this case bob smith\n",
        "\n",
        "* --append_only: this will allow us to add a tag at he end "
      ],
      "metadata": {
        "id": "EBdLelNpDjYc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!python scripts/filename_replace.py \\\n",
        "--img_dir output \\\n",
        "--find \"man\" \\\n",
        "--replace \"bob smith\""
      ],
      "metadata": {
        "id": "6Y1md3OHAvhw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now we can chose to create text files based on our file names, this is usefull for images with very long discriptions or tag list, windows has a limit of 256 characters, and files will not transfer correctly to a windows program if they are longer, moving these files in a zip is fine however and causes no issues\n"
      ],
      "metadata": {
        "id": "W0MspWmXJQuc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!python scripts/createtxtfromfilename.py"
      ],
      "metadata": {
        "id": "BpvenvyQJr9b"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Compress our images "
      ],
      "metadata": {
        "id": "boVkDsiWJ_-P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!python scripts/compress_img.py \\\n",
        "--img_dir output \\\n",
        "--out_dir output/compressed_images \\\n",
        "--max_mp 1.5 \n",
        "#--overwrite False \\\n",
        "#--Quality 95  \\\n",
        "#--noresize False \\\n",
        "#--delete \\"
      ],
      "metadata": {
        "id": "F6QYfylhKAII"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HBrWnu1C_lN9"
      },
      "source": [
        "## Download your DataSet from EveryDream/output\n",
        "\n",
        "If you're on a colab you can use the cell below to push your output to your Gdrive."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ldW2sDLcAVgz"
      },
      "outputs": [],
      "source": [
        "\n",
        "!mkdir /content/drive/MyDrive/Auto_Data_sets\n",
        "!cp -r output/ /content/drive/MyDrive/Auto_Data_sets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B-HFqbP4AVgz"
      },
      "source": [
        "## If not on colab/gdrive, the following will zip up your files for extraction\n",
        "\n",
        "You'll still need to use your runtime's own download feature to download the zip.\n",
        "\n",
        "![output zip](https://github.com/victorchall/EveryDream/blob/main/demo/output_zip.png?raw=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SVa80mrKAVg0"
      },
      "outputs": [],
      "source": [
        "import patoolib\n",
        "from google.colab import files\n",
        "import os\n",
        "\n",
        "Zip_Location = \"/content/drive/MyDrive/output\" #@param {type:\"string\"}\n",
        "#@markdown * this is the location containing your captioned images\n",
        "\n",
        "!mkdir output/zip\n",
        "!zip -r output/zip/output.zip \"$Zip_Location\"\n",
        "\n",
        "if os.path.exists(output/zip/output.zip):\n",
        "  files.download(output/zip/output.zip)\n",
        "  !rm output/zip/output.zip\n",
        "else:\n",
        "  print(\"Error: File not found at specified path.\")\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3.10.5 ('.venv': venv)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.5"
    },
    "vscode": {
      "interpreter": {
        "hash": "faf4a6abb601e3a9195ce3e9620411ceec233a951446de834cdf28542d2d93b4"
      }
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}