sign_language_model_maker.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install -q flatbuffers==2.0.0\n",
        "!pip install -q mediapipe==0.9.0\n",
        "!pip install -q mediapipe-model-maker"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ywvea0rRGlB6",
        "outputId": "f7ab31fe-6c62-4803-e0b9-f65cb92e4628"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m33.0/33.0 MB\u001b[0m \u001b[31m23.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.3/106.3 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.6/2.6 MB\u001b[0m \u001b[31m69.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m33.6/33.6 MB\u001b[0m \u001b[31m41.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m118.9/118.9 kB\u001b[0m \u001b[31m13.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.0/6.0 MB\u001b[0m \u001b[31m98.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m71.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m591.0/591.0 kB\u001b[0m \u001b[31m49.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m240.6/240.6 kB\u001b[0m \u001b[31m24.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m175.1/175.1 kB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.1/17.1 MB\u001b[0m \u001b[31m48.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for pyyaml (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "NUM_EXAMPLES = 5\n",
        "IMAGES_PATH = \"drive/Shareddrives/KPZ 2023 - rozpoznawanie języka migowego/data/prepared\"\n",
        "\n",
        "# Get the list of labels from the list of folder names.\n",
        "labels = []\n",
        "for i in os.listdir(IMAGES_PATH):\n",
        "  if os.path.isdir(os.path.join(IMAGES_PATH, i)):\n",
        "    labels.append(i)\n",
        "\n",
        "print(labels)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OwnuntWFG_BO",
        "outputId": "d2161208-e64b-46ae-e0cf-dc5bb87e2782"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['CZ', 'A', 'B', 'C', 'D', 'E', 'None', 'F', 'G', 'I', 'K', 'L', 'M', 'N', 'O', 'P', 'R', 'S', 'T', 'U', 'W', 'Y', 'Z', 'SZ', 'H']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# plt.close()\n",
        "\n",
        "# Show the images.\n",
        "for label in labels:\n",
        "  if label == 'None': continue\n",
        "\n",
        "  label_dir = os.path.join(IMAGES_PATH, label)\n",
        "  example_filenames = os.listdir(label_dir)[:NUM_EXAMPLES]\n",
        "  fig, axs = plt.subplots(1, NUM_EXAMPLES, figsize=(10,2))\n",
        "  for i in range(NUM_EXAMPLES):\n",
        "    axs[i].imshow(plt.imread(os.path.join(label_dir, example_filenames[i])))\n",
        "    axs[i].get_xaxis().set_visible(False)\n",
        "    axs[i].get_yaxis().set_visible(False)\n",
        "  fig.suptitle(f'Showing {NUM_EXAMPLES} examples for {label}')\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "2rxANRFVHmYY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Importing model maker library\n",
        "from mediapipe_model_maker import gesture_recognizer"
      ],
      "metadata": {
        "id": "QY4446oxINFN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "826e4c2a-dd70-4e98-f5be-23c8c7c198a9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/tensorflow_addons/utils/tfa_eol_msg.py:23: UserWarning: \n",
            "\n",
            "TensorFlow Addons (TFA) has ended development and introduction of new features.\n",
            "TFA has entered a minimal maintenance and release mode until a planned end of life in May 2024.\n",
            "Please modify downstream libraries to take dependencies from other repositories in our TensorFlow community (e.g. Keras, Keras-CV, and Keras-NLP). \n",
            "\n",
            "For more information see: https://github.com/tensorflow/addons/issues/2807 \n",
            "\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Loading the sign-language image archive...\")\n",
        "print(\"This takes a while\")\n",
        "data = gesture_recognizer.Dataset.from_folder(\n",
        "    dirname=IMAGES_PATH,\n",
        "    hparams=gesture_recognizer.HandDataPreprocessingParams(\n",
        "        shuffle=True,\n",
        "        min_detection_confidence=0.75\n",
        "    )\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hMZpIFqRIVYO",
        "outputId": "431714ce-82c8-4a23-e43e-038547c0f00d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading the sign-language image archive...\n",
            "This takes a while\n",
            "Downloading https://storage.googleapis.com/mediapipe-assets/gesture_embedder.tar.gz to /tmp/model_maker/gesture_recognizer/gesture_embedder\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Splitting the archive into training, validation and test dataset...\")\n",
        "train_data, rest_data = data.split(0.8)\n",
        "validation_data, test_data = rest_data.split(0.5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "33J39odzMYpR",
        "outputId": "d8181e45-5f8a-4455-b488-ff119521a40e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Splitting the archive into training, validation and test dataset...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Creating learning parameters\")\n",
        "hparams = gesture_recognizer.HParams(\n",
        "    export_dir=\"pjm_recognizer_model\",\n",
        "    batch_size=3,\n",
        "    epochs=15,\n",
        "    gamma=2\n",
        ")\n",
        "print(\"Creating model options\")\n",
        "model_options = gesture_recognizer.ModelOptions(\n",
        "    dropout_rate=0.05 # hidden layers\n",
        ")\n",
        "print(\"Creating gesture recognizer options\")\n",
        "options = gesture_recognizer.GestureRecognizerOptions(\n",
        "    hparams=hparams,\n",
        "    model_options=model_options\n",
        ")\n",
        "\n",
        "print(\"Creating gesture recognizer\")\n",
        "model = gesture_recognizer.GestureRecognizer.create(\n",
        "    train_data=train_data,\n",
        "    validation_data=validation_data,\n",
        "    options=options\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VLJDeHk8MePl",
        "outputId": "e4f7a715-12c5-4964-b204-97527f37499d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Creating learning parameters\n",
            "Creating model options\n",
            "Creating gesture recognizer options\n",
            "Creating gesture recognizer\n",
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " hand_embedding (InputLayer)  [(None, 128)]            0         \n",
            "                                                                 \n",
            " batch_normalization (BatchN  (None, 128)              512       \n",
            " ormalization)                                                   \n",
            "                                                                 \n",
            " re_lu (ReLU)                (None, 128)               0         \n",
            "                                                                 \n",
            " dropout (Dropout)           (None, 128)               0         \n",
            "                                                                 \n",
            " custom_gesture_recognizer_o  (None, 25)               3225      \n",
            " ut (Dense)                                                      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 3,737\n",
            "Trainable params: 3,481\n",
            "Non-trainable params: 256\n",
            "_________________________________________________________________\n",
            "None\n",
            "Epoch 1/15\n",
            "578/578 [==============================] - 6s 7ms/step - loss: 1.9207 - categorical_accuracy: 0.3622 - val_loss: 0.8820 - val_categorical_accuracy: 0.6590 - lr: 0.0010\n",
            "Epoch 2/15\n",
            "578/578 [==============================] - 5s 9ms/step - loss: 1.0737 - categorical_accuracy: 0.5606 - val_loss: 0.6281 - val_categorical_accuracy: 0.7419 - lr: 9.9000e-04\n",
            "Epoch 3/15\n",
            "578/578 [==============================] - 4s 6ms/step - loss: 0.8685 - categorical_accuracy: 0.6188 - val_loss: 0.5517 - val_categorical_accuracy: 0.7373 - lr: 9.8010e-04\n",
            "Epoch 4/15\n",
            "578/578 [==============================] - 4s 7ms/step - loss: 0.7744 - categorical_accuracy: 0.6586 - val_loss: 0.5148 - val_categorical_accuracy: 0.7558 - lr: 9.7030e-04\n",
            "Epoch 5/15\n",
            "578/578 [==============================] - 6s 10ms/step - loss: 0.6990 - categorical_accuracy: 0.6707 - val_loss: 0.4857 - val_categorical_accuracy: 0.7604 - lr: 9.6060e-04\n",
            "Epoch 6/15\n",
            "578/578 [==============================] - 4s 6ms/step - loss: 0.6379 - categorical_accuracy: 0.7065 - val_loss: 0.4750 - val_categorical_accuracy: 0.7650 - lr: 9.5099e-04\n",
            "Epoch 7/15\n",
            "578/578 [==============================] - 7s 11ms/step - loss: 0.6177 - categorical_accuracy: 0.7128 - val_loss: 0.4754 - val_categorical_accuracy: 0.7650 - lr: 9.4148e-04\n",
            "Epoch 8/15\n",
            "578/578 [==============================] - 4s 6ms/step - loss: 0.5771 - categorical_accuracy: 0.7249 - val_loss: 0.4535 - val_categorical_accuracy: 0.7742 - lr: 9.3207e-04\n",
            "Epoch 9/15\n",
            "578/578 [==============================] - 4s 6ms/step - loss: 0.5658 - categorical_accuracy: 0.7220 - val_loss: 0.4403 - val_categorical_accuracy: 0.7788 - lr: 9.2274e-04\n",
            "Epoch 10/15\n",
            "578/578 [==============================] - 6s 10ms/step - loss: 0.5405 - categorical_accuracy: 0.7382 - val_loss: 0.4414 - val_categorical_accuracy: 0.7788 - lr: 9.1352e-04\n",
            "Epoch 11/15\n",
            "578/578 [==============================] - 3s 6ms/step - loss: 0.5189 - categorical_accuracy: 0.7388 - val_loss: 0.4285 - val_categorical_accuracy: 0.7788 - lr: 9.0438e-04\n",
            "Epoch 12/15\n",
            "578/578 [==============================] - 6s 10ms/step - loss: 0.5070 - categorical_accuracy: 0.7457 - val_loss: 0.4245 - val_categorical_accuracy: 0.7834 - lr: 8.9534e-04\n",
            "Epoch 13/15\n",
            "578/578 [==============================] - 5s 8ms/step - loss: 0.4928 - categorical_accuracy: 0.7555 - val_loss: 0.4261 - val_categorical_accuracy: 0.7880 - lr: 8.8638e-04\n",
            "Epoch 14/15\n",
            "578/578 [==============================] - 4s 6ms/step - loss: 0.4745 - categorical_accuracy: 0.7468 - val_loss: 0.4172 - val_categorical_accuracy: 0.7880 - lr: 8.7752e-04\n",
            "Epoch 15/15\n",
            "578/578 [==============================] - 4s 6ms/step - loss: 0.4755 - categorical_accuracy: 0.7578 - val_loss: 0.4113 - val_categorical_accuracy: 0.7880 - lr: 8.6875e-04\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "loss, acc = model.evaluate(test_data, batch_size=3)\n",
        "print(f\"Test loss:{loss}, Test accuracy:{acc}\")\n",
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XzK3flUeNJeC",
        "outputId": "a120ccd2-bf0d-4487-f97f-3f33233d4b90"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "73/73 [==============================] - 2s 6ms/step - loss: 0.3467 - categorical_accuracy: 0.8111\n",
            "Test loss:0.3467472493648529, Test accuracy:0.8110598921775818\n",
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " hand_embedding (InputLayer)  [(None, 128)]            0         \n",
            "                                                                 \n",
            " batch_normalization (BatchN  (None, 128)              512       \n",
            " ormalization)                                                   \n",
            "                                                                 \n",
            " re_lu (ReLU)                (None, 128)               0         \n",
            "                                                                 \n",
            " dropout (Dropout)           (None, 128)               0         \n",
            "                                                                 \n",
            " custom_gesture_recognizer_o  (None, 25)               3225      \n",
            " ut (Dense)                                                      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 3,737\n",
            "Trainable params: 3,481\n",
            "Non-trainable params: 256\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.export_model()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TFdse5glp-gp",
        "outputId": "00ff663e-c4f7-4abd-a168-ebabaddbbfe0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading https://storage.googleapis.com/mediapipe-assets/gesture_embedder.tflite to /tmp/model_maker/gesture_recognizer/gesture_embedder.tflite\n",
            "Downloading https://storage.googleapis.com/mediapipe-assets/canned_gesture_classifier.tflite to /tmp/model_maker/gesture_recognizer/canned_gesture_classifier.tflite\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!mv pjm_recognizer_model/gesture_recognizer.task pjm_v3.task"
      ],
      "metadata": {
        "id": "HBJKiHNZqHZ8"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}