notebooks/3_embeddings_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "92727a9b",
   "metadata": {},
   "source": [
    "# Convert audio signals to pre-trained CNN embeddings\n",
    "\n",
    "The audio signals are being converted to pre-trained CNN embeddings, which will be used for classification tasks. This process involves extracting high-level features from the audio data using pre-trained convolutional neural network models.\n",
    "\n",
    "Three popular pre-trained CNN models for classifying audio events have been selected:\n",
    "- VGGish: https://github.com/tensorflow/models/tree/master/research/audioset/vggish\n",
    "- YAMNet: https://github.com/tensorflow/models/tree/master/research/audioset/yamnet\n",
    "- PANNs: https://github.com/qiuqiangkong/audioset_tagging_cnn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e77891e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-10 14:36:19.071527: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 AVX512F AVX512_VNNI FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "from IPython import display\n",
    "# import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random as python_random\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "from tensorflow.keras import regularizers\n",
    "\n",
    "from lazypredict.Supervised import LazyClassifier\n",
    "\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "\n",
    "yamnet_model = hub.load('https://tfhub.dev/google/yamnet/1') # if there's an \"SavedModel file does not exist at:\", delete that folder and rerun it\n",
    "vggish_model = hub.load('https://tfhub.dev/google/vggish/1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "de65538e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file</th>\n",
       "      <th>source</th>\n",
       "      <th>category</th>\n",
       "      <th>weight</th>\n",
       "      <th>fold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>../data/interim/GoogleAudioSet_unbalanced_list...</td>\n",
       "      <td>Google_city</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>../data/interim/GoogleAudioSet_unbalanced_list...</td>\n",
       "      <td>Google_city</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>../data/interim/GoogleAudioSet_unbalanced_list...</td>\n",
       "      <td>Google_nature</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>../data/interim/GoogleAudioSet_unbalanced_list...</td>\n",
       "      <td>Google_city</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>../data/interim/GoogleAudioSet_unbalanced_list...</td>\n",
       "      <td>Google_city</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13662</th>\n",
       "      <td>../data/interim/GoogleAudioSet_unbalanced_list...</td>\n",
       "      <td>Google_city</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13663</th>\n",
       "      <td>../data/interim/GoogleAudioSet_unbalanced_list...</td>\n",
       "      <td>Google_city</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13664</th>\n",
       "      <td>../data/interim/GoogleAudioSet_unbalanced_list...</td>\n",
       "      <td>Google_city</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13665</th>\n",
       "      <td>../data/interim/GoogleAudioSet_unbalanced_list...</td>\n",
       "      <td>Google_city</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13666</th>\n",
       "      <td>../data/interim/GoogleAudioSet_unbalanced_list...</td>\n",
       "      <td>Google_city</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13667 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    file         source  \\\n",
       "0      ../data/interim/GoogleAudioSet_unbalanced_list...    Google_city   \n",
       "1      ../data/interim/GoogleAudioSet_unbalanced_list...    Google_city   \n",
       "2      ../data/interim/GoogleAudioSet_unbalanced_list...  Google_nature   \n",
       "3      ../data/interim/GoogleAudioSet_unbalanced_list...    Google_city   \n",
       "4      ../data/interim/GoogleAudioSet_unbalanced_list...    Google_city   \n",
       "...                                                  ...            ...   \n",
       "13662  ../data/interim/GoogleAudioSet_unbalanced_list...    Google_city   \n",
       "13663  ../data/interim/GoogleAudioSet_unbalanced_list...    Google_city   \n",
       "13664  ../data/interim/GoogleAudioSet_unbalanced_list...    Google_city   \n",
       "13665  ../data/interim/GoogleAudioSet_unbalanced_list...    Google_city   \n",
       "13666  ../data/interim/GoogleAudioSet_unbalanced_list...    Google_city   \n",
       "\n",
       "       category  weight  fold  \n",
       "0             1       1     0  \n",
       "1             1       1     8  \n",
       "2             0       1     5  \n",
       "3             1       1     1  \n",
       "4             1       1     1  \n",
       "...         ...     ...   ...  \n",
       "13662         1       1     5  \n",
       "13663         1       1     3  \n",
       "13664         1       1     8  \n",
       "13665         1       1     8  \n",
       "13666         1       1     0  \n",
       "\n",
       "[13667 rows x 5 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all = pd.read_csv('../train_val_test_split/train_val_test_GoogleAudioSet.csv', index_col=0)\n",
    "df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "20a5f3d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_preprocessing(filename):\n",
    "    import pickle\n",
    "    \n",
    "    file = open(filename, 'rb')\n",
    "    output = pickle.load(file)\n",
    "#     output = pd.read_pickle(file)\n",
    "    wav_raw = output['y']\n",
    "    wav_bg = output['bg_y']\n",
    "    wav_fg = output['fg_y']\n",
    "    \n",
    "    df_indices_raw = output['df_indices']\n",
    "    df_indices_fg = output['df_indices_fg']\n",
    "    df_indices_bg = output['df_indices_bg']\n",
    "    \n",
    "    wt = output['wt']\n",
    "    mps_raw = output['mps']\n",
    "    mps_fg = output['mps_fg']\n",
    "    mps_bg = output['mps_bg']\n",
    "    mps_raw = mps_raw[:,wt<=100].reshape(-1) # exclude the temporal modulation frequency >100 Hz\n",
    "    mps_fg = mps_fg[:,wt<=100].reshape(-1)\n",
    "    mps_bg = mps_bg[:,wt<=100].reshape(-1)\n",
    "    \n",
    "    file.close()\n",
    "\n",
    "    # Extract YAMNet embeddings for each frame\n",
    "    scores, embedding_tensor_raw, spectrogram = yamnet_model(wav_raw)\n",
    "    embedding_tensor_raw = tf.reduce_mean(embedding_tensor_raw, axis=0).numpy()\n",
    "\n",
    "    scores, embedding_tensor_bg, spectrogram = yamnet_model(wav_bg)\n",
    "    embedding_tensor_bg = tf.reduce_mean(embedding_tensor_bg, axis=0).numpy()\n",
    "\n",
    "    scores, embedding_tensor_fg, spectrogram = yamnet_model(wav_fg)\n",
    "    embedding_tensor_fg = tf.reduce_mean(embedding_tensor_fg, axis=0).numpy()\n",
    "    \n",
    "    \n",
    "    # Extract VGGish embeddings for each frame\n",
    "    vggish_embedding_raw = tf.reduce_mean(vggish_model(wav_raw), axis=0).numpy()\n",
    "    vggish_embedding_bg = tf.reduce_mean(vggish_model(wav_bg), axis=0).numpy()\n",
    "    vggish_embedding_fg = tf.reduce_mean(vggish_model(wav_fg), axis=0).numpy()\n",
    "   \n",
    "    return embedding_tensor_raw, embedding_tensor_bg, embedding_tensor_fg, mps_raw, mps_bg, mps_fg, df_indices_raw, df_indices_bg, df_indices_fg, vggish_embedding_raw, vggish_embedding_bg, vggish_embedding_fg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16e31444",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import librosa\n",
    "import panns_inference\n",
    "from panns_inference import AudioTagging, SoundEventDetection, labels\n",
    "import pickle\n",
    "from librosa import resample\n",
    "import time\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "\n",
    "wav_raw_list = []\n",
    "wav_bg_list = []\n",
    "wav_fg_list = []\n",
    "for index, row in df_all.iterrows():\n",
    "    file = open(row['file'], 'rb')\n",
    "    output = pickle.load(file)\n",
    "    wav_raw_list.append(np.pad(output['y'], (0,160000-len(output['y'])),'mean'))\n",
    "    wav_bg_list.append(np.pad(output['bg_y'], (0,160000-len(output['bg_y'])),'mean'))\n",
    "    wav_fg_list.append(np.pad(output['fg_y'], (0,160000-len(output['fg_y'])),'mean'))\n",
    "    file.close()\n",
    "    \n",
    "wav_raw = resample(np.stack(wav_raw_list), orig_sr=16000, target_sr=32000, axis=1)\n",
    "del wav_raw_list\n",
    "wav_bg = resample(np.stack(wav_bg_list), orig_sr=16000, target_sr=32000, axis=1)\n",
    "del wav_bg_list\n",
    "wav_fg = resample(np.stack(wav_fg_list), orig_sr=16000, target_sr=32000, axis=1)\n",
    "del wav_fg_list\n",
    "    \n",
    "\n",
    "n_file = 0\n",
    "clipwise_output_raw_list = []\n",
    "clipwise_output_bg_list = []\n",
    "clipwise_output_fg_list = []\n",
    "embedding_raw_list = []\n",
    "embedding_bg_list = []\n",
    "embedding_fg_list = []\n",
    "file_step = 100\n",
    "\n",
    "at = AudioTagging(checkpoint_path=None, device='cuda')\n",
    "while n_file < len(df_all):\n",
    "    print(str(n_file))\n",
    "    (clipwise_output_raw, embedding_raw) = at.inference(wav_raw[n_file:min(n_file+file_step,len(df_all))])\n",
    "    (clipwise_output_bg, embedding_bg) = at.inference(wav_bg[n_file:min(n_file+file_step,len(df_all))])\n",
    "    (clipwise_output_fg, embedding_fg) = at.inference(wav_fg[n_file:min(n_file+file_step,len(df_all))])\n",
    "    n_file += file_step\n",
    "    \n",
    "    clipwise_output_raw_list.append(clipwise_output_raw)\n",
    "    clipwise_output_bg_list.append(clipwise_output_bg)\n",
    "    clipwise_output_fg_list.append(clipwise_output_fg)\n",
    "    embedding_raw_list.append(embedding_raw)\n",
    "    embedding_bg_list.append(embedding_bg)\n",
    "    embedding_fg_list.append(embedding_fg)\n",
    "\n",
    "    \n",
    "panns_clip_raw = np.concatenate(clipwise_output_raw_list, axis=0)\n",
    "del clipwise_output_raw_list\n",
    "\n",
    "panns_clip_bg = np.concatenate(clipwise_output_bg_list, axis=0)\n",
    "del clipwise_output_bg_list\n",
    "\n",
    "panns_clip_fg = np.concatenate(clipwise_output_fg_list, axis=0)\n",
    "del clipwise_output_fg_list\n",
    "\n",
    "panns_embedding_raw = np.concatenate(embedding_raw_list, axis=0)\n",
    "del embedding_raw_list\n",
    "\n",
    "panns_embedding_bg = np.concatenate(embedding_bg_list, axis=0)\n",
    "del embedding_bg_list\n",
    "\n",
    "panns_embedding_fg = np.concatenate(embedding_fg_list, axis=0)\n",
    "del embedding_fg_list\n",
    "\n",
    "\n",
    "\n",
    "print(time.time()-start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "25e3376a",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('../data/processed/20230304/panns_clip_raw.npy', panns_clip_raw)\n",
    "np.save('../data/processed/20230304/panns_clip_bg.npy', panns_clip_bg)\n",
    "np.save('../data/processed/20230304/panns_clip_fg.npy', panns_clip_fg)\n",
    "np.save('../data/processed/20230304/panns_embedding_raw.npy', panns_embedding_raw)\n",
    "np.save('../data/processed/20230304/panns_embedding_bg.npy', panns_embedding_bg)\n",
    "np.save('../data/processed/20230304/panns_embedding_fg.npy', panns_embedding_fg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0b907e47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seconds: 6113.771539211273\n"
     ]
    }
   ],
   "source": [
    "embedding_raw_list = []\n",
    "embedding_bg_list = []\n",
    "embedding_fg_list = []\n",
    "mps_raw_list = []\n",
    "mps_bg_list = []\n",
    "mps_fg_list = []\n",
    "indices_raw_list = []\n",
    "indices_bg_list = []\n",
    "indices_fg_list = []\n",
    "vgg_raw_list = []\n",
    "vgg_bg_list = []\n",
    "vgg_fg_list = []\n",
    "\n",
    "import time\n",
    "start_time = time.time()\n",
    "\n",
    "\n",
    "for index, row in df_all.iterrows():\n",
    "    embedding_tensor_raw, embedding_tensor_bg, embedding_tensor_fg, mps_raw, mps_bg, mps_fg, df_indices_raw, df_indices_bg, df_indices_fg, vggish_embedding_raw, vggish_embedding_bg, vggish_embedding_fg = data_preprocessing(row['file'])\n",
    "    \n",
    "    embedding_raw_list.append(embedding_tensor_raw)\n",
    "    embedding_bg_list.append(embedding_tensor_bg)\n",
    "    embedding_fg_list.append(embedding_tensor_fg)\n",
    "    mps_raw_list.append(mps_raw)\n",
    "    mps_bg_list.append(mps_bg)\n",
    "    mps_fg_list.append(mps_fg)\n",
    "    indices_raw_list.append(df_indices_raw)\n",
    "    indices_bg_list.append(df_indices_bg)\n",
    "    indices_fg_list.append(df_indices_fg)\n",
    "    vgg_raw_list.append(vggish_embedding_raw)\n",
    "    vgg_bg_list.append(vggish_embedding_bg)\n",
    "    vgg_fg_list.append(vggish_embedding_fg)\n",
    "\n",
    "    \n",
    "print('seconds: '+str(time.time()-start_time))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d1bda728",
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_raw_matrix = np.stack(embedding_raw_list, axis=0)\n",
    "embedding_bg_matrix = np.stack(embedding_bg_list, axis=0)\n",
    "embedding_fg_matrix = np.stack(embedding_fg_list, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "08770546",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_indices_raw = pd.concat(indices_raw_list, ignore_index=True)\n",
    "df_indices_bg = pd.concat(indices_bg_list, ignore_index=True)\n",
    "df_indices_fg = pd.concat(indices_fg_list, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "18e2aecc",
   "metadata": {},
   "outputs": [],
   "source": [
    "mps_raw_matrix = np.stack(mps_raw_list)\n",
    "mps_bg_matrix = np.stack(mps_bg_list)\n",
    "mps_fg_matrix = np.stack(mps_fg_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eb88ce4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "vgg_raw_matrix = np.stack(vgg_raw_list, axis=0)\n",
    "vgg_bg_matrix = np.stack(vgg_bg_list, axis=0)\n",
    "vgg_fg_matrix = np.stack(vgg_fg_list, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c9b29161",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('../data/processed/20230304/embedding_raw_matrix.npy', embedding_raw_matrix)\n",
    "np.save('../data/processed/20230304/embedding_bg_matrix.npy', embedding_bg_matrix)\n",
    "np.save('../data/processed/20230304/embedding_fg_matrix.npy', embedding_fg_matrix)\n",
    "\n",
    "np.save('../data/processed/20230304/vgg_raw_matrix.npy', vgg_raw_matrix)\n",
    "np.save('../data/processed/20230304/vgg_bg_matrix.npy', vgg_bg_matrix)\n",
    "np.save('../data/processed/20230304/vgg_fg_matrix.npy', vgg_fg_matrix)\n",
    "\n",
    "np.save('../data/processed/20230304/mps_raw_matrix.npy', mps_raw_matrix)\n",
    "np.save('../data/processed/20230304/mps_bg_matrix.npy', mps_bg_matrix)\n",
    "np.save('../data/processed/20230304/mps_fg_matrix.npy', mps_fg_matrix)\n",
    "\n",
    "df_indices_raw.to_csv('../data/processed/20230304/df_indices_raw.csv')\n",
    "df_indices_bg.to_csv('../data/processed/20230304/df_indices_bg.csv')\n",
    "df_indices_fg.to_csv('../data/processed/20230304/df_indices_fg.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e561ca42",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}