Skip to content

WIP: Added notebook and utils for basic processing: undistortion and depth reprojection #1

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

WIP: Added notebook and utils for basic processing: undistortion and depth reprojection #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5e8bd79
Added notebook for basic processing: undistortion and depth reprojection
anastasiia-kornilova Jun 18, 2022
fb2722d
Added renaming to seconds format for images and fixed bug with saving…
anastasiia-kornilova Jun 20, 2022
f23eb2e
Added notebook to generate all needed for ORB from basically processe…
anastasiia-kornilova Jun 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 162 additions & 0 deletions Basic processing.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "6da669a4-f30b-4d57-b340-c283502ab356",
"metadata": {},
"source": [
"### Basic processing of Samsung data from Azure cameras\n",
"\n",
"It includes undistortion of color and depth images and reprojection of depth image to undistrorted color virtual camera.\n",
"\n",
"Parameters to set: `SEQUENCE_PATH` - path for recorded sequence, `OUTPUT_DIR` - where postprocessed data will be located.\n",
"\n",
"**Attention**: calibration file `calib_params.yaml` for every camera should be located in original camera data directories."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "10d3a173-38a2-4b7a-abb4-82f2a85dc42d",
"metadata": {},
"outputs": [],
"source": [
"import cv2\n",
"import imageio\n",
"import os\n",
"import shutil\n",
"\n",
"from tqdm import tqdm\n",
"\n",
"from utils.io_utils import filename_to_seconds\n",
"from utils.calib_io_utils import load_azure_params\n",
"from utils.cv_utils import undistort_image, reproject_depth"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "26e668be-ceda-4694-9514-a595d16f4cf6",
"metadata": {},
"outputs": [],
"source": [
"SEQUENCE_PATH = '/home/anasyasiia/Downloads/2022-04-16-12-15-47_round0/2022-04-16-12-15-47_round0/'\n",
"OUTPUT_DIR = 'output'"
]
},
{
"cell_type": "markdown",
"id": "2ccca8bf-0dcb-470f-a622-8dc60075de6c",
"metadata": {},
"source": [
"### List recorded cameras"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e4862016-0023-4752-818e-2cf4707e399f",
"metadata": {},
"outputs": [],
"source": [
"camera_names = os.listdir(SEQUENCE_PATH)\n",
"camera_names"
]
},
{
"cell_type": "markdown",
"id": "1b394bb3-aab5-4209-80d8-68fbb71f8153",
"metadata": {},
"source": [
"### Prepare infrastructure for postprocessed data"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3dc3e95d-26af-4aea-a2d5-b5489a6c5979",
"metadata": {},
"outputs": [],
"source": [
"dataset_basedir = os.path.basename(os.path.normpath(SEQUENCE_PATH))\n",
"postprocessed_data_dir = os.path.join(OUTPUT_DIR, dataset_basedir)\n",
"\n",
"if os.path.exists(postprocessed_data_dir):\n",
" shutil.rmtree(postprocessed_data_dir)\n",
" \n",
"os.mkdir(postprocessed_data_dir)\n",
"\n",
"for camera_name in camera_names:\n",
" os.mkdir(os.path.join(postprocessed_data_dir, camera_name))\n",
" os.mkdir(os.path.join(postprocessed_data_dir, camera_name, 'color'))\n",
" os.mkdir(os.path.join(postprocessed_data_dir, camera_name, 'depth'))"
]
},
{
"cell_type": "markdown",
"id": "b0556619-dc87-41d3-b6ba-fa78c5e41b91",
"metadata": {},
"source": [
"### Iterate over cameras and do undistortion and reprojection "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d7364fda-9a92-4608-a4e5-17e6888afa1f",
"metadata": {},
"outputs": [],
"source": [
"for camera_name in camera_names:\n",
" camera_data_path = os.path.join(SEQUENCE_PATH, camera_name)\n",
" azure_calib_path = os.path.join(camera_data_path, 'calib_params.json')\n",
"\n",
" color_camera, depth_camera, T_depth2color = load_azure_params(azure_calib_path)\n",
"\n",
" color_images_dir = os.path.join(camera_data_path, 'color')\n",
" color_image_names = os.listdir(color_images_dir)\n",
" color_image_names.sort()\n",
" postprocessed_color_dir = os.path.join(postprocessed_data_dir, camera_name, 'color')\n",
"\n",
" for color_image_name in tqdm(color_image_names):\n",
" image = cv2.imread(os.path.join(color_images_dir, color_image_name), -1)\n",
" undistorted_image = undistort_image(image, color_camera)\n",
" new_color_image_name = filename_to_seconds(color_image_name)\n",
" imageio.imwrite(os.path.join(postprocessed_color_dir, new_color_image_name), image)\n",
"\n",
" depth_images_dir = os.path.join(camera_data_path, 'depth')\n",
" depth_image_names = os.listdir(depth_images_dir)\n",
" depth_image_names.sort()\n",
" postprocessed_depth_dir = os.path.join(postprocessed_data_dir, camera_names, 'depth')\n",
"\n",
" for depth_image_name in tqdm(depth_image_names):\n",
" image = cv2.imread(os.path.join(depth_images_dir, depth_image_name), -1)\n",
" undistorted_image = undistort_image(image, depth_camera)\n",
" reprojected_image = reproject_depth(undistorted_image, depth_camera, color_camera, T_depth2color)\n",
" new_depth_image_name = filename_to_seconds(depth_image_name)\n",
" imageio.imwrite(os.path.join(postprocessed_depth_dir, new_depth_image_name), reprojected_image)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
196 changes: 196 additions & 0 deletions TUM conversion (one view).ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 96,
"id": "592949e2-2689-4e9c-90a2-ab08dede8d97",
"metadata": {},
"outputs": [],
"source": [
"ONE_CAMERA_SEQUENCE_PATH = 'output/2022-04-16-12-15-47_round0/3m/'"
]
},
{
"cell_type": "code",
"execution_count": 97,
"id": "888d1905-7246-4f64-a212-222554a4165f",
"metadata": {},
"outputs": [],
"source": [
"import cv2\n",
"import numpy as np\n",
"import os\n",
"from pathlib import Path"
]
},
{
"cell_type": "markdown",
"id": "4cb81d62-1c7c-43cf-be44-14ed4ce2c297",
"metadata": {},
"source": [
"### Assoсiate color and depth timestamps"
]
},
{
"cell_type": "code",
"execution_count": 88,
"id": "8ee3a09a-8851-4943-8055-f8028cf8155b",
"metadata": {},
"outputs": [],
"source": [
"def read_folder(path_to_folder: str):\n",
" \"\"\"\n",
" Reads all images' paths and timestamps from folder\n",
" :param path_to_folder: path to folder to read\n",
" :return: dictionary where keys are timestamps and values are path to image\n",
" \"\"\"\n",
" files = os.listdir(path_to_folder)\n",
" timestamps = [float(Path(file).stem) for file in files]\n",
" files = [os.path.join(os.path.basename(path_to_folder), x) for x in files]\n",
" timestamp_image_kvp = dict(zip(timestamps, files))\n",
" return timestamp_image_kvp\n",
"\n",
"\n",
"def associate(\n",
" color_images: dict, depth_images: dict, offset: float, max_difference: float\n",
") -> list:\n",
" \"\"\"\n",
" Associates color and depth images\n",
" :param color_images: (timestamp, path) KVP for color images\n",
" :param depth_images: (timestamp, path) KVP for depth images\n",
" :param offset: time offset added to the timestamps of the depth images\n",
" :param max_difference: maximally allowed time difference for matching entries\n",
" :return: best matches for color and depth images\n",
" \"\"\"\n",
" first_keys = np.asarray(list(color_images.keys()))\n",
" second_keys = np.asarray(list(depth_images.keys()))\n",
" best_matches = list()\n",
" for timestamp in first_keys:\n",
" best_match = second_keys[np.argmin(np.abs(second_keys + offset - timestamp))]\n",
" if abs(best_match + offset - timestamp) < max_difference:\n",
" best_matches.append((timestamp, best_match))\n",
" return sorted(best_matches)\n"
]
},
{
"cell_type": "code",
"execution_count": 102,
"id": "f8c9873f-ddd7-4db4-aaf3-564f0254d765",
"metadata": {},
"outputs": [],
"source": [
"def reformat_names_to_seconds(path):\n",
" names = os.listdir(path)\n",
" names.sort()\n",
"\n",
" for name in names:\n",
" new_name = name[:6] + '.' + name[6:]\n",
"\n",
" shutil.move(os.path.join(path, name), \n",
" os.path.join(path, new_name))\n",
" \n",
"color_path = 'output/2022-04-16-12-15-47_round0/3m/color/'\n",
"depth_path = 'output/2022-04-16-12-15-47_round0/3m/depth/'\n",
"# reformat_names_to_seconds(color_path)\n",
"reformat_names_to_seconds(depth_path)"
]
},
{
"cell_type": "code",
"execution_count": 103,
"id": "42290078-32fc-4063-bc4f-93675323205e",
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "could not convert string to float: '.ipynb.'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-103-e3de070124cc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mfirst_list_M\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread_folder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mONE_CAMERA_SEQUENCE_PATH\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'color'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0msecond_list_M\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mread_folder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mONE_CAMERA_SEQUENCE_PATH\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'depth'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mmatches_M\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0massociate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfirst_list_M\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecond_list_M\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1e-3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-88-3409ddbda62a>\u001b[0m in \u001b[0;36mread_folder\u001b[0;34m(path_to_folder)\u001b[0m\n\u001b[1;32m 6\u001b[0m \"\"\"\n\u001b[1;32m 7\u001b[0m \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_to_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mtimestamps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mPath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstem\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfile\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasename\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_to_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0mtimestamp_image_kvp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimestamps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-88-3409ddbda62a>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 6\u001b[0m \"\"\"\n\u001b[1;32m 7\u001b[0m \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_to_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mtimestamps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mPath\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstem\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfile\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 9\u001b[0m \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasename\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_to_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0mtimestamp_image_kvp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimestamps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mValueError\u001b[0m: could not convert string to float: '.ipynb.'"
]
}
],
"source": [
"first_list_M = read_folder(os.path.join(ONE_CAMERA_SEQUENCE_PATH, 'color'))\n",
"second_list_M = read_folder(os.path.join(ONE_CAMERA_SEQUENCE_PATH, 'depth'))\n",
"\n",
"matches_M = associate(first_list_M, second_list_M, 0, 1e-3)\n",
"\n",
"with open(os.path.join(ONE_CAMERA_SEQUENCE_PATH, 'associations.txt'), 'w') as f:\n",
" for (a, b) in matches_M:\n",
" f.write('{0} {1} {2} {3}\\n'.format(\n",
" a,\n",
" first_list_M[a],\n",
" b,\n",
" second_list_M[b]\n",
" ))"
]
},
{
"cell_type": "markdown",
"id": "c6e53f2e-ec3f-4970-bc37-8a6a0262b20f",
"metadata": {},
"source": [
"### Create ORB config"
]
},
{
"cell_type": "code",
"execution_count": 100,
"id": "f1fc972a-96f8-489c-903e-6855bb4dbe6a",
"metadata": {},
"outputs": [],
"source": [
"import shutil\n",
"\n",
"K_undist = np.load(os.path.join(ONE_CAMERA_SEQUENCE_PATH, 'calib_params.npy'))\n",
"\n",
"new_config_path = os.path.join(ONE_CAMERA_SEQUENCE_PATH, 'ORB_one_view_config.yaml')\n",
"shutil.copyfile('configs/ORB_one_view_config.yaml', new_config_path)\n",
"\n",
"with open(new_config_path, 'r') as f:\n",
" lines[10] = 'Camera1.fx: {}\\n'.format(K_undist[0, 0])\n",
" lines[11] = 'Camera1.fy: {}\\n'.format(K_undist[1, 1])\n",
" lines[12] = 'Camera1.cx: {}\\n'.format(K_undist[0, 2])\n",
" lines[13] = 'Camera1.cy: {}\\n'.format(K_undist[1, 2])\n",
"\n",
"with open(new_config_path, 'w') as f:\n",
" f.writelines(lines)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2fdc4b71-7da4-456d-912b-dbc4c48048a5",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading