From 78d02961ee13a1717e4a01fa95a3c55a5d7d0521 Mon Sep 17 00:00:00 2001 From: TimAdams84 Date: Tue, 9 Jul 2024 10:37:14 +0200 Subject: [PATCH] fix: notebooks --- README.md | 9 -- notebooks/test.ipynb | 288 ------------------------------------------- 2 files changed, 297 deletions(-) delete mode 100644 notebooks/test.ipynb diff --git a/README.md b/README.md index 3ec9d29..e5d80c2 100644 --- a/README.md +++ b/README.md @@ -5,15 +5,6 @@ INDEX is an intelligent data steward toolbox that leverages Large Language Model embeddings for automated Data-Harmonization. -## Table of Contents -- [Introduction](#introduction) -- [Installation](#installation) - - [Local Development Server](#local-development-server) - - [Starting the backend](#starting-the-backend) - - [Starting the frontend](#starting-the-frontend) - - [Docker](#docker) -- [Configuration](#configuration) - ## Introduction INDEX uses vector embeddings from variable descriptions to suggest mappings for datasets based on their semantic similarity. Mappings are stored with their vector representations in a knowledge base, where they can be used for subsequent harmonisation tasks, potentially improving suggestions with each iteration. The tool is designed to be configurable and extendable, adapting for specific use-cases through customizable models and databases. diff --git a/notebooks/test.ipynb b/notebooks/test.ipynb deleted file mode 100644 index a482cb2..0000000 --- a/notebooks/test.ipynb +++ /dev/null @@ -1,288 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import json" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'title': {'text': 't-SNE Embeddings of Database Mappings'},\n", - " 'xaxis': {'title': {'text': 't-SNE Component 1'}},\n", - " 'yaxis': {'title': {'text': 't-SNE Component 2'}},\n", - " 'template': {'data': {'histogram2dcontour': [{'type': 'histogram2dcontour',\n", - " 'colorbar': {'outlinewidth': 0, 'ticks': ''},\n", - " 'colorscale': [[0.0, '#0d0887'],\n", - " [0.1111111111111111, '#46039f'],\n", - " [0.2222222222222222, '#7201a8'],\n", - " [0.3333333333333333, '#9c179e'],\n", - " [0.4444444444444444, '#bd3786'],\n", - " [0.5555555555555556, '#d8576b'],\n", - " [0.6666666666666666, '#ed7953'],\n", - " [0.7777777777777778, '#fb9f3a'],\n", - " [0.8888888888888888, '#fdca26'],\n", - " [1.0, '#f0f921']]}],\n", - " 'choropleth': [{'type': 'choropleth',\n", - " 'colorbar': {'outlinewidth': 0, 'ticks': ''}}],\n", - " 'histogram2d': [{'type': 'histogram2d',\n", - " 'colorbar': {'outlinewidth': 0, 'ticks': ''},\n", - " 'colorscale': [[0.0, '#0d0887'],\n", - " [0.1111111111111111, '#46039f'],\n", - " [0.2222222222222222, '#7201a8'],\n", - " [0.3333333333333333, '#9c179e'],\n", - " [0.4444444444444444, '#bd3786'],\n", - " [0.5555555555555556, '#d8576b'],\n", - " [0.6666666666666666, '#ed7953'],\n", - " [0.7777777777777778, '#fb9f3a'],\n", - " [0.8888888888888888, '#fdca26'],\n", - " [1.0, '#f0f921']]}],\n", - " 'heatmap': [{'type': 'heatmap',\n", - " 'colorbar': {'outlinewidth': 0, 'ticks': ''},\n", - " 'colorscale': [[0.0, '#0d0887'],\n", - " [0.1111111111111111, '#46039f'],\n", - " [0.2222222222222222, '#7201a8'],\n", - " [0.3333333333333333, '#9c179e'],\n", - " [0.4444444444444444, '#bd3786'],\n", - " [0.5555555555555556, '#d8576b'],\n", - " [0.6666666666666666, '#ed7953'],\n", - " [0.7777777777777778, '#fb9f3a'],\n", - " [0.8888888888888888, '#fdca26'],\n", - " [1.0, '#f0f921']]}],\n", - " 'heatmapgl': [{'type': 'heatmapgl',\n", - " 'colorbar': {'outlinewidth': 0, 'ticks': ''},\n", - " 'colorscale': [[0.0, '#0d0887'],\n", - " [0.1111111111111111, '#46039f'],\n", - " [0.2222222222222222, '#7201a8'],\n", - " [0.3333333333333333, '#9c179e'],\n", - " [0.4444444444444444, '#bd3786'],\n", - " [0.5555555555555556, '#d8576b'],\n", - " [0.6666666666666666, '#ed7953'],\n", - " [0.7777777777777778, '#fb9f3a'],\n", - " [0.8888888888888888, '#fdca26'],\n", - " [1.0, '#f0f921']]}],\n", - " 'contourcarpet': [{'type': 'contourcarpet',\n", - " 'colorbar': {'outlinewidth': 0, 'ticks': ''}}],\n", - " 'contour': [{'type': 'contour',\n", - " 'colorbar': {'outlinewidth': 0, 'ticks': ''},\n", - " 'colorscale': [[0.0, '#0d0887'],\n", - " [0.1111111111111111, '#46039f'],\n", - " [0.2222222222222222, '#7201a8'],\n", - " [0.3333333333333333, '#9c179e'],\n", - " [0.4444444444444444, '#bd3786'],\n", - " [0.5555555555555556, '#d8576b'],\n", - " [0.6666666666666666, '#ed7953'],\n", - " [0.7777777777777778, '#fb9f3a'],\n", - " [0.8888888888888888, '#fdca26'],\n", - " [1.0, '#f0f921']]}],\n", - " 'surface': [{'type': 'surface',\n", - " 'colorbar': {'outlinewidth': 0, 'ticks': ''},\n", - " 'colorscale': [[0.0, '#0d0887'],\n", - " [0.1111111111111111, '#46039f'],\n", - " [0.2222222222222222, '#7201a8'],\n", - " [0.3333333333333333, '#9c179e'],\n", - " [0.4444444444444444, '#bd3786'],\n", - " [0.5555555555555556, '#d8576b'],\n", - " [0.6666666666666666, '#ed7953'],\n", - " [0.7777777777777778, '#fb9f3a'],\n", - " [0.8888888888888888, '#fdca26'],\n", - " [1.0, '#f0f921']]}],\n", - " 'mesh3d': [{'type': 'mesh3d',\n", - " 'colorbar': {'outlinewidth': 0, 'ticks': ''}}],\n", - " 'scatter': [{'fillpattern': {'fillmode': 'overlay',\n", - " 'size': 10,\n", - " 'solidity': 0.2},\n", - " 'type': 'scatter'}],\n", - " 'parcoords': [{'type': 'parcoords',\n", - " 'line': {'colorbar': {'outlinewidth': 0, 'ticks': ''}}}],\n", - " 'scatterpolargl': [{'type': 'scatterpolargl',\n", - " 'marker': {'colorbar': {'outlinewidth': 0, 'ticks': ''}}}],\n", - " 'bar': [{'error_x': {'color': '#2a3f5f'},\n", - " 'error_y': {'color': '#2a3f5f'},\n", - " 'marker': {'line': {'color': '#E5ECF6', 'width': 0.5},\n", - " 'pattern': {'fillmode': 'overlay', 'size': 10, 'solidity': 0.2}},\n", - " 'type': 'bar'}],\n", - " 'scattergeo': [{'type': 'scattergeo',\n", - " 'marker': {'colorbar': {'outlinewidth': 0, 'ticks': ''}}}],\n", - " 'scatterpolar': [{'type': 'scatterpolar',\n", - " 'marker': {'colorbar': {'outlinewidth': 0, 'ticks': ''}}}],\n", - " 'histogram': [{'marker': {'pattern': {'fillmode': 'overlay',\n", - " 'size': 10,\n", - " 'solidity': 0.2}},\n", - " 'type': 'histogram'}],\n", - " 'scattergl': [{'type': 'scattergl',\n", - " 'marker': {'colorbar': {'outlinewidth': 0, 'ticks': ''}}}],\n", - " 'scatter3d': [{'type': 'scatter3d',\n", - " 'line': {'colorbar': {'outlinewidth': 0, 'ticks': ''}},\n", - " 'marker': {'colorbar': {'outlinewidth': 0, 'ticks': ''}}}],\n", - " 'scattermapbox': [{'type': 'scattermapbox',\n", - " 'marker': {'colorbar': {'outlinewidth': 0, 'ticks': ''}}}],\n", - " 'scatterternary': [{'type': 'scatterternary',\n", - " 'marker': {'colorbar': {'outlinewidth': 0, 'ticks': ''}}}],\n", - " 'scattercarpet': [{'type': 'scattercarpet',\n", - " 'marker': {'colorbar': {'outlinewidth': 0, 'ticks': ''}}}],\n", - " 'carpet': [{'aaxis': {'endlinecolor': '#2a3f5f',\n", - " 'gridcolor': 'white',\n", - " 'linecolor': 'white',\n", - " 'minorgridcolor': 'white',\n", - " 'startlinecolor': '#2a3f5f'},\n", - " 'baxis': {'endlinecolor': '#2a3f5f',\n", - " 'gridcolor': 'white',\n", - " 'linecolor': 'white',\n", - " 'minorgridcolor': 'white',\n", - " 'startlinecolor': '#2a3f5f'},\n", - " 'type': 'carpet'}],\n", - " 'table': [{'cells': {'fill': {'color': '#EBF0F8'},\n", - " 'line': {'color': 'white'}},\n", - " 'header': {'fill': {'color': '#C8D4E3'}, 'line': {'color': 'white'}},\n", - " 'type': 'table'}],\n", - " 'barpolar': [{'marker': {'line': {'color': '#E5ECF6', 'width': 0.5},\n", - " 'pattern': {'fillmode': 'overlay', 'size': 10, 'solidity': 0.2}},\n", - " 'type': 'barpolar'}],\n", - " 'pie': [{'automargin': True, 'type': 'pie'}]},\n", - " 'layout': {'autotypenumbers': 'strict',\n", - " 'colorway': ['#636efa',\n", - " '#EF553B',\n", - " '#00cc96',\n", - " '#ab63fa',\n", - " '#FFA15A',\n", - " '#19d3f3',\n", - " '#FF6692',\n", - " '#B6E880',\n", - " '#FF97FF',\n", - " '#FECB52'],\n", - " 'font': {'color': '#2a3f5f'},\n", - " 'hovermode': 'closest',\n", - " 'hoverlabel': {'align': 'left'},\n", - " 'paper_bgcolor': 'white',\n", - " 'plot_bgcolor': '#E5ECF6',\n", - " 'polar': {'bgcolor': '#E5ECF6',\n", - " 'angularaxis': {'gridcolor': 'white', 'linecolor': 'white', 'ticks': ''},\n", - " 'radialaxis': {'gridcolor': 'white', 'linecolor': 'white', 'ticks': ''}},\n", - " 'ternary': {'bgcolor': '#E5ECF6',\n", - " 'aaxis': {'gridcolor': 'white', 'linecolor': 'white', 'ticks': ''},\n", - " 'baxis': {'gridcolor': 'white', 'linecolor': 'white', 'ticks': ''},\n", - " 'caxis': {'gridcolor': 'white', 'linecolor': 'white', 'ticks': ''}},\n", - " 'coloraxis': {'colorbar': {'outlinewidth': 0, 'ticks': ''}},\n", - " 'colorscale': {'sequential': [[0.0, '#0d0887'],\n", - " [0.1111111111111111, '#46039f'],\n", - " [0.2222222222222222, '#7201a8'],\n", - " [0.3333333333333333, '#9c179e'],\n", - " [0.4444444444444444, '#bd3786'],\n", - " [0.5555555555555556, '#d8576b'],\n", - " [0.6666666666666666, '#ed7953'],\n", - " [0.7777777777777778, '#fb9f3a'],\n", - " [0.8888888888888888, '#fdca26'],\n", - " [1.0, '#f0f921']],\n", - " 'sequentialminus': [[0.0, '#0d0887'],\n", - " [0.1111111111111111, '#46039f'],\n", - " [0.2222222222222222, '#7201a8'],\n", - " [0.3333333333333333, '#9c179e'],\n", - " [0.4444444444444444, '#bd3786'],\n", - " [0.5555555555555556, '#d8576b'],\n", - " [0.6666666666666666, '#ed7953'],\n", - " [0.7777777777777778, '#fb9f3a'],\n", - " [0.8888888888888888, '#fdca26'],\n", - " [1.0, '#f0f921']],\n", - " 'diverging': [[0, '#8e0152'],\n", - " [0.1, '#c51b7d'],\n", - " [0.2, '#de77ae'],\n", - " [0.3, '#f1b6da'],\n", - " [0.4, '#fde0ef'],\n", - " [0.5, '#f7f7f7'],\n", - " [0.6, '#e6f5d0'],\n", - " [0.7, '#b8e186'],\n", - " [0.8, '#7fbc41'],\n", - " [0.9, '#4d9221'],\n", - " [1, '#276419']]},\n", - " 'xaxis': {'gridcolor': 'white',\n", - " 'linecolor': 'white',\n", - " 'ticks': '',\n", - " 'title': {'standoff': 15},\n", - " 'zerolinecolor': 'white',\n", - " 'automargin': True,\n", - " 'zerolinewidth': 2},\n", - " 'yaxis': {'gridcolor': 'white',\n", - " 'linecolor': 'white',\n", - " 'ticks': '',\n", - " 'title': {'standoff': 15},\n", - " 'zerolinecolor': 'white',\n", - " 'automargin': True,\n", - " 'zerolinewidth': 2},\n", - " 'scene': {'xaxis': {'backgroundcolor': '#E5ECF6',\n", - " 'gridcolor': 'white',\n", - " 'linecolor': 'white',\n", - " 'showbackground': True,\n", - " 'ticks': '',\n", - " 'zerolinecolor': 'white',\n", - " 'gridwidth': 2},\n", - " 'yaxis': {'backgroundcolor': '#E5ECF6',\n", - " 'gridcolor': 'white',\n", - " 'linecolor': 'white',\n", - " 'showbackground': True,\n", - " 'ticks': '',\n", - " 'zerolinecolor': 'white',\n", - " 'gridwidth': 2},\n", - " 'zaxis': {'backgroundcolor': '#E5ECF6',\n", - " 'gridcolor': 'white',\n", - " 'linecolor': 'white',\n", - " 'showbackground': True,\n", - " 'ticks': '',\n", - " 'zerolinecolor': 'white',\n", - " 'gridwidth': 2}},\n", - " 'shapedefaults': {'line': {'color': '#2a3f5f'}},\n", - " 'annotationdefaults': {'arrowcolor': '#2a3f5f',\n", - " 'arrowhead': 0,\n", - " 'arrowwidth': 1},\n", - " 'geo': {'bgcolor': 'white',\n", - " 'landcolor': '#E5ECF6',\n", - " 'subunitcolor': 'white',\n", - " 'showland': True,\n", - " 'showlakes': True,\n", - " 'lakecolor': 'white'},\n", - " 'title': {'x': 0.05},\n", - " 'mapbox': {'style': 'light'}}}}" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "obj = json.load(open(\"/home/ceb/Downloads/response_1720097172939.json\"))\n", - "obj[\"layout\"]" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}