From 022727cb9148c3e8d36ad87595d54801c71d2827 Mon Sep 17 00:00:00 2001 From: Lei Wang Date: Fri, 27 Sep 2024 17:25:23 +0800 Subject: [PATCH] support graph schema and data source api in coordinator Signed-off-by: Lei Wang --- .../controllers/data_source_controller.py | 20 +- .../server/controllers/graph_controller.py | 8 +- .../flex/server/controllers/job_controller.py | 20 +- scripts/controller.py | 205 +++- scripts/gart_cli.py | 16 +- vegito/test/schema/ldbc-data-source.json | 1092 +++++++++++++++++ vegito/test/schema/ldbc-graph-schema.json | 722 +++++++++++ 7 files changed, 2058 insertions(+), 25 deletions(-) create mode 100644 vegito/test/schema/ldbc-data-source.json create mode 100644 vegito/test/schema/ldbc-graph-schema.json diff --git a/coordinator/flex/server/controllers/data_source_controller.py b/coordinator/flex/server/controllers/data_source_controller.py index 47a2e8e..b02473d 100644 --- a/coordinator/flex/server/controllers/data_source_controller.py +++ b/coordinator/flex/server/controllers/data_source_controller.py @@ -7,6 +7,9 @@ from flex.server.models.schema_mapping import SchemaMapping # noqa: E501 from flex.server import util +import os +import requests +import json def bind_datasource_in_batch(graph_id, schema_mapping): # noqa: E501 """bind_datasource_in_batch @@ -20,10 +23,19 @@ def bind_datasource_in_batch(graph_id, schema_mapping): # noqa: E501 :rtype: Union[str, Tuple[str, int], Tuple[str, int, Dict[str, str]] """ - if connexion.request.is_json: - schema_mapping = SchemaMapping.from_dict(connexion.request.get_json()) # noqa: E501 - return 'do some magic!' - + gart_controller_server = os.getenv("GART_CONTROLLER_SERVER", "127.0.0.1:8080") + if not gart_controller_server.startswith(("http://", "https://")): + gart_controller_server = f"http://{gart_controller_server}" + + if not isinstance(schema_mapping, dict): + schema_mapping = json.loads(schema_mapping) + + response = requests.post( + f"{gart_controller_server}/submit-data-source", + headers={"Content-Type": "application/json"}, + data=json.dumps({"schema": json.dumps(schema_mapping)}), + ) + return (response.text, response.status_code) def get_datasource_by_id(graph_id): # noqa: E501 """get_datasource_by_id diff --git a/coordinator/flex/server/controllers/graph_controller.py b/coordinator/flex/server/controllers/graph_controller.py index 0340e7a..7db7030 100644 --- a/coordinator/flex/server/controllers/graph_controller.py +++ b/coordinator/flex/server/controllers/graph_controller.py @@ -211,14 +211,14 @@ def create_graph(create_graph_request): # noqa: E501 result_dict["graph_id"] = create_graph_request["name"] global GRAPH_ID GRAPH_ID = result_dict["graph_id"] - create_graph_request = create_graph_request["schema"] - create_graph_request_yaml = yaml.dump(create_graph_request) gart_controller_server = os.getenv("GART_CONTROLLER_SERVER", "127.0.0.1:8080") if not gart_controller_server.startswith(("http://", "https://")): gart_controller_server = f"http://{gart_controller_server}" + response = requests.post( - f"{gart_controller_server}/submit-config", - data={"schema": create_graph_request_yaml}, + f"{gart_controller_server}/submit-graph-schema", + headers={"Content-Type": "application/json"}, + data=json.dumps({"schema": json.dumps(create_graph_request)}), ) return (CreateGraphResponse.from_dict(result_dict), response.status_code) diff --git a/coordinator/flex/server/controllers/job_controller.py b/coordinator/flex/server/controllers/job_controller.py index 092879a..8ddcb2a 100644 --- a/coordinator/flex/server/controllers/job_controller.py +++ b/coordinator/flex/server/controllers/job_controller.py @@ -14,6 +14,7 @@ import os import etcd3 from urllib.parse import urlparse +import json RUNNING = None @@ -169,6 +170,19 @@ def submit_dataloading_job(graph_id, dataloading_job_config): # noqa: E501 :rtype: Union[CreateDataloadingJobResponse, Tuple[CreateDataloadingJobResponse, int], Tuple[CreateDataloadingJobResponse, int, Dict[str, str]] """ - if connexion.request.is_json: - dataloading_job_config = DataloadingJobConfig.from_dict(connexion.request.get_json()) # noqa: E501 - return 'do some magic!' + gart_controller_server = os.getenv("GART_CONTROLLER_SERVER", "127.0.0.1:8080") + if not gart_controller_server.startswith(("http://", "https://")): + gart_controller_server = f"http://{gart_controller_server}" + + if not isinstance(dataloading_job_config, dict): + dataloading_job_config = json.loads(dataloading_job_config) + + response = requests.post( + f"{gart_controller_server}/submit-data-loading", + headers={"Content-Type": "application/json"}, + data=json.dumps({"schema": json.dumps(dataloading_job_config)}), + ) + + result_dict = {} + result_dict["job_id"] = "0" + return (CreateDataloadingJobResponse.from_dict(result_dict), response.status_code) diff --git a/scripts/controller.py b/scripts/controller.py index 1a735df..3ca0d59 100755 --- a/scripts/controller.py +++ b/scripts/controller.py @@ -11,6 +11,7 @@ import socket import json from datetime import datetime, timezone +import yaml app = Flask(__name__) port = int(os.getenv("CONTROLLER_FLASK_PORT", 5000)) @@ -29,7 +30,7 @@ def submit_config(): except Exception as e: return jsonify({"error": str(e)}), 400 elif "schema" in request.form: - content = request.form["schema"].encode('utf-8') + content = request.form["schema"].encode("utf-8") else: return jsonify({"error": "No file part or config string in the request"}), 400 @@ -60,7 +61,7 @@ def submit_pgql_config(): except Exception as e: return jsonify({"error": str(e)}), 400 elif "schema" in request.form: - content = request.form["schema"].encode('utf-8') + content = request.form["schema"].encode("utf-8") else: return jsonify({"error": "No file part or config string in the request"}), 400 @@ -90,7 +91,186 @@ def submit_pgql_config(): except Exception as e: time.sleep(5) return "PGQL config submitted", 200 - + + +@app.route("/submit-graph-schema", methods=["POST"]) +def submit_graph_schema(): + graph_schema = request.json.get("schema").encode("utf-8") + # graph_schema = request.form["schema"].encode('utf-8') + etcd_server = os.getenv("ETCD_SERVICE", "etcd") + if not etcd_server.startswith(("http://", "https://")): + etcd_server = f"http://{etcd_server}" + etcd_prefix = os.getenv("ETCD_PREFIX", "gart_meta_") + etcd_host = etcd_server.split("://")[1].split(":")[0] + etcd_port = etcd_server.split(":")[2] + etcd_client = etcd3.client(host=etcd_host, port=etcd_port) + with open("/tmp/graph_schema.json", "wb") as f: + f.write(graph_schema) + try: + etcd_client.put(etcd_prefix + "gart_graph_schema_json", graph_schema) + return "Graph schema submitted", 200 + except Exception as e: + return "Failed to submit graph schema: " + str(e), 500 + + +@app.route("/submit-data-source", methods=["POST"]) +def submit_data_source(): + data_source_config = request.json.get("schema").encode("utf-8") + + etcd_server = os.getenv("ETCD_SERVICE", "etcd") + if not etcd_server.startswith(("http://", "https://")): + etcd_server = f"http://{etcd_server}" + etcd_prefix = os.getenv("ETCD_PREFIX", "gart_meta_") + etcd_host = etcd_server.split("://")[1].split(":")[0] + etcd_port = etcd_server.split(":")[2] + etcd_client = etcd3.client(host=etcd_host, port=etcd_port) + + try: + etcd_client.put(etcd_prefix + "gart_data_source_json", data_source_config) + return "Data source submitted", 200 + except Exception as e: + return "Failed to submit data source: " + str(e), 500 + + +@app.route("/submit-data-loading", methods=["POST"]) +def submit_data_loading(): + etcd_server = os.getenv("ETCD_SERVICE", "etcd") + if not etcd_server.startswith(("http://", "https://")): + etcd_server = f"http://{etcd_server}" + etcd_prefix = os.getenv("ETCD_PREFIX", "gart_meta_") + etcd_host = etcd_server.split("://")[1].split(":")[0] + etcd_port = etcd_server.split(":")[2] + etcd_client = etcd3.client(host=etcd_host, port=etcd_port) + try: + graph_schema, _ = etcd_client.get(etcd_prefix + "gart_graph_schema_json") + except Exception as e: + return "Failed to get graph schema: " + str(e), 500 + + try: + data_source_config, _ = etcd_client.get(etcd_prefix + "gart_data_source_json") + except Exception as e: + return "Failed to get data source: " + str(e), 500 + + graph_schema = json.loads(graph_schema.decode("utf-8")) + data_source_config = json.loads(data_source_config.decode("utf-8")) + result_dict = {} + result_dict["graph"] = graph_schema["name"] + graph_schema = graph_schema["schema"] + result_dict["loadingConfig"] = {} + result_dict["loadingConfig"]["dataSource"] = "rdbms" + result_dict["loadingConfig"]["method"] = "append" + result_dict["loadingConfig"]["enableRowStore"] = False + db_name = os.getenv("DB_NAME", "rdbms") + result_dict["loadingConfig"]["database"] = db_name + + vertex_mappings_dict = {} + vertex_types_list = [] + vertex_types_info = graph_schema["vertex_types"] + for vertex_id in range(len(vertex_types_info)): + vertex_type_element = {} + vertex_type_element["type_name"] = vertex_types_info[vertex_id]["type_name"] + mappings_list = [] + for table_id in range(len(vertex_types_info)): + if ( + vertex_type_element["type_name"] + == data_source_config["vertex_mappings"][table_id]["type_name"] + ): + vertex_type_element["dataSourceName"] = data_source_config[ + "vertex_mappings" + ][table_id]["inputs"][0] + pk_prop_name = vertex_types_info[vertex_id]["primary_keys"][0] + column_mappings = data_source_config["vertex_mappings"][table_id][ + "column_mappings" + ] + for column_id in range(len(column_mappings)): + mappings_element = {} + mappings_element["property"] = column_mappings[column_id][ + "property" + ] + mappings_element["dataField"] = {} + mappings_element["dataField"]["name"] = column_mappings[column_id][ + "column" + ]["name"] + mappings_list.append(mappings_element) + if pk_prop_name == column_mappings[column_id]["property"]: + vertex_type_element["idFieldName"] = column_mappings[column_id][ + "column" + ]["name"] + break + vertex_type_element["mappings"] = mappings_list + vertex_types_list.append(vertex_type_element) + + vertex_mappings_dict["vertex_types"] = vertex_types_list + + edge_mappings_dict = {} + edge_types_list = [] + edge_types_info = graph_schema["edge_types"] + for edge_id in range(len(edge_types_info)): + edge_type_element = {} + edge_type_element["type_pair"] = {} + edge_type_element["type_pair"]["edge"] = edge_types_info[edge_id]["type_name"] + edge_type_element["type_pair"]["source_vertex"] = edge_types_info[edge_id][ + "vertex_type_pair_relations" + ][0]["source_vertex"] + edge_type_element["type_pair"]["destination_vertex"] = edge_types_info[edge_id][ + "vertex_type_pair_relations" + ][0]["destination_vertex"] + for table_id in range(len(edge_types_info)): + if ( + edge_type_element["type_pair"]["edge"] + == data_source_config["edge_mappings"][table_id]["type_triplet"]["edge"] + ): + edge_type_element["dataSourceName"] = data_source_config[ + "edge_mappings" + ][table_id]["inputs"][0] + edge_type_element["sourceVertexMappings"] = [ + { + "dataField": { + "name": data_source_config["edge_mappings"][table_id][ + "source_vertex_mappings" + ][0]["column"]["name"] + } + } + ] + edge_type_element["destinationVertexMappings"] = [ + { + "dataField": { + "name": data_source_config["edge_mappings"][table_id][ + "destination_vertex_mappings" + ][0]["column"]["name"] + } + } + ] + data_field_mappings_list = [] + column_mappings = data_source_config["edge_mappings"][table_id][ + "column_mappings" + ] + for column_id in range(len(column_mappings)): + mappings_element = {} + mappings_element["property"] = column_mappings[column_id][ + "property" + ] + mappings_element["dataField"] = {} + mappings_element["dataField"]["name"] = column_mappings[column_id][ + "column" + ]["name"] + data_field_mappings_list.append(mappings_element) + edge_type_element["dataFieldMappings"] = data_field_mappings_list + break + edge_types_list.append(edge_type_element) + + edge_mappings_dict["edge_types"] = edge_types_list + + result_dict["vertexMappings"] = vertex_mappings_dict + result_dict["edgeMappings"] = edge_mappings_dict + + result_dict_str = yaml.dump(result_dict) + + try: + etcd_client.put(etcd_prefix + "gart_rg_mapping_yaml", result_dict_str) + return "Data loading config submitted", 200 + except Exception as e: + return "Failed to submit data loading config: " + str(e), 500 @app.route("/control/pause", methods=["POST"]) @@ -155,13 +335,22 @@ def get_read_epoch_by_timestamp(): unix_time = int(dt.timestamp()) epoch_unix_time_pairs = get_all_available_read_epochs_internal()[1] # iterate through the list of epoch_unix_time pairs from end to start - for epoch, unix_time_epoch, num_vertices, num_edges in reversed(epoch_unix_time_pairs): + for epoch, unix_time_epoch, num_vertices, num_edges in reversed( + epoch_unix_time_pairs + ): if unix_time_epoch <= unix_time: converted_time = datetime.fromtimestamp(unix_time_epoch) # convert time into local time zone - converted_time = converted_time.replace(tzinfo=timezone.utc).astimezone(tz=None) + converted_time = converted_time.replace(tzinfo=timezone.utc).astimezone( + tz=None + ) formatted_time = converted_time.strftime("%Y-%m-%d %H:%M:%S") - result = {"version_id": str(epoch), "creation_time": formatted_time, "num_vertices": num_vertices, "num_edges": num_edges} + result = { + "version_id": str(epoch), + "creation_time": formatted_time, + "num_vertices": num_vertices, + "num_edges": num_edges, + } return json.dumps(result), 200 return "No read epoch found", 200 @@ -385,7 +574,9 @@ def get_all_available_read_epochs_internal(): converted_time = converted_time.replace(tzinfo=timezone.utc).astimezone(tz=None) formatted_time = converted_time.strftime("%Y-%m-%d %H:%M:%S") available_epochs.append([epoch, formatted_time, num_vertices, num_edges]) - available_epochs_internal.append([epoch, latest_timestamp, num_vertices, num_edges]) + available_epochs_internal.append( + [epoch, latest_timestamp, num_vertices, num_edges] + ) return [available_epochs, available_epochs_internal] diff --git a/scripts/gart_cli.py b/scripts/gart_cli.py index 42572c3..4768abd 100755 --- a/scripts/gart_cli.py +++ b/scripts/gart_cli.py @@ -112,7 +112,8 @@ def get_version_by_timestamp(ctx, timestamp): return format_timestamp = timestamp.replace(" ", "%20") response = requests.get( - f"{endpoint}/api/v1/graph/{GRAPH_ID}/version/{format_timestamp}") + f"{endpoint}/api/v1/graph/{GRAPH_ID}/version/{format_timestamp}" + ) click.echo(f"Version at {timestamp}: {response.text}") @@ -133,13 +134,13 @@ def submit_config(ctx, config_path): payload = { "name": graph_name, "description": graph_name, - "schema": yaml_content + "schema": yaml_content, } try: response = requests.post( f"{endpoint}/api/v1/graph/yaml", headers={"Content-Type": "application/json"}, - data=json.dumps(payload) + data=json.dumps(payload), ) response.raise_for_status() click.echo(f"Success: Server responded with {response.status_code} status") @@ -163,7 +164,7 @@ def submit_pgql_config(ctx, config_path): with open(config_path, "r") as file: pgql_content = file.read() - match = re.search(r'CREATE PROPERTY GRAPH (\w+)', pgql_content) + match = re.search(r"CREATE PROPERTY GRAPH (\w+)", pgql_content) if match: graph_name = match.group(1) else: @@ -172,13 +173,13 @@ def submit_pgql_config(ctx, config_path): payload = { "name": graph_name, "description": graph_name, - "schema": pgql_content + "schema": pgql_content, } try: response = requests.post( f"{endpoint}/api/v1/graph/pgql", headers={"Content-Type": "application/json"}, - data=json.dumps(payload) + data=json.dumps(payload), ) response.raise_for_status() click.echo(f"Success: Server responded with {response.status_code} status") @@ -228,7 +229,8 @@ def change_graph_version_gie(ctx, graph_version): return response = requests.post( - f"{endpoint}/api/v1/graph/{GRAPH_ID}/version", data={"read_epoch": graph_version} + f"{endpoint}/api/v1/graph/{GRAPH_ID}/version", + data={"read_epoch": graph_version}, ) click.echo(f"Changed graph version to {graph_version}: {response.text}") diff --git a/vegito/test/schema/ldbc-data-source.json b/vegito/test/schema/ldbc-data-source.json new file mode 100644 index 0000000..68b0448 --- /dev/null +++ b/vegito/test/schema/ldbc-data-source.json @@ -0,0 +1,1092 @@ +{ + "vertex_mappings": [ + { + "type_name": "organisation", + "inputs": [ + "organisation" + ], + "column_mappings": [ + { + "column": { + "index": 0, + "name": "org_id" + }, + "property": "org_id" + }, + { + "column": { + "index": 1, + "name": "org_type" + }, + "property": "org_type" + }, + { + "column": { + "index": 2, + "name": "org_name" + }, + "property": "org_name" + }, + { + "column": { + "index": 3, + "name": "org_url" + }, + "property": "org_url" + } + ] + }, + { + "type_name": "place", + "inputs": [ + "place" + ], + "column_mappings": [ + { + "column": { + "index": 0, + "name": "pla_id" + }, + "property": "pla_id" + }, + { + "column": { + "index": 1, + "name": "pla_name" + }, + "property": "pla_name" + }, + { + "column": { + "index": 2, + "name": "pla_url" + }, + "property": "pla_url" + }, + { + "column": { + "index": 3, + "name": "pla_type" + }, + "property": "pla_type" + } + ] + }, + { + "type_name": "tag", + "inputs": [ + "tag" + ], + "column_mappings": [ + { + "column": { + "index": 0, + "name": "tag_id" + }, + "property": "tag_id" + }, + { + "column": { + "index": 1, + "name": "tag_name" + }, + "property": "tag_name" + }, + { + "column": { + "index": 2, + "name": "tag_url" + }, + "property": "tag_url" + } + ] + }, + { + "type_name": "tagclass", + "inputs": [ + "tagclass" + ], + "column_mappings": [ + { + "column": { + "index": 0, + "name": "tagc_id" + }, + "property": "tagc_id" + }, + { + "column": { + "index": 1, + "name": "tagc_name" + }, + "property": "tagc_name" + }, + { + "column": { + "index": 2, + "name": "tagc_url" + }, + "property": "tagc_url" + } + ] + }, + { + "type_name": "person", + "inputs": [ + "person" + ], + "column_mappings": [ + { + "column": { + "index": 0, + "name": "p_id" + }, + "property": "p_id" + }, + { + "column": { + "index": 1, + "name": "p_first_name" + }, + "property": "p_first_name" + }, + { + "column": { + "index": 2, + "name": "p_last_name" + }, + "property": "p_last_name" + }, + { + "column": { + "index": 3, + "name": "p_gender" + }, + "property": "p_gender" + }, + { + "column": { + "index": 4, + "name": "p_birthday" + }, + "property": "p_birthday" + }, + { + "column": { + "index": 5, + "name": "p_creation_date" + }, + "property": "p_creation_date" + }, + { + "column": { + "index": 6, + "name": "p_location_ip" + }, + "property": "p_location_ip" + }, + { + "column": { + "index": 7, + "name": "p_browser_used" + }, + "property": "p_browser_used" + } + ] + }, + { + "type_name": "comment", + "inputs": [ + "comment" + ], + "column_mappings": [ + { + "column": { + "index": 0, + "name": "co_id" + }, + "property": "co_id" + }, + { + "column": { + "index": 1, + "name": "co_creation_date" + }, + "property": "co_creation_date" + }, + { + "column": { + "index": 2, + "name": "co_location_ip" + }, + "property": "co_location_ip" + }, + { + "column": { + "index": 3, + "name": "co_browser_used" + }, + "property": "co_browser_used" + }, + { + "column": { + "index": 4, + "name": "co_content" + }, + "property": "co_content" + }, + { + "column": { + "index": 5, + "name": "co_length" + }, + "property": "co_length" + } + ] + }, + { + "type_name": "post", + "inputs": [ + "post" + ], + "column_mappings": [ + { + "column": { + "index": 0, + "name": "po_id" + }, + "property": "po_id" + }, + { + "column": { + "index": 1, + "name": "po_image_file" + }, + "property": "po_image_file" + }, + { + "column": { + "index": 2, + "name": "po_creation_date" + }, + "property": "po_creation_date" + }, + { + "column": { + "index": 3, + "name": "po_location_ip" + }, + "property": "po_location_ip" + }, + { + "column": { + "index": 4, + "name": "po_browser_used" + }, + "property": "po_browser_used" + }, + { + "column": { + "index": 5, + "name": "po_language" + }, + "property": "po_language" + }, + { + "column": { + "index": 6, + "name": "po_content" + }, + "property": "po_content" + }, + { + "column": { + "index": 7, + "name": "po_length" + }, + "property": "po_length" + } + ] + }, + { + "type_name": "forum", + "inputs": [ + "forum" + ], + "column_mappings": [ + { + "column": { + "index": 0, + "name": "fo_id" + }, + "property": "fo_id" + }, + { + "column": { + "index": 1, + "name": "fo_title" + }, + "property": "fo_title" + }, + { + "column": { + "index": 2, + "name": "fo_creation_date" + }, + "property": "fo_creation_date" + } + ] + } + ], + "edge_mappings": [ + { + "type_triplet": { + "edge": "org_islocationin", + "source_vertex": "organisation", + "destination_vertex": "place" + }, + "inputs": [ + "org_islocationin" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "ispartof", + "source_vertex": "place", + "destination_vertex": "place" + }, + "inputs": [ + "ispartof" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "issubclassof", + "source_vertex": "tagclass", + "destination_vertex": "tagclass" + }, + "inputs": [ + "issubclassof" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "hastype", + "source_vertex": "tag", + "destination_vertex": "tagclass" + }, + "inputs": [ + "hastype" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "comment_hascreator", + "source_vertex": "comment", + "destination_vertex": "person" + }, + "inputs": [ + "comment_hascreator" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "comment_hastag", + "source_vertex": "comment", + "destination_vertex": "tag" + }, + "inputs": [ + "comment_hastag" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "comment_islocationin", + "source_vertex": "comment", + "destination_vertex": "place" + }, + "inputs": [ + "comment_islocationin" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "replyof_comment", + "source_vertex": "comment", + "destination_vertex": "comment" + }, + "inputs": [ + "replyof_comment" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "replyof_post", + "source_vertex": "comment", + "destination_vertex": "post" + }, + "inputs": [ + "replyof_post" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "post_hascreator", + "source_vertex": "post", + "destination_vertex": "person" + }, + "inputs": [ + "post_hascreator" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "post_hastag", + "source_vertex": "post", + "destination_vertex": "tag" + }, + "inputs": [ + "post_hastag" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "post_islocationin", + "source_vertex": "post", + "destination_vertex": "place" + }, + "inputs": [ + "post_islocationin" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "forum_containerof", + "source_vertex": "forum", + "destination_vertex": "post" + }, + "inputs": [ + "forum_containerof" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "forum_hasmoderator", + "source_vertex": "forum", + "destination_vertex": "person" + }, + "inputs": [ + "forum_hasmoderator" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "forum_hastag", + "source_vertex": "forum", + "destination_vertex": "tag" + }, + "inputs": [ + "forum_hastag" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "person_hasinterest", + "source_vertex": "person", + "destination_vertex": "tag" + }, + "inputs": [ + "person_hasinterest" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "person_islocationin", + "source_vertex": "person", + "destination_vertex": "place" + }, + "inputs": [ + "person_islocationin" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + + ] + }, + { + "type_triplet": { + "edge": "forum_hasmember", + "source_vertex": "forum", + "destination_vertex": "person" + }, + "inputs": [ + "forum_hasmember" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + { + "column": { + "index": 2, + "name": "fo_hm_join_date" + }, + "property": "fo_hm_join_date" + } + ] + }, + { + "type_triplet": { + "edge": "knows", + "source_vertex": "person", + "destination_vertex": "person" + }, + "inputs": [ + "knows" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + { + "column": { + "index": 2, + "name": "kn_creation_date" + }, + "property": "kn_creation_date" + } + ] + }, + { + "type_triplet": { + "edge": "likes_comment", + "source_vertex": "person", + "destination_vertex": "comment" + }, + "inputs": [ + "likes_comment" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + { + "column": { + "index": 2, + "name": "likes_co_creation_date" + }, + "property": "likes_co_creation_date" + } + ] + }, + { + "type_triplet": { + "edge": "likes_post", + "source_vertex": "person", + "destination_vertex": "post" + }, + "inputs": [ + "likes_post" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + { + "column": { + "index": 2, + "name": "likes_po_creation_date" + }, + "property": "likes_po_creation_date" + } + ] + }, + { + "type_triplet": { + "edge": "studyat", + "source_vertex": "person", + "destination_vertex": "organisation" + }, + "inputs": [ + "studyat" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + { + "column": { + "index": 2, + "name": "sa_class_year" + }, + "property": "sa_class_year" + } + ] + }, + { + "type_triplet": { + "edge": "workat", + "source_vertex": "person", + "destination_vertex": "organisation" + }, + "inputs": [ + "workat" + ], + "source_vertex_mappings": [ + { + "column": { + "index": 0, + "name": "src" + }, + "property": "id" + } + ], + "destination_vertex_mappings": [ + { + "column": { + "index": 1, + "name": "dst" + }, + "property": "id" + } + ], + "column_mappings": [ + { + "column": { + "index": 2, + "name": "wa_work_from" + }, + "property": "wa_work_from" + } + ] + } + ] +} \ No newline at end of file diff --git a/vegito/test/schema/ldbc-graph-schema.json b/vegito/test/schema/ldbc-graph-schema.json new file mode 100644 index 0000000..ad86587 --- /dev/null +++ b/vegito/test/schema/ldbc-graph-schema.json @@ -0,0 +1,722 @@ +{ + "name": "ldbc_graph", + "description": "This is a test graph", + "schema": { + "vertex_types": [ + { + "type_name": "organisation", + "properties": [ + { + "property_name": "org_id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_name": "org_type", + "property_type": { + "string": { + "long_text": "" + } + } + }, + { + "property_name": "org_name", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "org_url", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + } + ], + "primary_keys": [ + "org_id" + ] + }, + { + "type_name": "place", + "properties": [ + { + "property_name": "pla_id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_name": "pla_name", + "property_type": { + "string": { + "long_text": "" + } + } + }, + { + "property_name": "pla_url", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "pla_type", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + } + ], + "primary_keys": [ + "pla_id" + ] + }, + { + "type_name": "tag", + "properties": [ + { + "property_name": "tag_id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_name": "tag_name", + "property_type": { + "string": { + "long_text": "" + } + } + }, + { + "property_name": "tag_url", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + } + ], + "primary_keys": [ + "tag_id" + ] + }, + { + "type_name": "tagclass", + "properties": [ + { + "property_name": "tagc_id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_name": "tagc_name", + "property_type": { + "string": { + "long_text": "" + } + } + }, + { + "property_name": "tagc_url", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + } + ], + "primary_keys": [ + "tagc_id" + ] + }, + { + "type_name": "person", + "properties": [ + { + "property_name": "p_id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_name": "p_first_name", + "property_type": { + "string": { + "long_text": "" + } + } + }, + { + "property_name": "p_last_name", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "p_gender", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "p_birthday", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "p_creation_date", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "p_location_ip", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "p_browser_used", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + } + ], + "primary_keys": [ + "p_id" + ] + }, + { + "type_name": "comment", + "properties": [ + { + "property_name": "co_id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_name": "co_creation_date", + "property_type": { + "string": { + "long_text": "" + } + } + }, + { + "property_name": "co_location_ip", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "co_browser_used", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "co_content", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "co_length", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + } + ], + "primary_keys": [ + "co_id" + ] + }, + { + "type_name": "post", + "properties": [ + { + "property_name": "po_id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_name": "po_image_file", + "property_type": { + "string": { + "long_text": "" + } + } + }, + { + "property_name": "po_creation_date", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "po_location_ip", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "po_browser_used", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "po_language", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "po_content", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + }, + { + "property_name": "po_length", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + } + ], + "primary_keys": [ + "po_id" + ] + }, + { + "type_name": "forum", + "properties": [ + { + "property_name": "fo_id", + "property_type": { + "primitive_type": "DT_SIGNED_INT64" + } + }, + { + "property_name": "fo_title", + "property_type": { + "string": { + "long_text": "" + } + } + }, + { + "property_name": "fo_creation_date", + "property_type": { + "primitive_type": "DT_SIGNED_INT32" + } + } + ], + "primary_keys": [ + "fo_id" + ] + } + ], + "edge_types": [ + { + "type_name": "org_islocationin", + "vertex_type_pair_relations": [ + { + "source_vertex": "organisation", + "destination_vertex": "place", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "ispartof", + "vertex_type_pair_relations": [ + { + "source_vertex": "place", + "destination_vertex": "place", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "issubclassof", + "vertex_type_pair_relations": [ + { + "source_vertex": "tagclass", + "destination_vertex": "tagclass", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "hastype", + "vertex_type_pair_relations": [ + { + "source_vertex": "tag", + "destination_vertex": "tagclass", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "comment_hascreator", + "vertex_type_pair_relations": [ + { + "source_vertex": "comment", + "destination_vertex": "person", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "comment_hastag", + "vertex_type_pair_relations": [ + { + "source_vertex": "comment", + "destination_vertex": "tag", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "comment_islocationin", + "vertex_type_pair_relations": [ + { + "source_vertex": "comment", + "destination_vertex": "place", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "replyof_comment", + "vertex_type_pair_relations": [ + { + "source_vertex": "comment", + "destination_vertex": "comment", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "replyof_post", + "vertex_type_pair_relations": [ + { + "source_vertex": "comment", + "destination_vertex": "post", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "post_hascreator", + "vertex_type_pair_relations": [ + { + "source_vertex": "post", + "destination_vertex": "person", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "post_hastag", + "vertex_type_pair_relations": [ + { + "source_vertex": "post", + "destination_vertex": "tag", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "post_islocationin", + "vertex_type_pair_relations": [ + { + "source_vertex": "post", + "destination_vertex": "place", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "forum_containerof", + "vertex_type_pair_relations": [ + { + "source_vertex": "forum", + "destination_vertex": "post", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "forum_hasmoderator", + "vertex_type_pair_relations": [ + { + "source_vertex": "forum", + "destination_vertex": "person", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "forum_hastag", + "vertex_type_pair_relations": [ + { + "source_vertex": "forum", + "destination_vertex": "tag", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "person_hasinterest", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "tag", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "person_islocationin", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "place", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + + ], + "primary_keys": [ + + ] + }, + { + "type_name": "forum_hasmember", + "vertex_type_pair_relations": [ + { + "source_vertex": "forum", + "destination_vertex": "person", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + { + "property_name": "fo_hm_join_date", + "property_type": { + "primitive_type": "DT_DOUBLE" + } + } + ], + "primary_keys": [ + + ] + }, + { + "type_name": "knows", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "person", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + { + "property_name": "kn_creation_date", + "property_type": { + "primitive_type": "DT_DOUBLE" + } + } + ], + "primary_keys": [ + + ] + }, + { + "type_name": "likes_comment", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "comment", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + { + "property_name": "likes_co_creation_date", + "property_type": { + "primitive_type": "DT_DOUBLE" + } + } + ], + "primary_keys": [ + + ] + }, + { + "type_name": "likes_post", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "post", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + { + "property_name": "likes_po_creation_date", + "property_type": { + "primitive_type": "DT_DOUBLE" + } + } + ], + "primary_keys": [ + + ] + }, + { + "type_name": "studyat", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "organisation", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + { + "property_name": "sa_class_year", + "property_type": { + "primitive_type": "DT_DOUBLE" + } + } + ], + "primary_keys": [ + + ] + }, + { + "type_name": "workat", + "vertex_type_pair_relations": [ + { + "source_vertex": "person", + "destination_vertex": "organisation", + "relation": "MANY_TO_MANY" + } + ], + "properties": [ + { + "property_name": "wa_work_from", + "property_type": { + "primitive_type": "DT_DOUBLE" + } + } + ], + "primary_keys": [ + + ] + } + ] + } +} \ No newline at end of file