diff --git a/airflow_variables.json b/airflow_variables.json index a8066d7b..fbca13cd 100644 --- a/airflow_variables.json +++ b/airflow_variables.json @@ -91,7 +91,11 @@ "transaction_id", "account", "type" - ] + ], + "contract_data": ["last_modified_ledger"], + "contract_code": ["last_modified_ledger"], + "config_settings": ["last_modified_ledger"], + "expiration": ["last_modified_ledger"] }, "dbt_auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "dbt_auth_uri": "https://accounts.google.com/o/oauth2/auth", @@ -112,7 +116,7 @@ "partnership_assets__asset_activity_fact": false, "trade_agg": false }, - "dbt_image_name": "stellar/stellar-dbt:0643adc", + "dbt_image_name": "stellar/stellar-dbt:011d897", "dbt_job_execution_timeout_seconds": 6000, "dbt_job_retries": 1, "dbt_keyfile_profile": "", @@ -126,7 +130,7 @@ "dbt_token_uri": "https://oauth2.googleapis.com/token", "gcs_exported_data_bucket_name": "us-central1-hubble-2-d948d67b-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:6211230", + "image_name": "stellar/stellar-etl:e9c803c", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "kube_config_location": "", @@ -153,7 +157,12 @@ "signers": "account_signers.txt", "trades": "trades.txt", "transactions": "transactions.txt", - "trustlines": "trustlines.txt" + "trustlines": "trustlines.txt", + "contract_data": "contract_data.txt", + "contract_code": "contract_code.txt", + "config_settings": "config_settings.txt", + "expiration": "expiration.txt", + "diagnostic_events": "diagnostic_events.txt" }, "output_path": "/home/airflow/gcs/data/", "owner": "SDF", @@ -213,6 +222,22 @@ "trust_lines": { "type": "MONTH", "field": "batch_run_date" + }, + "contract_data": { + "field": "batch_run_date", + "type": "MONTH" + }, + "contract_code": { + "field": "batch_run_date", + "type": "MONTH" + }, + "config_settings": { + "field": "batch_run_date", + "type": "MONTH" + }, + "expiration": { + "field": "batch_run_date", + "type": "MONTH" } }, "public_dataset": "crypto_stellar_2", @@ -258,7 +283,12 @@ "trades": "history_trades", "transactions": "history_transactions", "trustlines": "trust_lines", - "enriched_history_operations": "enriched_history_operations" + "enriched_history_operations": "enriched_history_operations", + "contract_data": "contract_data", + "contract_code": "contract_code", + "config_settings": "config_settings", + "expiration": "expiration", + "diagnostic_events": "diagnostic_events" }, "task_timeout": { "build_batch_stats": 180, @@ -289,6 +319,7 @@ } }, "partners_bucket": "ext-partner-sftp", + "use_futurenet": "False", "currency_ohlc": { "currency": "euro_ohlc", "table_name": "euro_usd_ohlc", diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index a39193ea..fbdc4f87 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -78,7 +78,11 @@ "transaction_id", "account", "type" - ] + ], + "contract_data": ["last_modified_ledger"], + "contract_code": ["last_modified_ledger"], + "config_settings": ["last_modified_ledger"], + "expiration": ["last_modified_ledger"] }, "dbt_auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "dbt_auth_uri": "https://accounts.google.com/o/oauth2/auth", @@ -90,7 +94,7 @@ "partnership_assets__account_holders_activity_fact": true, "partnership_assets__asset_activity_fact": true }, - "dbt_image_name": "stellar/stellar-dbt:0643adc", + "dbt_image_name": "stellar/stellar-dbt:011d897", "dbt_job_execution_timeout_seconds": 300, "dbt_job_retries": 1, "dbt_keyfile_profile": "", @@ -104,7 +108,7 @@ "dbt_token_uri": "https://oauth2.googleapis.com/token", "gcs_exported_data_bucket_name": "us-central1-hubble-1pt5-dev-7db0e004-bucket", "gcs_exported_object_prefix": "dag-exported", - "image_name": "stellar/stellar-etl:6211230", + "image_name": "chowbao/stellar-etl:testnet-noUsdDb", "image_output_path": "/etl/exported_data/", "image_pull_policy": "IfNotPresent", "kube_config_location": "", @@ -130,7 +134,12 @@ "signers": "account_signers.txt", "trades": "trades.txt", "transactions": "transactions.txt", - "trustlines": "trustlines.txt" + "trustlines": "trustlines.txt", + "contract_data": "contract_data.txt", + "contract_code": "contract_code.txt", + "config_settings": "config_settings.txt", + "expiration": "expiration.txt", + "diagnostic_events": "diagnostic_events.txt" }, "output_path": "/home/airflow/gcs/data/", "owner": "SDF", @@ -194,6 +203,22 @@ "mgi": { "field": "tran_evnt_date", "type": "MONTH" + }, + "contract_data": { + "field": "batch_run_date", + "type": "MONTH" + }, + "contract_code": { + "field": "batch_run_date", + "type": "MONTH" + }, + "config_settings": { + "field": "batch_run_date", + "type": "MONTH" + }, + "expiration": { + "field": "batch_run_date", + "type": "MONTH" } }, "partners_data": { @@ -246,7 +271,12 @@ "trades": "history_trades", "transactions": "history_transactions", "trustlines": "trust_lines", - "enriched_history_operations": "enriched_history_operations" + "enriched_history_operations": "enriched_history_operations", + "contract_data": "contract_data", + "contract_code": "contract_code", + "config_settings": "config_settings", + "expiration": "expiration", + "diagnostic_events": "diagnostic_events" }, "task_timeout": { "build_batch_stats": 180, @@ -269,6 +299,7 @@ "sandbox_dataset": "crypto_stellar_internal_sandbox", "volume_config": {}, "volume_name": "etl-data", + "use_futurenet": "False", "currency_ohlc": { "currency": "euro_ohlc", "table_name": "euro_usd_ohlc", diff --git a/dags/asset_pricing_pipeline_dag.py b/dags/asset_pricing_pipeline_dag.py index b154a6fe..14120229 100644 --- a/dags/asset_pricing_pipeline_dag.py +++ b/dags/asset_pricing_pipeline_dag.py @@ -1,4 +1,5 @@ import datetime +import json from airflow import DAG from airflow.models.variable import Variable diff --git a/dags/bucket_list_dag.py b/dags/bucket_list_dag.py index 6307554f..2ccfc08a 100644 --- a/dags/bucket_list_dag.py +++ b/dags/bucket_list_dag.py @@ -44,6 +44,7 @@ public_project = Variable.get("public_project") public_dataset = Variable.get("public_dataset") use_testnet = ast.literal_eval(Variable.get("use_testnet")) +use_futurenet = ast.literal_eval(Variable.get("use_futurenet")) """ The time task reads in the execution time of the current run, as well as the next @@ -61,6 +62,7 @@ "export_accounts", file_names["accounts"], use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, ) export_bal_task = build_export_task( @@ -69,6 +71,7 @@ "export_claimable_balances", file_names["claimable_balances"], use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, ) export_off_task = build_export_task( @@ -77,6 +80,7 @@ "export_offers", file_names["offers"], use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, ) export_pool_task = build_export_task( @@ -85,6 +89,7 @@ "export_pools", file_names["liquidity_pools"], use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, ) export_sign_task = build_export_task( @@ -93,6 +98,7 @@ "export_signers", file_names["signers"], use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, ) export_trust_task = build_export_task( @@ -101,6 +107,7 @@ "export_trustlines", file_names["trustlines"], use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, ) diff --git a/dags/history_archive_with_captive_core_dag.py b/dags/history_archive_with_captive_core_dag.py index 3b4e9883..f337dbe1 100644 --- a/dags/history_archive_with_captive_core_dag.py +++ b/dags/history_archive_with_captive_core_dag.py @@ -23,7 +23,8 @@ dag = DAG( "history_archive_with_captive_core", default_args=get_default_dag_args(), - start_date=datetime.datetime(2022, 3, 11, 18, 30), + start_date=datetime.datetime(2023, 9, 20, 15, 0), + catchup=True, description="This DAG exports trades and operations from the history archive using CaptiveCore. This supports parsing sponsorship and AMMs.", schedule_interval="*/30 * * * *", params={ @@ -44,12 +45,13 @@ public_dataset = Variable.get("public_dataset") public_dataset_new = Variable.get("public_dataset_new") use_testnet = ast.literal_eval(Variable.get("use_testnet")) +use_futurenet = ast.literal_eval(Variable.get("use_futurenet")) """ The time task reads in the execution time of the current run, as well as the next execution time. It converts these two times into ledger ranges. """ -time_task = build_time_task(dag, use_testnet=use_testnet) +time_task = build_time_task(dag, use_testnet=use_testnet, use_futurenet=use_futurenet) """ The write batch stats task will take a snapshot of the DAG run_id, execution date, @@ -60,6 +62,7 @@ write_trade_stats = build_batch_stats(dag, table_names["trades"]) write_effects_stats = build_batch_stats(dag, table_names["effects"]) write_tx_stats = build_batch_stats(dag, table_names["transactions"]) +write_diagnostic_events_stats = build_batch_stats(dag, table_names["diagnostic_events"]) """ The export tasks call export commands on the Stellar ETL using the ledger range from the time task. @@ -76,6 +79,7 @@ "export_operations", file_names["operations"], use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, resource_cfg="cc", ) @@ -85,6 +89,7 @@ "export_trades", file_names["trades"], use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, resource_cfg="cc", ) @@ -94,6 +99,7 @@ "export_effects", "effects.txt", use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, resource_cfg="cc", ) @@ -103,6 +109,17 @@ "export_transactions", file_names["transactions"], use_testnet=use_testnet, + use_futurenet=use_futurenet, + use_gcs=True, + resource_cfg="cc", +) +diagnostic_events_export_task = build_export_task( + dag, + "archive", + "export_diagnostic_events", + file_names["diagnostic_events"], + use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, resource_cfg="cc", ) @@ -380,3 +397,4 @@ ) tx_export_task >> delete_old_tx_pub_task >> send_txs_to_pub_task >> wait_on_dag tx_export_task >> delete_old_tx_pub_new_task >> send_txs_to_pub_new_task >> wait_on_dag +(time_task >> write_diagnostic_events_stats >> diagnostic_events_export_task) diff --git a/dags/history_archive_without_captive_core_dag.py b/dags/history_archive_without_captive_core_dag.py index fcbc3144..68d15eee 100644 --- a/dags/history_archive_without_captive_core_dag.py +++ b/dags/history_archive_without_captive_core_dag.py @@ -22,7 +22,8 @@ dag = DAG( "history_archive_without_captive_core", default_args=get_default_dag_args(), - start_date=datetime.datetime(2022, 3, 11, 18, 30), + start_date=datetime.datetime(2023, 9, 20, 15, 0), + catchup=True, description="This DAG exports ledgers, transactions, and assets from the history archive to BigQuery. Incremental Loads", schedule_interval="*/15 * * * *", params={ @@ -43,12 +44,13 @@ public_dataset = Variable.get("public_dataset") public_dataset_new = Variable.get("public_dataset_new") use_testnet = ast.literal_eval(Variable.get("use_testnet")) +use_futurenet = ast.literal_eval(Variable.get("use_futurenet")) """ The time task reads in the execution time of the current run, as well as the next execution time. It converts these two times into ledger ranges. """ -time_task = build_time_task(dag, use_testnet=use_testnet) +time_task = build_time_task(dag, use_testnet=use_testnet, use_futurenet=use_futurenet) """ The write batch stats task will take a snapshot of the DAG run_id, execution date, @@ -73,6 +75,7 @@ "export_ledgers", file_names["ledgers"], use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, ) asset_export_task = build_export_task( @@ -81,6 +84,7 @@ "export_assets", file_names["assets"], use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, ) diff --git a/dags/marts_tables_dag.py b/dags/marts_tables_dag.py index 98626d38..5c447d50 100644 --- a/dags/marts_tables_dag.py +++ b/dags/marts_tables_dag.py @@ -10,16 +10,16 @@ dag = DAG( "marts_tables", default_args=get_default_dag_args(), - start_date=datetime.datetime(2023, 4, 4, 0, 0), + start_date=datetime.datetime(2015, 9, 30), description="This DAG runs dbt to create the tables for the models in marts/ but not any marts subdirectories.", schedule_interval="0 11 * * *", # Daily 11 AM UTC params={}, - catchup=False, + catchup=True, + max_active_runs=1, ) # tasks for staging tables for marts stg_history_transactions = build_dbt_task(dag, "stg_history_transactions") -stg_history_ledgers = build_dbt_task(dag, "stg_history_ledgers") stg_history_assets = build_dbt_task(dag, "stg_history_assets") stg_history_trades = build_dbt_task(dag, "stg_history_trades") @@ -31,11 +31,12 @@ # tasks for intermediate asset stats tables int_meaningful_asset_prices = build_dbt_task(dag, "int_meaningful_asset_prices") +int_asset_stats_agg = build_dbt_task(dag, "int_asset_stats_agg") stg_excluded_accounts = build_dbt_task(dag, "stg_excluded_accounts") stg_xlm_to_usd = build_dbt_task(dag, "stg_xlm_to_usd") # tasks for marts tables -agg_network_stats = build_dbt_task(dag, "agg_network_stats") +network_stats_agg = build_dbt_task(dag, "network_stats_agg") asset_stats_agg = build_dbt_task(dag, "asset_stats_agg") fee_stats_agg = build_dbt_task(dag, "fee_stats_agg") history_assets = build_dbt_task(dag, "history_assets") @@ -44,14 +45,14 @@ # DAG task graph # graph for marts tables -agg_network_stats +network_stats_agg liquidity_providers -int_meaningful_asset_prices >> asset_stats_agg -stg_excluded_accounts >> asset_stats_agg -stg_xlm_to_usd >> asset_stats_agg +int_meaningful_asset_prices >> int_asset_stats_agg +stg_excluded_accounts >> int_asset_stats_agg +stg_xlm_to_usd >> int_asset_stats_agg +int_asset_stats_agg >> asset_stats_agg stg_history_transactions >> fee_stats_agg -stg_history_ledgers >> fee_stats_agg stg_history_assets >> history_assets diff --git a/dags/state_table_dag.py b/dags/state_table_dag.py index 94c96980..62f83ea5 100644 --- a/dags/state_table_dag.py +++ b/dags/state_table_dag.py @@ -26,7 +26,7 @@ dag = DAG( "state_table_export", default_args=get_default_dag_args(), - start_date=datetime.datetime(2022, 3, 11, 19, 00), + start_date=datetime.datetime(2023, 9, 20, 15, 0), description="This DAG runs a bounded stellar-core instance, which allows it to export accounts, offers, liquidity pools, and trustlines to BigQuery.", schedule_interval="*/30 * * * *", params={ @@ -37,6 +37,7 @@ "subtract_data_interval": macros.subtract_data_interval, "batch_run_date_as_datetime_string": macros.batch_run_date_as_datetime_string, }, + catchup=True, ) file_names = Variable.get("output_file_names", deserialize_json=True) @@ -47,14 +48,16 @@ public_dataset = Variable.get("public_dataset") public_dataset_new = Variable.get("public_dataset_new") use_testnet = ast.literal_eval(Variable.get("use_testnet")) +use_futurenet = ast.literal_eval(Variable.get("use_futurenet")) -date_task = build_time_task(dag, use_testnet=use_testnet) +date_task = build_time_task(dag, use_testnet=use_testnet, use_futurenet=use_futurenet) changes_task = build_export_task( dag, "bounded-core", "export_ledger_entry_changes", file_names["changes"], use_testnet=use_testnet, + use_futurenet=use_futurenet, use_gcs=True, resource_cfg="state", ) @@ -70,6 +73,10 @@ write_pool_stats = build_batch_stats(dag, table_names["liquidity_pools"]) write_sign_stats = build_batch_stats(dag, table_names["signers"]) write_trust_stats = build_batch_stats(dag, table_names["trustlines"]) +write_contract_data_stats = build_batch_stats(dag, table_names["contract_data"]) +write_contract_code_stats = build_batch_stats(dag, table_names["contract_code"]) +write_config_settings_stats = build_batch_stats(dag, table_names["config_settings"]) +write_expiration_stats = build_batch_stats(dag, table_names["expiration"]) """ The delete partition task checks to see if the given partition/batch id exists in @@ -114,6 +121,18 @@ delete_trust_pub_new_task = build_delete_data_task( dag, public_project, public_dataset_new, table_names["trustlines"] ) +delete_contract_data_task = build_delete_data_task( + dag, public_project, public_dataset_new, table_names["contract_data"] +) +delete_contract_code_task = build_delete_data_task( + dag, public_project, public_dataset_new, table_names["contract_code"] +) +delete_config_settings_task = build_delete_data_task( + dag, public_project, public_dataset_new, table_names["config_settings"] +) +delete_expiration_task = build_delete_data_task( + dag, public_project, public_dataset_new, table_names["expiration"] +) """ The apply tasks receive the location of the file in Google Cloud storage through Airflow's XCOM system. @@ -260,6 +279,46 @@ partition=True, cluster=True, ) +send_contract_data_to_pub_task = build_gcs_to_bq_task( + dag, + changes_task.task_id, + public_project, + public_dataset_new, + table_names["contract_data"], + "/*-contract_data.txt", + partition=True, + cluster=True, +) +send_contract_code_to_pub_task = build_gcs_to_bq_task( + dag, + changes_task.task_id, + public_project, + public_dataset_new, + table_names["contract_code"], + "/*-contract_code.txt", + partition=True, + cluster=True, +) +send_config_settings_to_pub_task = build_gcs_to_bq_task( + dag, + changes_task.task_id, + public_project, + public_dataset_new, + table_names["config_settings"], + "/*-config_settings.txt", + partition=True, + cluster=True, +) +send_expiration_to_pub_task = build_gcs_to_bq_task( + dag, + changes_task.task_id, + public_project, + public_dataset_new, + table_names["expiration"], + "/*-expiration.txt", + partition=True, + cluster=True, +) date_task >> changes_task >> write_acc_stats >> delete_acc_task >> send_acc_to_bq_task write_acc_stats >> delete_acc_pub_new_task >> send_acc_to_pub_new_task @@ -292,3 +351,31 @@ >> send_trust_to_bq_task ) write_trust_stats >> delete_trust_pub_new_task >> send_trust_to_pub_new_task +( + date_task + >> changes_task + >> write_contract_data_stats + >> delete_contract_data_task + >> send_contract_data_to_pub_task +) +( + date_task + >> changes_task + >> write_contract_code_stats + >> delete_contract_code_task + >> send_contract_code_to_pub_task +) +( + date_task + >> changes_task + >> write_config_settings_stats + >> delete_config_settings_task + >> send_config_settings_to_pub_task +) +( + date_task + >> changes_task + >> write_expiration_stats + >> delete_expiration_task + >> send_expiration_to_pub_task +) diff --git a/dags/stellar_etl_airflow/build_export_task.py b/dags/stellar_etl_airflow/build_export_task.py index 8b1b3daf..ec0a02c0 100644 --- a/dags/stellar_etl_airflow/build_export_task.py +++ b/dags/stellar_etl_airflow/build_export_task.py @@ -15,13 +15,15 @@ from stellar_etl_airflow.default import alert_after_max_retries -def get_path_variables(use_testnet=False): +def get_path_variables(use_testnet=False, use_futurenet=False): """ Returns the image output path, core executable path, and core config path. """ config = "/etl/docker/stellar-core.cfg" if use_testnet: config = "/etl/docker/stellar-core_testnet.cfg" + elif use_futurenet: + config = "/etl/docker/stellar-core_futurenet.cfg" return "/usr/bin/stellar-core", config @@ -39,7 +41,12 @@ def select_correct_filename(cmd_type, base_name, batched_name): def generate_etl_cmd( - command, base_filename, cmd_type, use_gcs=False, use_testnet=False + command, + base_filename, + cmd_type, + use_gcs=False, + use_testnet=False, + use_futurenet=False, ): """ Runs the provided stellar-etl command with arguments that are appropriate for the command type. @@ -71,7 +78,7 @@ def generate_etl_cmd( if cmd_type in ("archive", "bounded-core"): end_ledger = '{{ [ti.xcom_pull(task_ids="get_ledger_range_from_times")["end"]-1, ti.xcom_pull(task_ids="get_ledger_range_from_times")["start"]] | max}}' - core_exec, core_cfg = get_path_variables(use_testnet) + core_exec, core_cfg = get_path_variables(use_testnet, use_futurenet) batch_filename = "-".join([start_ledger, end_ledger, base_filename]) run_id = "{{ run_id }}" @@ -134,6 +141,8 @@ def generate_etl_cmd( cmd.extend(["-u", metadata]) if use_testnet: cmd.append("--testnet") + elif use_futurenet: + cmd.append("--futurenet") cmd.append("--strict-export") return cmd, os.path.join(filepath, correct_filename) @@ -145,6 +154,7 @@ def build_export_task( filename, use_gcs=False, use_testnet=False, + use_futurenet=False, resource_cfg="default", ): """ @@ -161,7 +171,7 @@ def build_export_task( """ etl_cmd, output_file = generate_etl_cmd( - command, filename, cmd_type, use_gcs, use_testnet + command, filename, cmd_type, use_gcs, use_testnet, use_futurenet ) etl_cmd_string = " ".join(etl_cmd) config_file_location = Variable.get("kube_config_location") diff --git a/dags/stellar_etl_airflow/build_gcs_to_bq_task.py b/dags/stellar_etl_airflow/build_gcs_to_bq_task.py index 86f7864a..d91858bb 100644 --- a/dags/stellar_etl_airflow/build_gcs_to_bq_task.py +++ b/dags/stellar_etl_airflow/build_gcs_to_bq_task.py @@ -131,7 +131,8 @@ def build_gcs_to_bq_task( destination_project_dataset_table=f"{project_name}.{dataset_name}.{data_type}{staging_table_suffix}", write_disposition="WRITE_APPEND", create_disposition="CREATE_IF_NEEDED", - max_bad_records=10, + schema_update_option="ALLOW_FIELD_ADDITION", + max_bad_records=0, time_partitioning=time_partition, cluster_fields=cluster_fields, export_task_id=export_task_id, @@ -166,7 +167,7 @@ def build_gcs_to_bq_task( destination_project_dataset_table=f"{project_name}.{dataset_name}.{data_type}{staging_table_suffix}", write_disposition="WRITE_APPEND", create_disposition="CREATE_IF_NEEDED", - max_bad_records=10, + max_bad_records=0, time_partitioning=time_partition, cluster_fields=cluster_fields, on_failure_callback=alert_after_max_retries, diff --git a/dags/stellar_etl_airflow/build_time_task.py b/dags/stellar_etl_airflow/build_time_task.py index c85816d4..87f31242 100644 --- a/dags/stellar_etl_airflow/build_time_task.py +++ b/dags/stellar_etl_airflow/build_time_task.py @@ -13,7 +13,11 @@ def build_time_task( - dag, use_testnet=False, use_next_exec_time=True, resource_cfg="default" + dag, + use_testnet=False, + use_next_exec_time=True, + resource_cfg="default", + use_futurenet=False, ): """ Creates a task to run the get_ledger_range_from_times command from the stellar-etl Docker image. The start time is the previous @@ -47,6 +51,8 @@ def build_time_task( logging.info(f"Constructing command with args: {args}") if use_testnet: args.append("--testnet") + elif use_futurenet: + args.append("--futurenet") config_file_location = Variable.get("kube_config_location") in_cluster = False if config_file_location else True resources_requests = ( diff --git a/schemas/config_settings_schema.json b/schemas/config_settings_schema.json new file mode 100644 index 00000000..cb3eb2e9 --- /dev/null +++ b/schemas/config_settings_schema.json @@ -0,0 +1,281 @@ +[ + { + "mode": "NULLABLE", + "name": "config_setting_id", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "contract_max_size_bytes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_max_instructions", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "tx_max_instructions", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "fee_rate_per_instructions_increment", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "tx_memory_limit", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_max_read_ledger_entries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_max_read_bytes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_max_write_ledger_entries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_max_write_bytes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "tx_max_read_ledger_entries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "tx_max_read_bytes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "tx_max_write_ledger_entries", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "tx_max_write_bytes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "fee_read_ledger_entry", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "fee_write_ledger_entry", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "fee_read_1kb", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "bucket_list_target_size_bytes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "write_fee_1kb_bucket_list_low", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "write_fee_1kb_bucket_list_high", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "bucket_list_write_fee_growth_factor", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "fee_historical_1kb", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "tx_max_contract_events_size_bytes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "fee_contract_events_1kb", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_max_txs_size_bytes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "tx_max_size_bytes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "fee_tx_size_1kb", + "type": "INTEGER" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "ExtV", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "ConstTerm", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "LinearTerm", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "contract_cost_params_cpu_insns", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "ExtV", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "ConstTerm", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "LinearTerm", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "contract_cost_params_mem_bytes", + "type": "RECORD" + }, + { + "mode": "NULLABLE", + "name": "contract_data_key_size_bytes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "contract_data_entry_size_bytes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "max_entry_expiration", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "min_temp_entry_expiration", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "min_persistent_entry_expiration", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "auto_bump_ledgers", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "persistent_rent_rate_denominator", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "temp_rent_rate_denominator", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "max_entries_to_expire", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "bucket_list_size_window_sample_size", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "eviction_scan_size", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "starting_eviction_scan_level", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_max_tx_count", + "type": "INTEGER" + }, + { + "mode": "REPEATED", + "name": "bucket_list_size_window", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "last_modified_ledger", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_entry_change", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "deleted", + "type": "BOOLEAN" + }, + { + "mode": "NULLABLE", + "name": "batch_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "batch_run_date", + "type": "DATETIME" + }, + { + "mode": "NULLABLE", + "name": "batch_insert_ts", + "type": "TIMESTAMP" + } +] diff --git a/schemas/contract_code_schema.json b/schemas/contract_code_schema.json new file mode 100644 index 00000000..7c141cc9 --- /dev/null +++ b/schemas/contract_code_schema.json @@ -0,0 +1,42 @@ +[ + { + "mode": "NULLABLE", + "name": "contract_code_hash", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "contract_code_ext_v", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "last_modified_ledger", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_entry_change", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "deleted", + "type": "BOOLEAN" + }, + { + "mode": "NULLABLE", + "name": "batch_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "batch_run_date", + "type": "DATETIME" + }, + { + "mode": "NULLABLE", + "name": "batch_insert_ts", + "type": "TIMESTAMP" + } +] diff --git a/schemas/contract_data_schema.json b/schemas/contract_data_schema.json new file mode 100644 index 00000000..351e0f9c --- /dev/null +++ b/schemas/contract_data_schema.json @@ -0,0 +1,72 @@ +[ + { + "mode": "NULLABLE", + "name": "contract_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "contract_key_type", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "contract_durability", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_code", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_issuer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_type", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "balance_holder", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "balance", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "last_modified_ledger", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_entry_change", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "deleted", + "type": "BOOLEAN" + }, + { + "mode": "NULLABLE", + "name": "batch_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "batch_run_date", + "type": "DATETIME" + }, + { + "mode": "NULLABLE", + "name": "batch_insert_ts", + "type": "TIMESTAMP" + } +] diff --git a/schemas/expiration_schema.json b/schemas/expiration_schema.json new file mode 100644 index 00000000..e1389a97 --- /dev/null +++ b/schemas/expiration_schema.json @@ -0,0 +1,42 @@ +[ + { + "mode": "NULLABLE", + "name": "key_hash", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "expiration_ledger_seq", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "last_modified_ledger", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "ledger_entry_change", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "deleted", + "type": "BOOLEAN" + }, + { + "mode": "NULLABLE", + "name": "batch_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "batch_run_date", + "type": "DATETIME" + }, + { + "mode": "NULLABLE", + "name": "batch_insert_ts", + "type": "TIMESTAMP" + } +] diff --git a/schemas/history_effects_schema.json b/schemas/history_effects_schema.json index 63a096fe..1e6f0d91 100644 --- a/schemas/history_effects_schema.json +++ b/schemas/history_effects_schema.json @@ -760,6 +760,26 @@ "mode": "NULLABLE", "name": "bought_asset_issuer", "type": "STRING" + }, + { + "mode": "REPEATED", + "name": "entries", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "ledgers_to_expire", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "contract", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "contract_event_type", + "type": "STRING" } ], "mode": "NULLABLE", diff --git a/schemas/history_operations_schema.json b/schemas/history_operations_schema.json index ecf6191e..bd1678fb 100644 --- a/schemas/history_operations_schema.json +++ b/schemas/history_operations_schema.json @@ -968,6 +968,95 @@ "mode": "NULLABLE", "name": "shares", "type": "FLOAT" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "amount", + "type": "FLOAT" + }, + { + "mode": "NULLABLE", + "name": "asset_code", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_issuer", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "asset_type", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "from", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "to", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "type", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "asset_balance_changes", + "type": "RECORD" + }, + { + "fields": [ + { + "mode": "NULLABLE", + "name": "type", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "value", + "type": "STRING" + } + ], + "mode": "REPEATED", + "name": "parameters", + "type": "RECORD" + }, + { + "mode": "NULLABLE", + "name": "function", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "address", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "type", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "ledgers_to_expire", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "contract_id", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "contract_code_hash", + "type": "STRING" } ], "mode": "NULLABLE", diff --git a/schemas/history_transactions_schema.json b/schemas/history_transactions_schema.json index 5be534bb..ae61bcdd 100644 --- a/schemas/history_transactions_schema.json +++ b/schemas/history_transactions_schema.json @@ -149,6 +149,26 @@ "name": "extra_signers", "type": "string" }, + { + "mode": "NULLABLE", + "name": "refundable_fee", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "soroban_resources_instructions", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "soroban_resources_read_bytes", + "type": "INTEGER" + }, + { + "mode": "NULLABLE", + "name": "soroban_resources_write_bytes", + "type": "INTEGER" + }, { "mode": "NULLABLE", "name": "closed_at",