From 67b77fe7c5c8be8a7f2e52fb1a87c5c61593c829 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 25 Nov 2023 00:39:30 +0000 Subject: [PATCH] handing off to ridhi --- clustering_analysis.ipynb | 3862 ++++++++++++++----------------------- 1 file changed, 1412 insertions(+), 2450 deletions(-) diff --git a/clustering_analysis.ipynb b/clustering_analysis.ipynb index c1a1741..472e8d1 100644 --- a/clustering_analysis.ipynb +++ b/clustering_analysis.ipynb @@ -27,10 +27,269 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 1, "id": "241961fd-69dd-4036-839c-d5ff609e034a", "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ":: loading settings :: url = jar:file:/usr/lib/spark/jars/ivy-2.5.1.jar!/org/apache/ivy/core/settings/ivysettings.xml\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Ivy Default Cache set to: /root/.ivy2/cache\n", + "The jars for the packages stored in: /root/.ivy2/jars\n", + "com.johnsnowlabs.nlp#spark-nlp_2.12 added as a dependency\n", + "graphframes#graphframes added as a dependency\n", + ":: resolving dependencies :: org.apache.spark#spark-submit-parent-224c96f9-880e-4e05-9ecd-ae5053fbd4b8;1.0\n", + "\tconfs: [default]\n", + "\tfound com.johnsnowlabs.nlp#spark-nlp_2.12;4.4.0 in central\n", + "\tfound com.typesafe#config;1.4.2 in central\n", + "\tfound org.rocksdb#rocksdbjni;6.29.5 in central\n", + "\tfound com.amazonaws#aws-java-sdk-bundle;1.11.828 in central\n", + "\tfound com.github.universal-automata#liblevenshtein;3.0.0 in central\n", + "\tfound com.google.protobuf#protobuf-java-util;3.0.0-beta-3 in central\n", + "\tfound com.google.protobuf#protobuf-java;3.0.0-beta-3 in central\n", + "\tfound com.google.code.gson#gson;2.3 in central\n", + "\tfound it.unimi.dsi#fastutil;7.0.12 in central\n", + "\tfound org.projectlombok#lombok;1.16.8 in central\n", + "\tfound com.google.cloud#google-cloud-storage;2.16.0 in central\n", + "\tfound com.google.guava#guava;31.1-jre in central\n", + "\tfound com.google.guava#failureaccess;1.0.1 in central\n", + "\tfound com.google.guava#listenablefuture;9999.0-empty-to-avoid-conflict-with-guava in central\n", + "\tfound com.google.errorprone#error_prone_annotations;2.16 in central\n", + "\tfound com.google.j2objc#j2objc-annotations;1.3 in central\n", + "\tfound com.google.http-client#google-http-client;1.42.3 in central\n", + "\tfound io.opencensus#opencensus-contrib-http-util;0.31.1 in central\n", + "\tfound com.google.http-client#google-http-client-jackson2;1.42.3 in central\n", + "\tfound com.google.http-client#google-http-client-gson;1.42.3 in central\n", + "\tfound com.google.api-client#google-api-client;2.1.1 in central\n", + "\tfound commons-codec#commons-codec;1.15 in central\n", + "\tfound com.google.oauth-client#google-oauth-client;1.34.1 in central\n", + "\tfound com.google.http-client#google-http-client-apache-v2;1.42.3 in central\n", + "\tfound com.google.apis#google-api-services-storage;v1-rev20220705-2.0.0 in central\n", + "\tfound com.google.code.gson#gson;2.10 in central\n", + "\tfound com.google.cloud#google-cloud-core;2.9.0 in central\n", + "\tfound com.google.auto.value#auto-value-annotations;1.10.1 in central\n", + "\tfound com.google.cloud#google-cloud-core-http;2.9.0 in central\n", + "\tfound com.google.http-client#google-http-client-appengine;1.42.3 in central\n", + "\tfound com.google.api#gax-httpjson;0.105.1 in central\n", + "\tfound com.google.cloud#google-cloud-core-grpc;2.9.0 in central\n", + "\tfound io.grpc#grpc-core;1.51.0 in central\n", + "\tfound com.google.api#gax;2.20.1 in central\n", + "\tfound com.google.api#gax-grpc;2.20.1 in central\n", + "\tfound io.grpc#grpc-alts;1.51.0 in central\n", + "\tfound io.grpc#grpc-grpclb;1.51.0 in central\n", + "\tfound org.conscrypt#conscrypt-openjdk-uber;2.5.2 in central\n", + "\tfound io.grpc#grpc-protobuf;1.51.0 in central\n", + "\tfound com.google.auth#google-auth-library-credentials;1.13.0 in central\n", + "\tfound com.google.auth#google-auth-library-oauth2-http;1.13.0 in central\n", + "\tfound com.google.api#api-common;2.2.2 in central\n", + "\tfound javax.annotation#javax.annotation-api;1.3.2 in central\n", + "\tfound io.opencensus#opencensus-api;0.31.1 in central\n", + "\tfound io.grpc#grpc-context;1.51.0 in central\n", + "\tfound com.google.api.grpc#proto-google-iam-v1;1.6.22 in central\n", + "\tfound com.google.protobuf#protobuf-java;3.21.10 in central\n", + "\tfound com.google.protobuf#protobuf-java-util;3.21.10 in central\n", + "\tfound com.google.api.grpc#proto-google-common-protos;2.11.0 in central\n", + "\tfound org.threeten#threetenbp;1.6.4 in central\n", + "\tfound com.google.api.grpc#proto-google-cloud-storage-v2;2.16.0-alpha in central\n", + "\tfound com.google.api.grpc#grpc-google-cloud-storage-v2;2.16.0-alpha in central\n", + "\tfound com.google.api.grpc#gapic-google-cloud-storage-v2;2.16.0-alpha in central\n", + "\tfound com.fasterxml.jackson.core#jackson-core;2.14.1 in central\n", + "\tfound com.google.code.findbugs#jsr305;3.0.2 in central\n", + "\tfound io.grpc#grpc-api;1.51.0 in central\n", + "\tfound io.grpc#grpc-auth;1.51.0 in central\n", + "\tfound io.grpc#grpc-stub;1.51.0 in central\n", + "\tfound org.checkerframework#checker-qual;3.28.0 in central\n", + "\tfound com.google.api.grpc#grpc-google-iam-v1;1.6.22 in central\n", + "\tfound io.grpc#grpc-protobuf-lite;1.51.0 in central\n", + "\tfound com.google.android#annotations;4.1.1.4 in central\n", + "\tfound org.codehaus.mojo#animal-sniffer-annotations;1.22 in central\n", + "\tfound io.grpc#grpc-netty-shaded;1.51.0 in central\n", + "\tfound io.perfmark#perfmark-api;0.26.0 in central\n", + "\tfound io.grpc#grpc-googleapis;1.51.0 in central\n", + "\tfound io.grpc#grpc-xds;1.51.0 in central\n", + "\tfound io.opencensus#opencensus-proto;0.2.0 in central\n", + "\tfound io.grpc#grpc-services;1.51.0 in central\n", + "\tfound com.google.re2j#re2j;1.6 in central\n", + "\tfound com.navigamez#greex;1.0 in central\n", + "\tfound dk.brics.automaton#automaton;1.11-8 in central\n", + "\tfound com.johnsnowlabs.nlp#tensorflow-cpu_2.12;0.4.4 in central\n", + "\tfound graphframes#graphframes;0.8.2-spark3.1-s_2.12 in spark-packages\n", + "\tfound org.slf4j#slf4j-api;1.7.16 in central\n", + ":: resolution report :: resolve 1151ms :: artifacts dl 35ms\n", + "\t:: modules in use:\n", + "\tcom.amazonaws#aws-java-sdk-bundle;1.11.828 from central in [default]\n", + "\tcom.fasterxml.jackson.core#jackson-core;2.14.1 from central in [default]\n", + "\tcom.github.universal-automata#liblevenshtein;3.0.0 from central in [default]\n", + "\tcom.google.android#annotations;4.1.1.4 from central in [default]\n", + "\tcom.google.api#api-common;2.2.2 from central in [default]\n", + "\tcom.google.api#gax;2.20.1 from central in [default]\n", + "\tcom.google.api#gax-grpc;2.20.1 from central in [default]\n", + "\tcom.google.api#gax-httpjson;0.105.1 from central in [default]\n", + "\tcom.google.api-client#google-api-client;2.1.1 from central in [default]\n", + "\tcom.google.api.grpc#gapic-google-cloud-storage-v2;2.16.0-alpha from central in [default]\n", + "\tcom.google.api.grpc#grpc-google-cloud-storage-v2;2.16.0-alpha from central in [default]\n", + "\tcom.google.api.grpc#grpc-google-iam-v1;1.6.22 from central in [default]\n", + "\tcom.google.api.grpc#proto-google-cloud-storage-v2;2.16.0-alpha from central in [default]\n", + "\tcom.google.api.grpc#proto-google-common-protos;2.11.0 from central in [default]\n", + "\tcom.google.api.grpc#proto-google-iam-v1;1.6.22 from central in [default]\n", + "\tcom.google.apis#google-api-services-storage;v1-rev20220705-2.0.0 from central in [default]\n", + "\tcom.google.auth#google-auth-library-credentials;1.13.0 from central in [default]\n", + "\tcom.google.auth#google-auth-library-oauth2-http;1.13.0 from central in [default]\n", + "\tcom.google.auto.value#auto-value-annotations;1.10.1 from central in [default]\n", + "\tcom.google.cloud#google-cloud-core;2.9.0 from central in [default]\n", + "\tcom.google.cloud#google-cloud-core-grpc;2.9.0 from central in [default]\n", + "\tcom.google.cloud#google-cloud-core-http;2.9.0 from central in [default]\n", + "\tcom.google.cloud#google-cloud-storage;2.16.0 from central in [default]\n", + "\tcom.google.code.findbugs#jsr305;3.0.2 from central in [default]\n", + "\tcom.google.code.gson#gson;2.10 from central in [default]\n", + "\tcom.google.errorprone#error_prone_annotations;2.16 from central in [default]\n", + "\tcom.google.guava#failureaccess;1.0.1 from central in [default]\n", + "\tcom.google.guava#guava;31.1-jre from central in [default]\n", + "\tcom.google.guava#listenablefuture;9999.0-empty-to-avoid-conflict-with-guava from central in [default]\n", + "\tcom.google.http-client#google-http-client;1.42.3 from central in [default]\n", + "\tcom.google.http-client#google-http-client-apache-v2;1.42.3 from central in [default]\n", + "\tcom.google.http-client#google-http-client-appengine;1.42.3 from central in [default]\n", + "\tcom.google.http-client#google-http-client-gson;1.42.3 from central in [default]\n", + "\tcom.google.http-client#google-http-client-jackson2;1.42.3 from central in [default]\n", + "\tcom.google.j2objc#j2objc-annotations;1.3 from central in [default]\n", + "\tcom.google.oauth-client#google-oauth-client;1.34.1 from central in [default]\n", + "\tcom.google.protobuf#protobuf-java;3.21.10 from central in [default]\n", + "\tcom.google.protobuf#protobuf-java-util;3.21.10 from central in [default]\n", + "\tcom.google.re2j#re2j;1.6 from central in [default]\n", + "\tcom.johnsnowlabs.nlp#spark-nlp_2.12;4.4.0 from central in [default]\n", + "\tcom.johnsnowlabs.nlp#tensorflow-cpu_2.12;0.4.4 from central in [default]\n", + "\tcom.navigamez#greex;1.0 from central in [default]\n", + "\tcom.typesafe#config;1.4.2 from central in [default]\n", + "\tcommons-codec#commons-codec;1.15 from central in [default]\n", + "\tdk.brics.automaton#automaton;1.11-8 from central in [default]\n", + "\tgraphframes#graphframes;0.8.2-spark3.1-s_2.12 from spark-packages in [default]\n", + "\tio.grpc#grpc-alts;1.51.0 from central in [default]\n", + "\tio.grpc#grpc-api;1.51.0 from central in [default]\n", + "\tio.grpc#grpc-auth;1.51.0 from central in [default]\n", + "\tio.grpc#grpc-context;1.51.0 from central in [default]\n", + "\tio.grpc#grpc-core;1.51.0 from central in [default]\n", + "\tio.grpc#grpc-googleapis;1.51.0 from central in [default]\n", + "\tio.grpc#grpc-grpclb;1.51.0 from central in [default]\n", + "\tio.grpc#grpc-netty-shaded;1.51.0 from central in [default]\n", + "\tio.grpc#grpc-protobuf;1.51.0 from central in [default]\n", + "\tio.grpc#grpc-protobuf-lite;1.51.0 from central in [default]\n", + "\tio.grpc#grpc-services;1.51.0 from central in [default]\n", + "\tio.grpc#grpc-stub;1.51.0 from central in [default]\n", + "\tio.grpc#grpc-xds;1.51.0 from central in [default]\n", + "\tio.opencensus#opencensus-api;0.31.1 from central in [default]\n", + "\tio.opencensus#opencensus-contrib-http-util;0.31.1 from central in [default]\n", + "\tio.opencensus#opencensus-proto;0.2.0 from central in [default]\n", + "\tio.perfmark#perfmark-api;0.26.0 from central in [default]\n", + "\tit.unimi.dsi#fastutil;7.0.12 from central in [default]\n", + "\tjavax.annotation#javax.annotation-api;1.3.2 from central in [default]\n", + "\torg.checkerframework#checker-qual;3.28.0 from central in [default]\n", + "\torg.codehaus.mojo#animal-sniffer-annotations;1.22 from central in [default]\n", + "\torg.conscrypt#conscrypt-openjdk-uber;2.5.2 from central in [default]\n", + "\torg.projectlombok#lombok;1.16.8 from central in [default]\n", + "\torg.rocksdb#rocksdbjni;6.29.5 from central in [default]\n", + "\torg.slf4j#slf4j-api;1.7.16 from central in [default]\n", + "\torg.threeten#threetenbp;1.6.4 from central in [default]\n", + "\t:: evicted modules:\n", + "\tcom.google.protobuf#protobuf-java-util;3.0.0-beta-3 by [com.google.protobuf#protobuf-java-util;3.21.10] in [default]\n", + "\tcom.google.protobuf#protobuf-java;3.0.0-beta-3 by [com.google.protobuf#protobuf-java;3.21.10] in [default]\n", + "\tcom.google.code.gson#gson;2.3 by [com.google.code.gson#gson;2.10] in [default]\n", + "\t---------------------------------------------------------------------\n", + "\t| | modules || artifacts |\n", + "\t| conf | number| search|dwnlded|evicted|| number|dwnlded|\n", + "\t---------------------------------------------------------------------\n", + "\t| default | 75 | 0 | 0 | 3 || 72 | 0 |\n", + "\t---------------------------------------------------------------------\n", + ":: retrieving :: org.apache.spark#spark-submit-parent-224c96f9-880e-4e05-9ecd-ae5053fbd4b8\n", + "\tconfs: [default]\n", + "\t0 artifacts copied, 72 already retrieved (0kB/17ms)\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "23/11/24 23:34:31 INFO org.apache.spark.SparkEnv: Registering MapOutputTracker\n", + "23/11/24 23:34:31 INFO org.apache.spark.SparkEnv: Registering BlockManagerMaster\n", + "23/11/24 23:34:31 INFO org.apache.spark.SparkEnv: Registering BlockManagerMasterHeartbeat\n", + "23/11/24 23:34:31 INFO org.apache.spark.SparkEnv: Registering OutputCommitCoordinator\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.johnsnowlabs.nlp_spark-nlp_2.12-4.4.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/graphframes_graphframes-0.8.2-spark3.1-s_2.12.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.typesafe_config-1.4.2.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/org.rocksdb_rocksdbjni-6.29.5.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.amazonaws_aws-java-sdk-bundle-1.11.828.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.github.universal-automata_liblevenshtein-3.0.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.cloud_google-cloud-storage-2.16.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.navigamez_greex-1.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.johnsnowlabs.nlp_tensorflow-cpu_2.12-0.4.4.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/it.unimi.dsi_fastutil-7.0.12.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/org.projectlombok_lombok-1.16.8.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.guava_guava-31.1-jre.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.guava_failureaccess-1.0.1.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.16.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.j2objc_j2objc-annotations-1.3.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.http-client_google-http-client-1.42.3.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.opencensus_opencensus-contrib-http-util-0.31.1.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.http-client_google-http-client-jackson2-1.42.3.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.http-client_google-http-client-gson-1.42.3.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.api-client_google-api-client-2.1.1.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/commons-codec_commons-codec-1.15.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.oauth-client_google-oauth-client-1.34.1.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.http-client_google-http-client-apache-v2-1.42.3.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.apis_google-api-services-storage-v1-rev20220705-2.0.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.code.gson_gson-2.10.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.cloud_google-cloud-core-2.9.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.auto.value_auto-value-annotations-1.10.1.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.cloud_google-cloud-core-http-2.9.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.http-client_google-http-client-appengine-1.42.3.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.api_gax-httpjson-0.105.1.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.cloud_google-cloud-core-grpc-2.9.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-core-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.api_gax-2.20.1.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.api_gax-grpc-2.20.1.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-alts-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-grpclb-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/org.conscrypt_conscrypt-openjdk-uber-2.5.2.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-protobuf-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.auth_google-auth-library-credentials-1.13.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.auth_google-auth-library-oauth2-http-1.13.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.api_api-common-2.2.2.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/javax.annotation_javax.annotation-api-1.3.2.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.opencensus_opencensus-api-0.31.1.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-context-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.api.grpc_proto-google-iam-v1-1.6.22.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.protobuf_protobuf-java-3.21.10.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.protobuf_protobuf-java-util-3.21.10.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.api.grpc_proto-google-common-protos-2.11.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/org.threeten_threetenbp-1.6.4.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.api.grpc_proto-google-cloud-storage-v2-2.16.0-alpha.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.api.grpc_grpc-google-cloud-storage-v2-2.16.0-alpha.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.api.grpc_gapic-google-cloud-storage-v2-2.16.0-alpha.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.fasterxml.jackson.core_jackson-core-2.14.1.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.code.findbugs_jsr305-3.0.2.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-api-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-auth-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-stub-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/org.checkerframework_checker-qual-3.28.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.api.grpc_grpc-google-iam-v1-1.6.22.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-protobuf-lite-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.android_annotations-4.1.1.4.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/org.codehaus.mojo_animal-sniffer-annotations-1.22.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-netty-shaded-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.perfmark_perfmark-api-0.26.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-googleapis-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-xds-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.opencensus_opencensus-proto-0.2.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/io.grpc_grpc-services-1.51.0.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/com.google.re2j_re2j-1.6.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/dk.brics.automaton_automaton-1.11-8.jar added multiple times to distributed cache.\n", + "23/11/24 23:34:35 WARN org.apache.spark.deploy.yarn.Client: Same path resource file:///root/.ivy2/jars/org.slf4j_slf4j-api-1.7.16.jar added multiple times to distributed cache.\n" + ] + }, { "data": { "text/plain": [ @@ -39,22 +298,20 @@ " ('spark.eventLog.enabled', 'true'),\n", " ('spark.submit.pyFiles',\n", " '/root/.ivy2/jars/com.johnsnowlabs.nlp_spark-nlp_2.12-4.4.0.jar,/root/.ivy2/jars/graphframes_graphframes-0.8.2-spark3.1-s_2.12.jar,/root/.ivy2/jars/com.typesafe_config-1.4.2.jar,/root/.ivy2/jars/org.rocksdb_rocksdbjni-6.29.5.jar,/root/.ivy2/jars/com.amazonaws_aws-java-sdk-bundle-1.11.828.jar,/root/.ivy2/jars/com.github.universal-automata_liblevenshtein-3.0.0.jar,/root/.ivy2/jars/com.google.cloud_google-cloud-storage-2.16.0.jar,/root/.ivy2/jars/com.navigamez_greex-1.0.jar,/root/.ivy2/jars/com.johnsnowlabs.nlp_tensorflow-cpu_2.12-0.4.4.jar,/root/.ivy2/jars/it.unimi.dsi_fastutil-7.0.12.jar,/root/.ivy2/jars/org.projectlombok_lombok-1.16.8.jar,/root/.ivy2/jars/com.google.guava_guava-31.1-jre.jar,/root/.ivy2/jars/com.google.guava_failureaccess-1.0.1.jar,/root/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/root/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.16.jar,/root/.ivy2/jars/com.google.j2objc_j2objc-annotations-1.3.jar,/root/.ivy2/jars/com.google.http-client_google-http-client-1.42.3.jar,/root/.ivy2/jars/io.opencensus_opencensus-contrib-http-util-0.31.1.jar,/root/.ivy2/jars/com.google.http-client_google-http-client-jackson2-1.42.3.jar,/root/.ivy2/jars/com.google.http-client_google-http-client-gson-1.42.3.jar,/root/.ivy2/jars/com.google.api-client_google-api-client-2.1.1.jar,/root/.ivy2/jars/commons-codec_commons-codec-1.15.jar,/root/.ivy2/jars/com.google.oauth-client_google-oauth-client-1.34.1.jar,/root/.ivy2/jars/com.google.http-client_google-http-client-apache-v2-1.42.3.jar,/root/.ivy2/jars/com.google.apis_google-api-services-storage-v1-rev20220705-2.0.0.jar,/root/.ivy2/jars/com.google.code.gson_gson-2.10.jar,/root/.ivy2/jars/com.google.cloud_google-cloud-core-2.9.0.jar,/root/.ivy2/jars/com.google.auto.value_auto-value-annotations-1.10.1.jar,/root/.ivy2/jars/com.google.cloud_google-cloud-core-http-2.9.0.jar,/root/.ivy2/jars/com.google.http-client_google-http-client-appengine-1.42.3.jar,/root/.ivy2/jars/com.google.api_gax-httpjson-0.105.1.jar,/root/.ivy2/jars/com.google.cloud_google-cloud-core-grpc-2.9.0.jar,/root/.ivy2/jars/io.grpc_grpc-core-1.51.0.jar,/root/.ivy2/jars/com.google.api_gax-2.20.1.jar,/root/.ivy2/jars/com.google.api_gax-grpc-2.20.1.jar,/root/.ivy2/jars/io.grpc_grpc-alts-1.51.0.jar,/root/.ivy2/jars/io.grpc_grpc-grpclb-1.51.0.jar,/root/.ivy2/jars/org.conscrypt_conscrypt-openjdk-uber-2.5.2.jar,/root/.ivy2/jars/io.grpc_grpc-protobuf-1.51.0.jar,/root/.ivy2/jars/com.google.auth_google-auth-library-credentials-1.13.0.jar,/root/.ivy2/jars/com.google.auth_google-auth-library-oauth2-http-1.13.0.jar,/root/.ivy2/jars/com.google.api_api-common-2.2.2.jar,/root/.ivy2/jars/javax.annotation_javax.annotation-api-1.3.2.jar,/root/.ivy2/jars/io.opencensus_opencensus-api-0.31.1.jar,/root/.ivy2/jars/io.grpc_grpc-context-1.51.0.jar,/root/.ivy2/jars/com.google.api.grpc_proto-google-iam-v1-1.6.22.jar,/root/.ivy2/jars/com.google.protobuf_protobuf-java-3.21.10.jar,/root/.ivy2/jars/com.google.protobuf_protobuf-java-util-3.21.10.jar,/root/.ivy2/jars/com.google.api.grpc_proto-google-common-protos-2.11.0.jar,/root/.ivy2/jars/org.threeten_threetenbp-1.6.4.jar,/root/.ivy2/jars/com.google.api.grpc_proto-google-cloud-storage-v2-2.16.0-alpha.jar,/root/.ivy2/jars/com.google.api.grpc_grpc-google-cloud-storage-v2-2.16.0-alpha.jar,/root/.ivy2/jars/com.google.api.grpc_gapic-google-cloud-storage-v2-2.16.0-alpha.jar,/root/.ivy2/jars/com.fasterxml.jackson.core_jackson-core-2.14.1.jar,/root/.ivy2/jars/com.google.code.findbugs_jsr305-3.0.2.jar,/root/.ivy2/jars/io.grpc_grpc-api-1.51.0.jar,/root/.ivy2/jars/io.grpc_grpc-auth-1.51.0.jar,/root/.ivy2/jars/io.grpc_grpc-stub-1.51.0.jar,/root/.ivy2/jars/org.checkerframework_checker-qual-3.28.0.jar,/root/.ivy2/jars/com.google.api.grpc_grpc-google-iam-v1-1.6.22.jar,/root/.ivy2/jars/io.grpc_grpc-protobuf-lite-1.51.0.jar,/root/.ivy2/jars/com.google.android_annotations-4.1.1.4.jar,/root/.ivy2/jars/org.codehaus.mojo_animal-sniffer-annotations-1.22.jar,/root/.ivy2/jars/io.grpc_grpc-netty-shaded-1.51.0.jar,/root/.ivy2/jars/io.perfmark_perfmark-api-0.26.0.jar,/root/.ivy2/jars/io.grpc_grpc-googleapis-1.51.0.jar,/root/.ivy2/jars/io.grpc_grpc-xds-1.51.0.jar,/root/.ivy2/jars/io.opencensus_opencensus-proto-0.2.0.jar,/root/.ivy2/jars/io.grpc_grpc-services-1.51.0.jar,/root/.ivy2/jars/com.google.re2j_re2j-1.6.jar,/root/.ivy2/jars/dk.brics.automaton_automaton-1.11-8.jar,/root/.ivy2/jars/org.slf4j_slf4j-api-1.7.16.jar'),\n", - " ('spark.dataproc.metrics.listener.metrics.collector.hostname',\n", - " 'hub-msca-bdp-dphub-students-rohitk-m'),\n", - " ('spark.driver.port', '40031'),\n", " ('spark.dataproc.sql.joinConditionReorder.enabled', 'true'),\n", " ('spark.sql.autoBroadcastJoinThreshold', '191m'),\n", " ('spark.kryoserializer.buffer.max', '2000M'),\n", " ('spark.serializer', 'org.apache.spark.serializer.KryoSerializer'),\n", " ('spark.dataproc.sql.local.rank.pushdown.enabled', 'true'),\n", - " ('spark.ui.proxyBase', '/proxy/application_1700850934132_0002'),\n", - " ('spark.driver.host',\n", - " 'hub-msca-bdp-dphub-students-rohitk-m.c.msca-bdp-student-ap.internal'),\n", + " ('spark.driver.port', '41777'),\n", " ('spark.driver.maxResultSize', '0'),\n", " ('spark.yarn.unmanagedAM.enabled', 'true'),\n", + " ('spark.ui.proxyBase', '/proxy/application_1700868656493_0002'),\n", " ('spark.ui.filters',\n", " 'org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter'),\n", - " ('spark.app.startTime', '1700858434920'),\n", + " ('spark.app.startTime', '1700868871034'),\n", + " ('spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_URI_BASES',\n", + " 'http://hub-msca-bdp-dphub-students-test-rohitk-m:8088/proxy/application_1700868656493_0002'),\n", " ('spark.metrics.namespace',\n", " 'app_name:${spark.app.name}.app_id:${spark.app.id}'),\n", " ('spark.executor.memory', '4g'),\n", @@ -62,38 +319,29 @@ " ('spark.hadoop.hive.execution.engine', 'mr'),\n", " ('spark.executorEnv.PYTHONPATH',\n", " '{{PWD}}/pyspark.zip{{PWD}}/py4j-0.10.9-src.zip{{PWD}}/com.johnsnowlabs.nlp_spark-nlp_2.12-4.4.0.jar{{PWD}}/graphframes_graphframes-0.8.2-spark3.1-s_2.12.jar{{PWD}}/com.typesafe_config-1.4.2.jar{{PWD}}/org.rocksdb_rocksdbjni-6.29.5.jar{{PWD}}/com.amazonaws_aws-java-sdk-bundle-1.11.828.jar{{PWD}}/com.github.universal-automata_liblevenshtein-3.0.0.jar{{PWD}}/com.google.cloud_google-cloud-storage-2.16.0.jar{{PWD}}/com.navigamez_greex-1.0.jar{{PWD}}/com.johnsnowlabs.nlp_tensorflow-cpu_2.12-0.4.4.jar{{PWD}}/it.unimi.dsi_fastutil-7.0.12.jar{{PWD}}/org.projectlombok_lombok-1.16.8.jar{{PWD}}/com.google.guava_guava-31.1-jre.jar{{PWD}}/com.google.guava_failureaccess-1.0.1.jar{{PWD}}/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar{{PWD}}/com.google.errorprone_error_prone_annotations-2.16.jar{{PWD}}/com.google.j2objc_j2objc-annotations-1.3.jar{{PWD}}/com.google.http-client_google-http-client-1.42.3.jar{{PWD}}/io.opencensus_opencensus-contrib-http-util-0.31.1.jar{{PWD}}/com.google.http-client_google-http-client-jackson2-1.42.3.jar{{PWD}}/com.google.http-client_google-http-client-gson-1.42.3.jar{{PWD}}/com.google.api-client_google-api-client-2.1.1.jar{{PWD}}/commons-codec_commons-codec-1.15.jar{{PWD}}/com.google.oauth-client_google-oauth-client-1.34.1.jar{{PWD}}/com.google.http-client_google-http-client-apache-v2-1.42.3.jar{{PWD}}/com.google.apis_google-api-services-storage-v1-rev20220705-2.0.0.jar{{PWD}}/com.google.code.gson_gson-2.10.jar{{PWD}}/com.google.cloud_google-cloud-core-2.9.0.jar{{PWD}}/com.google.auto.value_auto-value-annotations-1.10.1.jar{{PWD}}/com.google.cloud_google-cloud-core-http-2.9.0.jar{{PWD}}/com.google.http-client_google-http-client-appengine-1.42.3.jar{{PWD}}/com.google.api_gax-httpjson-0.105.1.jar{{PWD}}/com.google.cloud_google-cloud-core-grpc-2.9.0.jar{{PWD}}/io.grpc_grpc-core-1.51.0.jar{{PWD}}/com.google.api_gax-2.20.1.jar{{PWD}}/com.google.api_gax-grpc-2.20.1.jar{{PWD}}/io.grpc_grpc-alts-1.51.0.jar{{PWD}}/io.grpc_grpc-grpclb-1.51.0.jar{{PWD}}/org.conscrypt_conscrypt-openjdk-uber-2.5.2.jar{{PWD}}/io.grpc_grpc-protobuf-1.51.0.jar{{PWD}}/com.google.auth_google-auth-library-credentials-1.13.0.jar{{PWD}}/com.google.auth_google-auth-library-oauth2-http-1.13.0.jar{{PWD}}/com.google.api_api-common-2.2.2.jar{{PWD}}/javax.annotation_javax.annotation-api-1.3.2.jar{{PWD}}/io.opencensus_opencensus-api-0.31.1.jar{{PWD}}/io.grpc_grpc-context-1.51.0.jar{{PWD}}/com.google.api.grpc_proto-google-iam-v1-1.6.22.jar{{PWD}}/com.google.protobuf_protobuf-java-3.21.10.jar{{PWD}}/com.google.protobuf_protobuf-java-util-3.21.10.jar{{PWD}}/com.google.api.grpc_proto-google-common-protos-2.11.0.jar{{PWD}}/org.threeten_threetenbp-1.6.4.jar{{PWD}}/com.google.api.grpc_proto-google-cloud-storage-v2-2.16.0-alpha.jar{{PWD}}/com.google.api.grpc_grpc-google-cloud-storage-v2-2.16.0-alpha.jar{{PWD}}/com.google.api.grpc_gapic-google-cloud-storage-v2-2.16.0-alpha.jar{{PWD}}/com.fasterxml.jackson.core_jackson-core-2.14.1.jar{{PWD}}/com.google.code.findbugs_jsr305-3.0.2.jar{{PWD}}/io.grpc_grpc-api-1.51.0.jar{{PWD}}/io.grpc_grpc-auth-1.51.0.jar{{PWD}}/io.grpc_grpc-stub-1.51.0.jar{{PWD}}/org.checkerframework_checker-qual-3.28.0.jar{{PWD}}/com.google.api.grpc_grpc-google-iam-v1-1.6.22.jar{{PWD}}/io.grpc_grpc-protobuf-lite-1.51.0.jar{{PWD}}/com.google.android_annotations-4.1.1.4.jar{{PWD}}/org.codehaus.mojo_animal-sniffer-annotations-1.22.jar{{PWD}}/io.grpc_grpc-netty-shaded-1.51.0.jar{{PWD}}/io.perfmark_perfmark-api-0.26.0.jar{{PWD}}/io.grpc_grpc-googleapis-1.51.0.jar{{PWD}}/io.grpc_grpc-xds-1.51.0.jar{{PWD}}/io.opencensus_opencensus-proto-0.2.0.jar{{PWD}}/io.grpc_grpc-services-1.51.0.jar{{PWD}}/com.google.re2j_re2j-1.6.jar{{PWD}}/dk.brics.automaton_automaton-1.11-8.jar{{PWD}}/org.slf4j_slf4j-api-1.7.16.jar'),\n", - " ('spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_HOSTS',\n", - " 'hub-msca-bdp-dphub-students-rohitk-m'),\n", " ('spark.executor.id', 'driver'),\n", " ('spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version', '2'),\n", " ('spark.dynamicAllocation.maxExecutors', '10000'),\n", + " ('spark.eventLog.dir',\n", + " 'gs://dataproc-temp-us-central1-635155370842-uzamlpgc/b1bf04a0-9010-4a11-a4e8-e9bc5a617d21/spark-job-history'),\n", " ('spark.yarn.dist.pyFiles',\n", " 'file:///root/.ivy2/jars/com.johnsnowlabs.nlp_spark-nlp_2.12-4.4.0.jar,file:///root/.ivy2/jars/graphframes_graphframes-0.8.2-spark3.1-s_2.12.jar,file:///root/.ivy2/jars/com.typesafe_config-1.4.2.jar,file:///root/.ivy2/jars/org.rocksdb_rocksdbjni-6.29.5.jar,file:///root/.ivy2/jars/com.amazonaws_aws-java-sdk-bundle-1.11.828.jar,file:///root/.ivy2/jars/com.github.universal-automata_liblevenshtein-3.0.0.jar,file:///root/.ivy2/jars/com.google.cloud_google-cloud-storage-2.16.0.jar,file:///root/.ivy2/jars/com.navigamez_greex-1.0.jar,file:///root/.ivy2/jars/com.johnsnowlabs.nlp_tensorflow-cpu_2.12-0.4.4.jar,file:///root/.ivy2/jars/it.unimi.dsi_fastutil-7.0.12.jar,file:///root/.ivy2/jars/org.projectlombok_lombok-1.16.8.jar,file:///root/.ivy2/jars/com.google.guava_guava-31.1-jre.jar,file:///root/.ivy2/jars/com.google.guava_failureaccess-1.0.1.jar,file:///root/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///root/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.16.jar,file:///root/.ivy2/jars/com.google.j2objc_j2objc-annotations-1.3.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-1.42.3.jar,file:///root/.ivy2/jars/io.opencensus_opencensus-contrib-http-util-0.31.1.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-jackson2-1.42.3.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-gson-1.42.3.jar,file:///root/.ivy2/jars/com.google.api-client_google-api-client-2.1.1.jar,file:///root/.ivy2/jars/commons-codec_commons-codec-1.15.jar,file:///root/.ivy2/jars/com.google.oauth-client_google-oauth-client-1.34.1.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-apache-v2-1.42.3.jar,file:///root/.ivy2/jars/com.google.apis_google-api-services-storage-v1-rev20220705-2.0.0.jar,file:///root/.ivy2/jars/com.google.code.gson_gson-2.10.jar,file:///root/.ivy2/jars/com.google.cloud_google-cloud-core-2.9.0.jar,file:///root/.ivy2/jars/com.google.auto.value_auto-value-annotations-1.10.1.jar,file:///root/.ivy2/jars/com.google.cloud_google-cloud-core-http-2.9.0.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-appengine-1.42.3.jar,file:///root/.ivy2/jars/com.google.api_gax-httpjson-0.105.1.jar,file:///root/.ivy2/jars/com.google.cloud_google-cloud-core-grpc-2.9.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-core-1.51.0.jar,file:///root/.ivy2/jars/com.google.api_gax-2.20.1.jar,file:///root/.ivy2/jars/com.google.api_gax-grpc-2.20.1.jar,file:///root/.ivy2/jars/io.grpc_grpc-alts-1.51.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-grpclb-1.51.0.jar,file:///root/.ivy2/jars/org.conscrypt_conscrypt-openjdk-uber-2.5.2.jar,file:///root/.ivy2/jars/io.grpc_grpc-protobuf-1.51.0.jar,file:///root/.ivy2/jars/com.google.auth_google-auth-library-credentials-1.13.0.jar,file:///root/.ivy2/jars/com.google.auth_google-auth-library-oauth2-http-1.13.0.jar,file:///root/.ivy2/jars/com.google.api_api-common-2.2.2.jar,file:///root/.ivy2/jars/javax.annotation_javax.annotation-api-1.3.2.jar,file:///root/.ivy2/jars/io.opencensus_opencensus-api-0.31.1.jar,file:///root/.ivy2/jars/io.grpc_grpc-context-1.51.0.jar,file:///root/.ivy2/jars/com.google.api.grpc_proto-google-iam-v1-1.6.22.jar,file:///root/.ivy2/jars/com.google.protobuf_protobuf-java-3.21.10.jar,file:///root/.ivy2/jars/com.google.protobuf_protobuf-java-util-3.21.10.jar,file:///root/.ivy2/jars/com.google.api.grpc_proto-google-common-protos-2.11.0.jar,file:///root/.ivy2/jars/org.threeten_threetenbp-1.6.4.jar,file:///root/.ivy2/jars/com.google.api.grpc_proto-google-cloud-storage-v2-2.16.0-alpha.jar,file:///root/.ivy2/jars/com.google.api.grpc_grpc-google-cloud-storage-v2-2.16.0-alpha.jar,file:///root/.ivy2/jars/com.google.api.grpc_gapic-google-cloud-storage-v2-2.16.0-alpha.jar,file:///root/.ivy2/jars/com.fasterxml.jackson.core_jackson-core-2.14.1.jar,file:///root/.ivy2/jars/com.google.code.findbugs_jsr305-3.0.2.jar,file:///root/.ivy2/jars/io.grpc_grpc-api-1.51.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-auth-1.51.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-stub-1.51.0.jar,file:///root/.ivy2/jars/org.checkerframework_checker-qual-3.28.0.jar,file:///root/.ivy2/jars/com.google.api.grpc_grpc-google-iam-v1-1.6.22.jar,file:///root/.ivy2/jars/io.grpc_grpc-protobuf-lite-1.51.0.jar,file:///root/.ivy2/jars/com.google.android_annotations-4.1.1.4.jar,file:///root/.ivy2/jars/org.codehaus.mojo_animal-sniffer-annotations-1.22.jar,file:///root/.ivy2/jars/io.grpc_grpc-netty-shaded-1.51.0.jar,file:///root/.ivy2/jars/io.perfmark_perfmark-api-0.26.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-googleapis-1.51.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-xds-1.51.0.jar,file:///root/.ivy2/jars/io.opencensus_opencensus-proto-0.2.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-services-1.51.0.jar,file:///root/.ivy2/jars/com.google.re2j_re2j-1.6.jar,file:///root/.ivy2/jars/dk.brics.automaton_automaton-1.11-8.jar,file:///root/.ivy2/jars/org.slf4j_slf4j-api-1.7.16.jar'),\n", " ('spark.yarn.am.attemptFailuresValidityInterval', '1h'),\n", " ('spark.app.name', 'Spark Updated Conf'),\n", " ('spark.sql.catalogImplementation', 'hive'),\n", - " ('spark.driver.appUIAddress',\n", - " 'http://hub-msca-bdp-dphub-students-rohitk-m.c.msca-bdp-student-ap.internal:33679'),\n", " ('spark.executorEnv.OPENBLAS_NUM_THREADS', '1'),\n", " ('spark.yarn.secondary.jars',\n", " 'com.johnsnowlabs.nlp_spark-nlp_2.12-4.4.0.jar,graphframes_graphframes-0.8.2-spark3.1-s_2.12.jar,com.typesafe_config-1.4.2.jar,org.rocksdb_rocksdbjni-6.29.5.jar,com.amazonaws_aws-java-sdk-bundle-1.11.828.jar,com.github.universal-automata_liblevenshtein-3.0.0.jar,com.google.cloud_google-cloud-storage-2.16.0.jar,com.navigamez_greex-1.0.jar,com.johnsnowlabs.nlp_tensorflow-cpu_2.12-0.4.4.jar,it.unimi.dsi_fastutil-7.0.12.jar,org.projectlombok_lombok-1.16.8.jar,com.google.guava_guava-31.1-jre.jar,com.google.guava_failureaccess-1.0.1.jar,com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,com.google.errorprone_error_prone_annotations-2.16.jar,com.google.j2objc_j2objc-annotations-1.3.jar,com.google.http-client_google-http-client-1.42.3.jar,io.opencensus_opencensus-contrib-http-util-0.31.1.jar,com.google.http-client_google-http-client-jackson2-1.42.3.jar,com.google.http-client_google-http-client-gson-1.42.3.jar,com.google.api-client_google-api-client-2.1.1.jar,commons-codec_commons-codec-1.15.jar,com.google.oauth-client_google-oauth-client-1.34.1.jar,com.google.http-client_google-http-client-apache-v2-1.42.3.jar,com.google.apis_google-api-services-storage-v1-rev20220705-2.0.0.jar,com.google.code.gson_gson-2.10.jar,com.google.cloud_google-cloud-core-2.9.0.jar,com.google.auto.value_auto-value-annotations-1.10.1.jar,com.google.cloud_google-cloud-core-http-2.9.0.jar,com.google.http-client_google-http-client-appengine-1.42.3.jar,com.google.api_gax-httpjson-0.105.1.jar,com.google.cloud_google-cloud-core-grpc-2.9.0.jar,io.grpc_grpc-core-1.51.0.jar,com.google.api_gax-2.20.1.jar,com.google.api_gax-grpc-2.20.1.jar,io.grpc_grpc-alts-1.51.0.jar,io.grpc_grpc-grpclb-1.51.0.jar,org.conscrypt_conscrypt-openjdk-uber-2.5.2.jar,io.grpc_grpc-protobuf-1.51.0.jar,com.google.auth_google-auth-library-credentials-1.13.0.jar,com.google.auth_google-auth-library-oauth2-http-1.13.0.jar,com.google.api_api-common-2.2.2.jar,javax.annotation_javax.annotation-api-1.3.2.jar,io.opencensus_opencensus-api-0.31.1.jar,io.grpc_grpc-context-1.51.0.jar,com.google.api.grpc_proto-google-iam-v1-1.6.22.jar,com.google.protobuf_protobuf-java-3.21.10.jar,com.google.protobuf_protobuf-java-util-3.21.10.jar,com.google.api.grpc_proto-google-common-protos-2.11.0.jar,org.threeten_threetenbp-1.6.4.jar,com.google.api.grpc_proto-google-cloud-storage-v2-2.16.0-alpha.jar,com.google.api.grpc_grpc-google-cloud-storage-v2-2.16.0-alpha.jar,com.google.api.grpc_gapic-google-cloud-storage-v2-2.16.0-alpha.jar,com.fasterxml.jackson.core_jackson-core-2.14.1.jar,com.google.code.findbugs_jsr305-3.0.2.jar,io.grpc_grpc-api-1.51.0.jar,io.grpc_grpc-auth-1.51.0.jar,io.grpc_grpc-stub-1.51.0.jar,org.checkerframework_checker-qual-3.28.0.jar,com.google.api.grpc_grpc-google-iam-v1-1.6.22.jar,io.grpc_grpc-protobuf-lite-1.51.0.jar,com.google.android_annotations-4.1.1.4.jar,org.codehaus.mojo_animal-sniffer-annotations-1.22.jar,io.grpc_grpc-netty-shaded-1.51.0.jar,io.perfmark_perfmark-api-0.26.0.jar,io.grpc_grpc-googleapis-1.51.0.jar,io.grpc_grpc-xds-1.51.0.jar,io.opencensus_opencensus-proto-0.2.0.jar,io.grpc_grpc-services-1.51.0.jar,com.google.re2j_re2j-1.6.jar,dk.brics.automaton_automaton-1.11-8.jar,org.slf4j_slf4j-api-1.7.16.jar'),\n", - " ('spark.history.fs.logDirectory',\n", - " 'gs://dataproc-temp-us-central1-635155370842-uzamlpgc/6e7bc176-17b3-42b1-8449-77c1f54798f9/spark-job-history'),\n", " ('spark.repl.local.jars',\n", " 'file:///root/.ivy2/jars/com.johnsnowlabs.nlp_spark-nlp_2.12-4.4.0.jar,file:///root/.ivy2/jars/graphframes_graphframes-0.8.2-spark3.1-s_2.12.jar,file:///root/.ivy2/jars/com.typesafe_config-1.4.2.jar,file:///root/.ivy2/jars/org.rocksdb_rocksdbjni-6.29.5.jar,file:///root/.ivy2/jars/com.amazonaws_aws-java-sdk-bundle-1.11.828.jar,file:///root/.ivy2/jars/com.github.universal-automata_liblevenshtein-3.0.0.jar,file:///root/.ivy2/jars/com.google.cloud_google-cloud-storage-2.16.0.jar,file:///root/.ivy2/jars/com.navigamez_greex-1.0.jar,file:///root/.ivy2/jars/com.johnsnowlabs.nlp_tensorflow-cpu_2.12-0.4.4.jar,file:///root/.ivy2/jars/it.unimi.dsi_fastutil-7.0.12.jar,file:///root/.ivy2/jars/org.projectlombok_lombok-1.16.8.jar,file:///root/.ivy2/jars/com.google.guava_guava-31.1-jre.jar,file:///root/.ivy2/jars/com.google.guava_failureaccess-1.0.1.jar,file:///root/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///root/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.16.jar,file:///root/.ivy2/jars/com.google.j2objc_j2objc-annotations-1.3.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-1.42.3.jar,file:///root/.ivy2/jars/io.opencensus_opencensus-contrib-http-util-0.31.1.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-jackson2-1.42.3.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-gson-1.42.3.jar,file:///root/.ivy2/jars/com.google.api-client_google-api-client-2.1.1.jar,file:///root/.ivy2/jars/commons-codec_commons-codec-1.15.jar,file:///root/.ivy2/jars/com.google.oauth-client_google-oauth-client-1.34.1.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-apache-v2-1.42.3.jar,file:///root/.ivy2/jars/com.google.apis_google-api-services-storage-v1-rev20220705-2.0.0.jar,file:///root/.ivy2/jars/com.google.code.gson_gson-2.10.jar,file:///root/.ivy2/jars/com.google.cloud_google-cloud-core-2.9.0.jar,file:///root/.ivy2/jars/com.google.auto.value_auto-value-annotations-1.10.1.jar,file:///root/.ivy2/jars/com.google.cloud_google-cloud-core-http-2.9.0.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-appengine-1.42.3.jar,file:///root/.ivy2/jars/com.google.api_gax-httpjson-0.105.1.jar,file:///root/.ivy2/jars/com.google.cloud_google-cloud-core-grpc-2.9.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-core-1.51.0.jar,file:///root/.ivy2/jars/com.google.api_gax-2.20.1.jar,file:///root/.ivy2/jars/com.google.api_gax-grpc-2.20.1.jar,file:///root/.ivy2/jars/io.grpc_grpc-alts-1.51.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-grpclb-1.51.0.jar,file:///root/.ivy2/jars/org.conscrypt_conscrypt-openjdk-uber-2.5.2.jar,file:///root/.ivy2/jars/io.grpc_grpc-protobuf-1.51.0.jar,file:///root/.ivy2/jars/com.google.auth_google-auth-library-credentials-1.13.0.jar,file:///root/.ivy2/jars/com.google.auth_google-auth-library-oauth2-http-1.13.0.jar,file:///root/.ivy2/jars/com.google.api_api-common-2.2.2.jar,file:///root/.ivy2/jars/javax.annotation_javax.annotation-api-1.3.2.jar,file:///root/.ivy2/jars/io.opencensus_opencensus-api-0.31.1.jar,file:///root/.ivy2/jars/io.grpc_grpc-context-1.51.0.jar,file:///root/.ivy2/jars/com.google.api.grpc_proto-google-iam-v1-1.6.22.jar,file:///root/.ivy2/jars/com.google.protobuf_protobuf-java-3.21.10.jar,file:///root/.ivy2/jars/com.google.protobuf_protobuf-java-util-3.21.10.jar,file:///root/.ivy2/jars/com.google.api.grpc_proto-google-common-protos-2.11.0.jar,file:///root/.ivy2/jars/org.threeten_threetenbp-1.6.4.jar,file:///root/.ivy2/jars/com.google.api.grpc_proto-google-cloud-storage-v2-2.16.0-alpha.jar,file:///root/.ivy2/jars/com.google.api.grpc_grpc-google-cloud-storage-v2-2.16.0-alpha.jar,file:///root/.ivy2/jars/com.google.api.grpc_gapic-google-cloud-storage-v2-2.16.0-alpha.jar,file:///root/.ivy2/jars/com.fasterxml.jackson.core_jackson-core-2.14.1.jar,file:///root/.ivy2/jars/com.google.code.findbugs_jsr305-3.0.2.jar,file:///root/.ivy2/jars/io.grpc_grpc-api-1.51.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-auth-1.51.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-stub-1.51.0.jar,file:///root/.ivy2/jars/org.checkerframework_checker-qual-3.28.0.jar,file:///root/.ivy2/jars/com.google.api.grpc_grpc-google-iam-v1-1.6.22.jar,file:///root/.ivy2/jars/io.grpc_grpc-protobuf-lite-1.51.0.jar,file:///root/.ivy2/jars/com.google.android_annotations-4.1.1.4.jar,file:///root/.ivy2/jars/org.codehaus.mojo_animal-sniffer-annotations-1.22.jar,file:///root/.ivy2/jars/io.grpc_grpc-netty-shaded-1.51.0.jar,file:///root/.ivy2/jars/io.perfmark_perfmark-api-0.26.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-googleapis-1.51.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-xds-1.51.0.jar,file:///root/.ivy2/jars/io.opencensus_opencensus-proto-0.2.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-services-1.51.0.jar,file:///root/.ivy2/jars/com.google.re2j_re2j-1.6.jar,file:///root/.ivy2/jars/dk.brics.automaton_automaton-1.11-8.jar,file:///root/.ivy2/jars/org.slf4j_slf4j-api-1.7.16.jar'),\n", " ('spark.sql.cbo.enabled', 'true'),\n", - " ('spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_URI_BASES',\n", - " 'http://hub-msca-bdp-dphub-students-rohitk-m:8088/proxy/application_1700850934132_0002'),\n", " ('spark.yarn.dist.jars',\n", " 'file:///root/.ivy2/jars/com.johnsnowlabs.nlp_spark-nlp_2.12-4.4.0.jar,file:///root/.ivy2/jars/graphframes_graphframes-0.8.2-spark3.1-s_2.12.jar,file:///root/.ivy2/jars/com.typesafe_config-1.4.2.jar,file:///root/.ivy2/jars/org.rocksdb_rocksdbjni-6.29.5.jar,file:///root/.ivy2/jars/com.amazonaws_aws-java-sdk-bundle-1.11.828.jar,file:///root/.ivy2/jars/com.github.universal-automata_liblevenshtein-3.0.0.jar,file:///root/.ivy2/jars/com.google.cloud_google-cloud-storage-2.16.0.jar,file:///root/.ivy2/jars/com.navigamez_greex-1.0.jar,file:///root/.ivy2/jars/com.johnsnowlabs.nlp_tensorflow-cpu_2.12-0.4.4.jar,file:///root/.ivy2/jars/it.unimi.dsi_fastutil-7.0.12.jar,file:///root/.ivy2/jars/org.projectlombok_lombok-1.16.8.jar,file:///root/.ivy2/jars/com.google.guava_guava-31.1-jre.jar,file:///root/.ivy2/jars/com.google.guava_failureaccess-1.0.1.jar,file:///root/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///root/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.16.jar,file:///root/.ivy2/jars/com.google.j2objc_j2objc-annotations-1.3.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-1.42.3.jar,file:///root/.ivy2/jars/io.opencensus_opencensus-contrib-http-util-0.31.1.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-jackson2-1.42.3.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-gson-1.42.3.jar,file:///root/.ivy2/jars/com.google.api-client_google-api-client-2.1.1.jar,file:///root/.ivy2/jars/commons-codec_commons-codec-1.15.jar,file:///root/.ivy2/jars/com.google.oauth-client_google-oauth-client-1.34.1.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-apache-v2-1.42.3.jar,file:///root/.ivy2/jars/com.google.apis_google-api-services-storage-v1-rev20220705-2.0.0.jar,file:///root/.ivy2/jars/com.google.code.gson_gson-2.10.jar,file:///root/.ivy2/jars/com.google.cloud_google-cloud-core-2.9.0.jar,file:///root/.ivy2/jars/com.google.auto.value_auto-value-annotations-1.10.1.jar,file:///root/.ivy2/jars/com.google.cloud_google-cloud-core-http-2.9.0.jar,file:///root/.ivy2/jars/com.google.http-client_google-http-client-appengine-1.42.3.jar,file:///root/.ivy2/jars/com.google.api_gax-httpjson-0.105.1.jar,file:///root/.ivy2/jars/com.google.cloud_google-cloud-core-grpc-2.9.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-core-1.51.0.jar,file:///root/.ivy2/jars/com.google.api_gax-2.20.1.jar,file:///root/.ivy2/jars/com.google.api_gax-grpc-2.20.1.jar,file:///root/.ivy2/jars/io.grpc_grpc-alts-1.51.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-grpclb-1.51.0.jar,file:///root/.ivy2/jars/org.conscrypt_conscrypt-openjdk-uber-2.5.2.jar,file:///root/.ivy2/jars/io.grpc_grpc-protobuf-1.51.0.jar,file:///root/.ivy2/jars/com.google.auth_google-auth-library-credentials-1.13.0.jar,file:///root/.ivy2/jars/com.google.auth_google-auth-library-oauth2-http-1.13.0.jar,file:///root/.ivy2/jars/com.google.api_api-common-2.2.2.jar,file:///root/.ivy2/jars/javax.annotation_javax.annotation-api-1.3.2.jar,file:///root/.ivy2/jars/io.opencensus_opencensus-api-0.31.1.jar,file:///root/.ivy2/jars/io.grpc_grpc-context-1.51.0.jar,file:///root/.ivy2/jars/com.google.api.grpc_proto-google-iam-v1-1.6.22.jar,file:///root/.ivy2/jars/com.google.protobuf_protobuf-java-3.21.10.jar,file:///root/.ivy2/jars/com.google.protobuf_protobuf-java-util-3.21.10.jar,file:///root/.ivy2/jars/com.google.api.grpc_proto-google-common-protos-2.11.0.jar,file:///root/.ivy2/jars/org.threeten_threetenbp-1.6.4.jar,file:///root/.ivy2/jars/com.google.api.grpc_proto-google-cloud-storage-v2-2.16.0-alpha.jar,file:///root/.ivy2/jars/com.google.api.grpc_grpc-google-cloud-storage-v2-2.16.0-alpha.jar,file:///root/.ivy2/jars/com.google.api.grpc_gapic-google-cloud-storage-v2-2.16.0-alpha.jar,file:///root/.ivy2/jars/com.fasterxml.jackson.core_jackson-core-2.14.1.jar,file:///root/.ivy2/jars/com.google.code.findbugs_jsr305-3.0.2.jar,file:///root/.ivy2/jars/io.grpc_grpc-api-1.51.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-auth-1.51.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-stub-1.51.0.jar,file:///root/.ivy2/jars/org.checkerframework_checker-qual-3.28.0.jar,file:///root/.ivy2/jars/com.google.api.grpc_grpc-google-iam-v1-1.6.22.jar,file:///root/.ivy2/jars/io.grpc_grpc-protobuf-lite-1.51.0.jar,file:///root/.ivy2/jars/com.google.android_annotations-4.1.1.4.jar,file:///root/.ivy2/jars/org.codehaus.mojo_animal-sniffer-annotations-1.22.jar,file:///root/.ivy2/jars/io.grpc_grpc-netty-shaded-1.51.0.jar,file:///root/.ivy2/jars/io.perfmark_perfmark-api-0.26.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-googleapis-1.51.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-xds-1.51.0.jar,file:///root/.ivy2/jars/io.opencensus_opencensus-proto-0.2.0.jar,file:///root/.ivy2/jars/io.grpc_grpc-services-1.51.0.jar,file:///root/.ivy2/jars/com.google.re2j_re2j-1.6.jar,file:///root/.ivy2/jars/dk.brics.automaton_automaton-1.11-8.jar,file:///root/.ivy2/jars/org.slf4j_slf4j-api-1.7.16.jar'),\n", " ('spark.dataproc.sql.parquet.enableFooterCache', 'true'),\n", " ('spark.driver.memory', '4g'),\n", " ('spark.sql.warehouse.dir', 'file:/spark-warehouse'),\n", - " ('spark.app.id', 'application_1700850934132_0002'),\n", " ('spark.yarn.executor.failuresValidityInterval', '1h'),\n", " ('spark.yarn.am.memory', '640m'),\n", - " ('spark.yarn.historyServer.address',\n", - " 'hub-msca-bdp-dphub-students-rohitk-m:18080'),\n", " ('spark.cores.max', '4'),\n", " ('spark.executor.cores', '4'),\n", " ('spark.jars.packages',\n", @@ -102,17 +350,28 @@ " ('spark.dataproc.listeners',\n", " 'com.google.cloud.spark.performance.DataprocMetricsListener'),\n", " ('spark.serializer.objectStreamReset', '100'),\n", - " ('spark.eventLog.dir',\n", - " 'gs://dataproc-temp-us-central1-635155370842-uzamlpgc/6e7bc176-17b3-42b1-8449-77c1f54798f9/spark-job-history'),\n", + " ('spark.history.fs.logDirectory',\n", + " 'gs://dataproc-temp-us-central1-635155370842-uzamlpgc/b1bf04a0-9010-4a11-a4e8-e9bc5a617d21/spark-job-history'),\n", " ('spark.submit.deployMode', 'client'),\n", + " ('spark.driver.host',\n", + " 'hub-msca-bdp-dphub-students-test-rohitk-m.c.msca-bdp-student-ap.internal'),\n", + " ('spark.yarn.historyServer.address',\n", + " 'hub-msca-bdp-dphub-students-test-rohitk-m:18080'),\n", " ('spark.sql.cbo.joinReorder.enabled', 'true'),\n", " ('spark.shuffle.service.enabled', 'true'),\n", + " ('spark.dataproc.metrics.listener.metrics.collector.hostname',\n", + " 'hub-msca-bdp-dphub-students-test-rohitk-m'),\n", " ('spark.scheduler.mode', 'FAIR'),\n", " ('spark.sql.adaptive.enabled', 'true'),\n", " ('spark.yarn.jars', 'local:/usr/lib/spark/jars/*'),\n", " ('spark.scheduler.minRegisteredResourcesRatio', '0.0'),\n", + " ('spark.driver.appUIAddress',\n", + " 'http://hub-msca-bdp-dphub-students-test-rohitk-m.c.msca-bdp-student-ap.internal:41013'),\n", + " ('spark.app.id', 'application_1700868656493_0002'),\n", " ('spark.master', 'yarn'),\n", " ('spark.ui.port', '0'),\n", + " ('spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_HOSTS',\n", + " 'hub-msca-bdp-dphub-students-test-rohitk-m'),\n", " ('spark.rpc.message.maxSize', '512'),\n", " ('spark.rdd.compress', 'True'),\n", " ('spark.task.maxFailures', '10'),\n", @@ -121,7 +380,7 @@ " ('spark.ui.showConsoleProgress', 'true')]" ] }, - "execution_count": 15, + "execution_count": 1, "metadata": {}, "output_type": "execute_result" } @@ -214,14 +473,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "Partitions: 544\n" + "Partitions: 534\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 16:=====================================================>(543 + 1) / 544]\r" + "[Stage 16:=====================================================>(532 + 2) / 534]\r" ] }, { @@ -231,327 +490,307 @@ "+-----------+------+\n", "|partitionId| count|\n", "+-----------+------+\n", - "| 42|305254|\n", - "| 41|305316|\n", - "| 40|305420|\n", - "| 38|305471|\n", - "| 39|305480|\n", - "| 37|305618|\n", - "| 36|305676|\n", - "| 35|305871|\n", - "| 34|305890|\n", - "| 33|305962|\n", - "| 32|305971|\n", - "| 31|306010|\n", - "| 29|306031|\n", - "| 30|306038|\n", - "| 28|306086|\n", - "| 27|306127|\n", - "| 26|306402|\n", - "| 25|306467|\n", - "| 24|306633|\n", - "| 23|306731|\n", - "| 22|307226|\n", - "| 243|328837|\n", - "| 242|328975|\n", - "| 241|329131|\n", - "| 240|329163|\n", - "| 239|329209|\n", - "| 237|329245|\n", - "| 235|329263|\n", - "| 238|329263|\n", - "| 234|329311|\n", - "| 236|329315|\n", - "| 232|329332|\n", - "| 233|329344|\n", - "| 231|329373|\n", - "| 228|329389|\n", - "| 229|329390|\n", - "| 227|329399|\n", - "| 225|329410|\n", - "| 226|329410|\n", - "| 224|329418|\n", - "| 230|329427|\n", - "| 223|329428|\n", - "| 220|329461|\n", - "| 222|329481|\n", - "| 221|329505|\n", - "| 217|329507|\n", - "| 218|329513|\n", - "| 219|329519|\n", - "| 216|329523|\n", - "| 214|329533|\n", - "| 213|329555|\n", - "| 215|329574|\n", - "| 211|329587|\n", - "| 212|329591|\n", - "| 208|329607|\n", - "| 210|329623|\n", - "| 206|329624|\n", - "| 209|329630|\n", - "| 207|329633|\n", - "| 205|329646|\n", - "| 202|329654|\n", - "| 204|329673|\n", - "| 203|329678|\n", - "| 194|329704|\n", - "| 201|329708|\n", - "| 200|329712|\n", - "| 191|329717|\n", - "| 189|329728|\n", - "| 188|329730|\n", - "| 199|329732|\n", - "| 193|329732|\n", - "| 198|329739|\n", - "| 190|329746|\n", - "| 195|329748|\n", - "| 196|329786|\n", + "| 33|152646|\n", + "| 233|328837|\n", + "| 232|328975|\n", + "| 231|329131|\n", + "| 230|329163|\n", + "| 229|329209|\n", + "| 227|329245|\n", + "| 228|329263|\n", + "| 225|329263|\n", + "| 224|329311|\n", + "| 226|329315|\n", + "| 222|329332|\n", + "| 223|329344|\n", + "| 221|329373|\n", + "| 218|329389|\n", + "| 219|329390|\n", + "| 217|329399|\n", + "| 215|329410|\n", + "| 216|329410|\n", + "| 214|329418|\n", + "| 220|329427|\n", + "| 213|329428|\n", + "| 210|329461|\n", + "| 212|329481|\n", + "| 211|329505|\n", + "| 207|329507|\n", + "| 208|329513|\n", + "| 209|329519|\n", + "| 206|329523|\n", + "| 204|329533|\n", + "| 203|329555|\n", + "| 205|329574|\n", + "| 201|329587|\n", + "| 202|329591|\n", + "| 198|329607|\n", + "| 200|329623|\n", + "| 196|329624|\n", + "| 199|329630|\n", + "| 197|329633|\n", + "| 195|329646|\n", + "| 192|329654|\n", + "| 194|329673|\n", + "| 193|329678|\n", + "| 184|329704|\n", + "| 191|329708|\n", + "| 190|329712|\n", + "| 181|329717|\n", + "| 179|329728|\n", + "| 178|329730|\n", + "| 183|329732|\n", + "| 189|329732|\n", + "| 188|329739|\n", + "| 180|329746|\n", + "| 185|329748|\n", "| 186|329786|\n", - "| 197|329786|\n", - "| 187|329787|\n", - "| 192|329793|\n", - "| 183|329804|\n", - "| 184|329805|\n", - "| 179|329807|\n", - "| 182|329814|\n", - "| 181|329819|\n", - "| 185|329843|\n", - "| 178|329849|\n", - "| 177|329861|\n", - "| 180|329865|\n", - "| 172|329866|\n", - "| 173|329871|\n", - "| 171|329879|\n", - "| 175|329889|\n", - "| 174|329892|\n", - "| 176|329900|\n", - "| 170|329937|\n", - "| 168|329956|\n", - "| 162|329961|\n", - "| 167|329962|\n", - "| 164|329965|\n", - "| 169|329974|\n", - "| 165|329980|\n", - "| 163|329985|\n", - "| 159|329993|\n", - "| 166|330000|\n", - "| 158|330005|\n", - "| 161|330014|\n", - "| 157|330021|\n", - "| 160|330027|\n", - "| 156|330059|\n", - "| 154|330066|\n", - "| 147|330070|\n", - "| 153|330083|\n", - "| 155|330084|\n", - "| 150|330084|\n", - "| 146|330089|\n", - "| 152|330090|\n", - "| 149|330099|\n", - "| 151|330101|\n", - "| 148|330103|\n", - "| 144|330116|\n", - "| 145|330129|\n", - "| 140|330133|\n", - "| 143|330135|\n", - "| 141|330139|\n", - "| 142|330148|\n", - "| 139|330159|\n", - "| 135|330178|\n", - "| 137|330181|\n", - "| 132|330186|\n", - "| 138|330189|\n", - "| 136|330205|\n", - "| 129|330208|\n", - "| 133|330219|\n", - "| 125|330220|\n", - "| 128|330227|\n", - "| 134|330230|\n", - "| 127|330236|\n", - "| 130|330243|\n", - "| 124|330245|\n", - "| 117|330249|\n", - "| 131|330252|\n", - "| 121|330255|\n", - "| 122|330258|\n", - "| 126|330268|\n", - "| 123|330270|\n", - "| 118|330275|\n", - "| 113|330282|\n", - "| 115|330282|\n", - "| 120|330296|\n", - "| 112|330297|\n", - "| 119|330306|\n", - "| 114|330308|\n", - "| 104|330338|\n", - "| 116|330339|\n", - "| 109|330355|\n", - "| 105|330363|\n", - "| 111|330366|\n", - "| 110|330372|\n", - "| 108|330383|\n", - "| 107|330385|\n", - "| 106|330402|\n", - "| 96|330413|\n", - "| 102|330415|\n", - "| 100|330418|\n", - "| 98|330418|\n", - "| 101|330419|\n", - "| 94|330425|\n", - "| 97|330426|\n", - "| 103|330430|\n", - "| 99|330430|\n", - "| 95|330436|\n", - "| 90|330456|\n", - "| 92|330470|\n", - "| 88|330471|\n", - "| 91|330474|\n", - "| 93|330477|\n", - "| 87|330494|\n", - "| 86|330501|\n", - "| 89|330512|\n", - "| 84|330524|\n", - "| 82|330550|\n", - "| 85|330570|\n", - "| 80|330570|\n", - "| 81|330579|\n", - "| 83|330585|\n", - "| 78|330622|\n", - "| 79|330625|\n", - "| 76|330625|\n", - "| 75|330642|\n", - "| 77|330646|\n", - "| 71|330651|\n", - "| 74|330653|\n", - "| 73|330667|\n", - "| 70|330690|\n", - "| 65|330704|\n", - "| 72|330709|\n", - "| 67|330721|\n", - "| 66|330724|\n", - "| 69|330737|\n", - "| 62|330758|\n", - "| 63|330762|\n", - "| 64|330762|\n", - "| 68|330766|\n", - "| 60|330782|\n", - "| 59|330784|\n", - "| 56|330801|\n", - "| 57|330805|\n", - "| 61|330807|\n", - "| 58|330837|\n", - "| 53|330868|\n", - "| 55|330869|\n", - "| 54|330885|\n", - "| 52|330918|\n", - "| 50|330944|\n", - "| 51|330963|\n", - "| 49|331028|\n", - "| 48|331034|\n", - "| 47|331050|\n", - "| 46|331114|\n", - "| 45|331284|\n", - "| 44|331416|\n", - "| 543|364094|\n", - "| 542|364374|\n", - "| 541|364493|\n", - "| 537|364581|\n", - "| 538|364599|\n", - "| 539|364616|\n", - "| 540|364617|\n", - "| 536|364654|\n", - "| 534|364709|\n", - "| 535|364756|\n", - "| 532|364784|\n", - "| 533|364810|\n", - "| 529|364899|\n", - "| 530|364903|\n", - "| 531|364944|\n", - "| 528|364957|\n", - "| 527|364961|\n", - "| 524|364971|\n", - "| 525|364988|\n", - "| 526|365006|\n", - "| 522|365011|\n", - "| 523|365051|\n", - "| 521|365057|\n", - "| 520|365079|\n", - "| 518|365083|\n", - "| 517|365090|\n", - "| 519|365097|\n", - "| 516|365122|\n", - "| 514|365165|\n", - "| 515|365179|\n", - "| 513|365224|\n", - "| 509|365252|\n", - "| 506|365253|\n", - "| 511|365255|\n", - "| 508|365272|\n", - "| 510|365277|\n", - "| 512|365278|\n", - "| 507|365302|\n", - "| 505|365347|\n", - "| 502|365377|\n", - "| 503|365394|\n", - "| 504|365395|\n", - "| 501|365409|\n", - "| 500|365431|\n", - "| 498|365447|\n", - "| 499|365454|\n", - "| 497|365519|\n", - "| 496|365528|\n", - "| 495|365536|\n", - "| 492|365541|\n", - "| 489|365547|\n", - "| 488|365552|\n", - "| 487|365554|\n", - "| 490|365569|\n", - "| 493|365574|\n", - "| 484|365576|\n", - "| 494|365595|\n", - "| 485|365602|\n", - "| 491|365622|\n", - "| 486|365622|\n", - "| 483|365650|\n", - "| 482|365684|\n", - "| 481|365705|\n", - "| 479|365750|\n", - "| 478|365773|\n", - "| 477|365793|\n", - "| 480|365801|\n", - "| 475|365806|\n", - "| 474|365806|\n", - "| 473|365828|\n", - "| 476|365846|\n", - "| 472|365909|\n", - "| 471|365965|\n", - "| 470|365975|\n", - "| 469|366026|\n", - "| 466|366051|\n", - "| 467|366057|\n", - "| 468|366080|\n", - "| 464|366105|\n", - "| 465|366117|\n", - "| 462|366150|\n", - "| 463|366160|\n", - "| 458|366193|\n", - "| 461|366200|\n", - "| 460|366214|\n", - "| 459|366217|\n", - "| 456|366297|\n", - "| 457|366320|\n", - "| 455|366371|\n", - "| 454|366383|\n", - "| 453|366422|\n", - "| 452|366461|\n", - "| 451|366589|\n", - "| 450|366617|\n", - "| 449|366758|\n", - "| 448|366799|\n", - "| 447|366883|\n", - "| 446|366901|\n", - "| 445|366940|\n", - "| 444|367122|\n", + "| 176|329786|\n", + "| 187|329786|\n", + "| 177|329787|\n", + "| 182|329793|\n", + "| 173|329804|\n", + "| 174|329805|\n", + "| 169|329807|\n", + "| 172|329814|\n", + "| 171|329819|\n", + "| 175|329843|\n", + "| 168|329849|\n", + "| 167|329861|\n", + "| 170|329865|\n", + "| 162|329866|\n", + "| 163|329871|\n", + "| 161|329879|\n", + "| 165|329889|\n", + "| 164|329892|\n", + "| 166|329900|\n", + "| 160|329937|\n", + "| 158|329956|\n", + "| 152|329961|\n", + "| 157|329962|\n", + "| 154|329965|\n", + "| 159|329974|\n", + "| 155|329980|\n", + "| 153|329985|\n", + "| 149|329993|\n", + "| 156|330000|\n", + "| 148|330005|\n", + "| 151|330014|\n", + "| 147|330021|\n", + "| 150|330027|\n", + "| 146|330059|\n", + "| 144|330066|\n", + "| 137|330070|\n", + "| 143|330083|\n", + "| 145|330084|\n", + "| 140|330084|\n", + "| 136|330089|\n", + "| 142|330090|\n", + "| 139|330099|\n", + "| 141|330101|\n", + "| 138|330103|\n", + "| 134|330116|\n", + "| 135|330129|\n", + "| 130|330133|\n", + "| 133|330135|\n", + "| 131|330139|\n", + "| 132|330148|\n", + "| 129|330159|\n", + "| 125|330178|\n", + "| 127|330181|\n", + "| 122|330186|\n", + "| 128|330189|\n", + "| 126|330205|\n", + "| 119|330208|\n", + "| 123|330219|\n", + "| 115|330220|\n", + "| 118|330227|\n", + "| 124|330230|\n", + "| 117|330236|\n", + "| 120|330243|\n", + "| 114|330245|\n", + "| 107|330249|\n", + "| 121|330252|\n", + "| 111|330255|\n", + "| 112|330258|\n", + "| 116|330268|\n", + "| 113|330270|\n", + "| 108|330275|\n", + "| 105|330282|\n", + "| 103|330282|\n", + "| 110|330296|\n", + "| 102|330297|\n", + "| 109|330306|\n", + "| 104|330308|\n", + "| 94|330338|\n", + "| 106|330339|\n", + "| 99|330355|\n", + "| 95|330363|\n", + "| 101|330366|\n", + "| 100|330372|\n", + "| 98|330383|\n", + "| 97|330385|\n", + "| 96|330402|\n", + "| 86|330413|\n", + "| 92|330415|\n", + "| 88|330418|\n", + "| 90|330418|\n", + "| 91|330419|\n", + "| 84|330425|\n", + "| 87|330426|\n", + "| 93|330430|\n", + "| 89|330430|\n", + "| 85|330436|\n", + "| 80|330456|\n", + "| 82|330470|\n", + "| 78|330471|\n", + "| 81|330474|\n", + "| 83|330477|\n", + "| 77|330494|\n", + "| 76|330501|\n", + "| 79|330512|\n", + "| 74|330524|\n", + "| 72|330550|\n", + "| 75|330570|\n", + "| 70|330570|\n", + "| 71|330579|\n", + "| 73|330585|\n", + "| 68|330622|\n", + "| 66|330625|\n", + "| 69|330625|\n", + "| 65|330642|\n", + "| 67|330646|\n", + "| 61|330651|\n", + "| 64|330653|\n", + "| 63|330667|\n", + "| 60|330690|\n", + "| 55|330704|\n", + "| 62|330709|\n", + "| 57|330721|\n", + "| 56|330724|\n", + "| 59|330737|\n", + "| 52|330758|\n", + "| 54|330762|\n", + "| 53|330762|\n", + "| 58|330766|\n", + "| 50|330782|\n", + "| 49|330784|\n", + "| 46|330801|\n", + "| 47|330805|\n", + "| 51|330807|\n", + "| 48|330837|\n", + "| 43|330868|\n", + "| 45|330869|\n", + "| 44|330885|\n", + "| 42|330918|\n", + "| 40|330944|\n", + "| 41|330963|\n", + "| 39|331028|\n", + "| 38|331034|\n", + "| 37|331050|\n", + "| 36|331114|\n", + "| 35|331284|\n", + "| 34|331416|\n", + "| 533|364094|\n", + "| 532|364374|\n", + "| 531|364493|\n", + "| 527|364581|\n", + "| 528|364599|\n", + "| 529|364616|\n", + "| 530|364617|\n", + "| 526|364654|\n", + "| 524|364709|\n", + "| 525|364756|\n", + "| 522|364784|\n", + "| 523|364810|\n", + "| 519|364899|\n", + "| 520|364903|\n", + "| 521|364944|\n", + "| 518|364957|\n", + "| 517|364961|\n", + "| 514|364971|\n", + "| 515|364988|\n", + "| 516|365006|\n", + "| 512|365011|\n", + "| 513|365051|\n", + "| 511|365057|\n", + "| 510|365079|\n", + "| 508|365083|\n", + "| 507|365090|\n", + "| 509|365097|\n", + "| 506|365122|\n", + "| 504|365165|\n", + "| 505|365179|\n", + "| 503|365224|\n", + "| 499|365252|\n", + "| 496|365253|\n", + "| 501|365255|\n", + "| 498|365272|\n", + "| 500|365277|\n", + "| 502|365278|\n", + "| 497|365302|\n", + "| 495|365347|\n", + "| 492|365377|\n", + "| 493|365394|\n", + "| 494|365395|\n", + "| 491|365409|\n", + "| 490|365431|\n", + "| 488|365447|\n", + "| 489|365454|\n", + "| 487|365519|\n", + "| 486|365528|\n", + "| 485|365536|\n", + "| 482|365541|\n", + "| 479|365547|\n", + "| 478|365552|\n", + "| 477|365554|\n", + "| 480|365569|\n", + "| 483|365574|\n", + "| 474|365576|\n", + "| 484|365595|\n", + "| 475|365602|\n", + "| 476|365622|\n", + "| 481|365622|\n", + "| 473|365650|\n", + "| 472|365684|\n", + "| 471|365705|\n", + "| 469|365750|\n", + "| 468|365773|\n", + "| 467|365793|\n", + "| 470|365801|\n", + "| 465|365806|\n", + "| 464|365806|\n", + "| 463|365828|\n", + "| 466|365846|\n", + "| 462|365909|\n", + "| 461|365965|\n", + "| 460|365975|\n", + "| 459|366026|\n", + "| 456|366051|\n", + "| 457|366057|\n", + "| 458|366080|\n", + "| 454|366105|\n", + "| 455|366117|\n", + "| 452|366150|\n", + "| 453|366160|\n", + "| 448|366193|\n", + "| 451|366200|\n", + "| 450|366214|\n", + "| 449|366217|\n", + "| 446|366297|\n", + "| 447|366320|\n", + "| 445|366371|\n", + "| 444|366383|\n", + "| 443|366422|\n", + "| 442|366461|\n", + "| 441|366589|\n", + "| 440|366617|\n", + "| 439|366758|\n", + "| 438|366799|\n", + "| 437|366883|\n", + "| 436|366901|\n", + "| 435|366940|\n", + "| 434|367122|\n", "| 21|380513|\n", "| 20|380565|\n", "| 19|380749|\n", @@ -574,207 +813,217 @@ "| 1|382022|\n", "| 2|382029|\n", "| 0|382095|\n", - "| 342|420259|\n", - "| 343|420346|\n", - "| 341|420485|\n", - "| 340|420525|\n", - "| 339|420707|\n", - "| 336|421031|\n", - "| 337|421040|\n", - "| 338|421052|\n", - "| 334|421107|\n", - "| 335|421142|\n", - "| 333|421374|\n", - "| 330|421440|\n", - "| 332|421479|\n", - "| 331|421531|\n", - "| 327|421574|\n", - "| 328|421603|\n", - "| 329|421610|\n", - "| 326|421612|\n", - "| 322|421670|\n", - "| 320|421675|\n", - "| 325|421679|\n", - "| 324|421681|\n", - "| 319|421687|\n", - "| 323|421687|\n", - "| 321|421699|\n", - "| 318|421751|\n", - "| 315|421832|\n", - "| 310|421867|\n", - "| 316|421897|\n", - "| 312|421903|\n", - "| 317|421911|\n", - "| 314|421918|\n", - "| 313|421920|\n", - "| 311|421950|\n", - "| 309|421972|\n", - "| 307|421988|\n", - "| 308|422019|\n", - "| 305|422072|\n", - "| 303|422083|\n", - "| 306|422091|\n", - "| 304|422095|\n", - "| 302|422097|\n", - "| 298|422103|\n", - "| 300|422114|\n", - "| 301|422116|\n", - "| 295|422134|\n", - "| 299|422155|\n", - "| 296|422155|\n", - "| 290|422185|\n", - "| 297|422193|\n", - "| 294|422194|\n", - "| 292|422207|\n", - "| 291|422218|\n", - "| 293|422236|\n", - "| 288|422238|\n", - "| 286|422255|\n", - "| 289|422265|\n", - "| 287|422266|\n", - "| 285|422305|\n", - "| 283|422307|\n", - "| 284|422346|\n", - "| 282|422350|\n", - "| 281|422354|\n", - "| 280|422372|\n", - "| 279|422415|\n", - "| 278|422498|\n", - "| 277|422501|\n", - "| 276|422508|\n", - "| 275|422549|\n", - "| 274|422557|\n", - "| 273|422591|\n", - "| 272|422625|\n", - "| 270|422634|\n", - "| 269|422671|\n", - "| 268|422673|\n", - "| 271|422692|\n", - "| 267|422694|\n", - "| 265|422761|\n", - "| 262|422777|\n", - "| 260|422788|\n", - "| 263|422795|\n", - "| 266|422803|\n", - "| 264|422807|\n", - "| 258|422838|\n", - "| 259|422839|\n", - "| 261|422841|\n", - "| 257|422852|\n", - "| 256|422891|\n", - "| 252|422904|\n", - "| 255|422925|\n", - "| 254|422986|\n", - "| 253|423003|\n", - "| 250|423197|\n", - "| 251|423202|\n", - "| 248|423231|\n", - "| 249|423262|\n", - "| 246|423376|\n", - "| 247|423402|\n", - "| 245|423403|\n", - "| 244|423762|\n", - "| 43|457702|\n", - "| 443|569570|\n", - "| 442|570154|\n", - "| 441|570301|\n", - "| 440|570372|\n", - "| 439|570572|\n", - "| 438|570655|\n", - "| 436|570763|\n", - "| 437|570781|\n", - "| 434|570870|\n", - "| 435|570872|\n", - "| 433|570953|\n", - "| 432|570979|\n", - "| 431|571069|\n", - "| 429|571096|\n", - "| 430|571097|\n", - "| 428|571127|\n", - "| 427|571153|\n", - "| 426|571185|\n", - "| 425|571201|\n", - "| 424|571286|\n", - "| 423|571425|\n", - "| 422|571449|\n", - "| 417|571506|\n", - "| 420|571528|\n", - "| 421|571532|\n", - "| 419|571553|\n", - "| 418|571585|\n", - "| 416|571595|\n", - "| 414|571645|\n", - "| 415|571657|\n", - "| 413|571742|\n", - "| 412|571766|\n", - "| 411|571796|\n", - "| 409|571842|\n", - "| 410|571847|\n", - "| 407|571874|\n", - "| 408|571913|\n", - "| 406|571925|\n", - "| 405|571966|\n", - "| 404|571983|\n", - "| 402|571993|\n", - "| 403|572020|\n", - "| 401|572123|\n", - "| 397|572181|\n", - "| 400|572182|\n", - "| 399|572183|\n", - "| 398|572189|\n", - "| 396|572212|\n", - "| 395|572244|\n", - "| 393|572249|\n", - "| 394|572276|\n", - "| 392|572302|\n", - "| 391|572344|\n", - "| 390|572361|\n", - "| 389|572382|\n", - "| 388|572394|\n", - "| 387|572428|\n", - "| 386|572438|\n", - "| 385|572493|\n", - "| 383|572545|\n", - "| 384|572565|\n", - "| 382|572569|\n", - "| 381|572600|\n", - "| 380|572604|\n", - "| 379|572634|\n", - "| 378|572647|\n", - "| 376|572742|\n", - "| 375|572742|\n", - "| 377|572755|\n", - "| 374|572798|\n", - "| 372|572800|\n", - "| 373|572816|\n", - "| 371|572868|\n", - "| 370|572895|\n", - "| 369|572907|\n", - "| 368|572924|\n", - "| 367|572957|\n", - "| 366|573022|\n", - "| 364|573102|\n", - "| 365|573104|\n", - "| 362|573132|\n", - "| 363|573145|\n", - "| 361|573173|\n", - "| 360|573187|\n", - "| 358|573262|\n", - "| 359|573270|\n", - "| 357|573334|\n", - "| 356|573372|\n", - "| 355|573425|\n", - "| 354|573556|\n", - "| 353|573584|\n", - "| 352|573658|\n", - "| 351|573676|\n", - "| 350|573781|\n", - "| 349|573977|\n", - "| 347|574013|\n", - "| 348|574040|\n", - "| 346|574185|\n", - "| 345|574318|\n", - "| 344|574727|\n", + "| 332|420259|\n", + "| 333|420346|\n", + "| 331|420485|\n", + "| 330|420525|\n", + "| 329|420707|\n", + "| 326|421031|\n", + "| 327|421040|\n", + "| 328|421052|\n", + "| 324|421107|\n", + "| 325|421142|\n", + "| 323|421374|\n", + "| 320|421440|\n", + "| 322|421479|\n", + "| 321|421531|\n", + "| 317|421574|\n", + "| 318|421603|\n", + "| 319|421610|\n", + "| 316|421612|\n", + "| 312|421670|\n", + "| 310|421675|\n", + "| 315|421679|\n", + "| 314|421681|\n", + "| 309|421687|\n", + "| 313|421687|\n", + "| 311|421699|\n", + "| 308|421751|\n", + "| 305|421832|\n", + "| 300|421867|\n", + "| 306|421897|\n", + "| 302|421903|\n", + "| 307|421911|\n", + "| 304|421918|\n", + "| 303|421920|\n", + "| 301|421950|\n", + "| 299|421972|\n", + "| 297|421988|\n", + "| 298|422019|\n", + "| 295|422072|\n", + "| 293|422083|\n", + "| 296|422091|\n", + "| 294|422095|\n", + "| 292|422097|\n", + "| 288|422103|\n", + "| 290|422114|\n", + "| 291|422116|\n", + "| 285|422134|\n", + "| 286|422155|\n", + "| 289|422155|\n", + "| 280|422185|\n", + "| 287|422193|\n", + "| 284|422194|\n", + "| 282|422207|\n", + "| 281|422218|\n", + "| 283|422236|\n", + "| 278|422238|\n", + "| 276|422255|\n", + "| 279|422265|\n", + "| 277|422266|\n", + "| 275|422305|\n", + "| 273|422307|\n", + "| 274|422346|\n", + "| 272|422350|\n", + "| 271|422354|\n", + "| 270|422372|\n", + "| 269|422415|\n", + "| 268|422498|\n", + "| 267|422501|\n", + "| 266|422508|\n", + "| 265|422549|\n", + "| 264|422557|\n", + "| 263|422591|\n", + "| 262|422625|\n", + "| 260|422634|\n", + "| 259|422671|\n", + "| 258|422673|\n", + "| 261|422692|\n", + "| 257|422694|\n", + "| 255|422761|\n", + "| 252|422777|\n", + "| 250|422788|\n", + "| 253|422795|\n", + "| 256|422803|\n", + "| 254|422807|\n", + "| 248|422838|\n", + "| 249|422839|\n", + "| 251|422841|\n", + "| 247|422852|\n", + "| 246|422891|\n", + "| 242|422904|\n", + "| 245|422925|\n", + "| 244|422986|\n", + "| 243|423003|\n", + "| 240|423197|\n", + "| 241|423202|\n", + "| 238|423231|\n", + "| 239|423262|\n", + "| 236|423376|\n", + "| 237|423402|\n", + "| 235|423403|\n", + "| 234|423762|\n", + "| 433|569570|\n", + "| 432|570154|\n", + "| 431|570301|\n", + "| 430|570372|\n", + "| 429|570572|\n", + "| 428|570655|\n", + "| 426|570763|\n", + "| 427|570781|\n", + "| 424|570870|\n", + "| 425|570872|\n", + "| 423|570953|\n", + "| 422|570979|\n", + "| 421|571069|\n", + "| 419|571096|\n", + "| 420|571097|\n", + "| 418|571127|\n", + "| 417|571153|\n", + "| 416|571185|\n", + "| 415|571201|\n", + "| 414|571286|\n", + "| 413|571425|\n", + "| 412|571449|\n", + "| 407|571506|\n", + "| 410|571528|\n", + "| 411|571532|\n", + "| 409|571553|\n", + "| 408|571585|\n", + "| 406|571595|\n", + "| 404|571645|\n", + "| 405|571657|\n", + "| 403|571742|\n", + "| 402|571766|\n", + "| 401|571796|\n", + "| 399|571842|\n", + "| 400|571847|\n", + "| 397|571874|\n", + "| 398|571913|\n", + "| 396|571925|\n", + "| 395|571966|\n", + "| 394|571983|\n", + "| 392|571993|\n", + "| 393|572020|\n", + "| 391|572123|\n", + "| 387|572181|\n", + "| 390|572182|\n", + "| 389|572183|\n", + "| 388|572189|\n", + "| 386|572212|\n", + "| 385|572244|\n", + "| 383|572249|\n", + "| 384|572276|\n", + "| 382|572302|\n", + "| 381|572344|\n", + "| 380|572361|\n", + "| 379|572382|\n", + "| 378|572394|\n", + "| 377|572428|\n", + "| 376|572438|\n", + "| 375|572493|\n", + "| 373|572545|\n", + "| 374|572565|\n", + "| 372|572569|\n", + "| 371|572600|\n", + "| 370|572604|\n", + "| 369|572634|\n", + "| 368|572647|\n", + "| 365|572742|\n", + "| 366|572742|\n", + "| 367|572755|\n", + "| 364|572798|\n", + "| 362|572800|\n", + "| 363|572816|\n", + "| 361|572868|\n", + "| 360|572895|\n", + "| 359|572907|\n", + "| 358|572924|\n", + "| 357|572957|\n", + "| 356|573022|\n", + "| 354|573102|\n", + "| 355|573104|\n", + "| 352|573132|\n", + "| 353|573145|\n", + "| 351|573173|\n", + "| 350|573187|\n", + "| 348|573262|\n", + "| 349|573270|\n", + "| 347|573334|\n", + "| 346|573372|\n", + "| 345|573425|\n", + "| 344|573556|\n", + "| 343|573584|\n", + "| 342|573658|\n", + "| 341|573676|\n", + "| 340|573781|\n", + "| 339|573977|\n", + "| 337|574013|\n", + "| 338|574040|\n", + "| 336|574185|\n", + "| 335|574318|\n", + "| 334|574727|\n", + "| 32|610310|\n", + "| 31|610736|\n", + "| 30|610951|\n", + "| 29|611294|\n", + "| 28|611761|\n", + "| 27|611933|\n", + "| 26|612048|\n", + "| 25|612117|\n", + "| 24|612529|\n", + "| 23|613100|\n", + "| 22|613957|\n", "+-----------+------+\n", "\n" ] @@ -813,7 +1062,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 19:=====================================================>(543 + 1) / 544]\r" + "[Stage 19:=====================================================>(532 + 2) / 534]\r" ] }, { @@ -837,606 +1086,606 @@ "+-----------+------+\n", "|partitionId| count|\n", "+-----------+------+\n", - "| 25|362151|\n", - "| 33|362151|\n", + "| 26|362149|\n", + "| 27|362149|\n", + "| 24|362149|\n", + "| 25|362149|\n", + "| 28|362150|\n", + "| 33|362150|\n", + "| 30|362150|\n", + "| 35|362150|\n", + "| 31|362151|\n", + "| 29|362151|\n", "| 32|362151|\n", - "| 38|362151|\n", - "| 39|362152|\n", - "| 29|362152|\n", - "| 598|362152|\n", - "| 15|362152|\n", + "| 64|362151|\n", + "| 34|362151|\n", + "| 63|362152|\n", + "| 57|362152|\n", + "| 62|362152|\n", + "| 56|362152|\n", "| 37|362152|\n", - "| 597|362152|\n", - "| 17|362152|\n", - "| 16|362152|\n", - "| 24|362152|\n", - "| 26|362152|\n", - "| 40|362152|\n", - "| 8|362153|\n", - "| 13|362153|\n", - "| 41|362153|\n", - "| 11|362153|\n", - "| 34|362153|\n", - "| 35|362153|\n", - "| 31|362153|\n", - "| 18|362153|\n", - "| 14|362153|\n", - "| 27|362153|\n", - "| 30|362153|\n", + "| 65|362152|\n", + "| 74|362152|\n", + "| 55|362152|\n", + "| 36|362152|\n", + "| 49|362153|\n", "| 20|362153|\n", - "| 23|362153|\n", - "| 63|362154|\n", - "| 22|362154|\n", - "| 64|362154|\n", - "| 10|362154|\n", + "| 54|362153|\n", + "| 73|362153|\n", + "| 19|362153|\n", + "| 75|362153|\n", + "| 44|362153|\n", + "| 58|362153|\n", + "| 71|362153|\n", + "| 61|362153|\n", + "| 66|362153|\n", + "| 72|362153|\n", + "| 67|362153|\n", + "| 59|362153|\n", + "| 39|362153|\n", + "| 40|362154|\n", + "| 69|362154|\n", + "| 45|362154|\n", + "| 50|362154|\n", + "| 47|362154|\n", + "| 160|362154|\n", + "| 70|362154|\n", "| 48|362154|\n", - "| 19|362154|\n", - "| 599|362154|\n", - "| 5|362154|\n", - "| 21|362154|\n", - "| 9|362154|\n", - "| 525|362154|\n", - "| 3|362154|\n", - "| 42|362154|\n", - "| 28|362154|\n", - "| 462|362154|\n", - "| 596|362154|\n", - "| 36|362154|\n", - "| 12|362154|\n", - "| 65|362155|\n", - "| 524|362155|\n", - "| 539|362155|\n", - "| 526|362155|\n", - "| 43|362155|\n", - "| 47|362155|\n", - "| 4|362155|\n", - "| 530|362155|\n", - "| 538|362155|\n", + "| 76|362154|\n", + "| 60|362154|\n", + "| 53|362154|\n", + "| 15|362154|\n", + "| 43|362154|\n", + "| 68|362154|\n", + "| 46|362154|\n", + "| 16|362154|\n", + "| 38|362154|\n", + "| 23|362154|\n", + "| 22|362154|\n", + "| 164|362155|\n", + "| 51|362155|\n", + "| 77|362155|\n", + "| 163|362155|\n", + "| 21|362155|\n", + "| 159|362155|\n", + "| 158|362155|\n", "| 52|362155|\n", - "| 46|362155|\n", - "| 49|362155|\n", - "| 6|362155|\n", - "| 44|362155|\n", - "| 2|362155|\n", - "| 0|362155|\n", - "| 463|362155|\n", - "| 7|362155|\n", - "| 465|362156|\n", - "| 51|362156|\n", - "| 532|362156|\n", - "| 467|362156|\n", - "| 53|362156|\n", - "| 527|362156|\n", - "| 50|362156|\n", - "| 459|362156|\n", - "| 45|362156|\n", - "| 62|362156|\n", - "| 528|362156|\n", - "| 464|362156|\n", - "| 592|362156|\n", - "| 593|362156|\n", - "| 534|362156|\n", - "| 595|362156|\n", - "| 1|362156|\n", - "| 460|362156|\n", - "| 454|362156|\n", - "| 535|362156|\n", - "| 66|362156|\n", - "| 461|362156|\n", - "| 61|362156|\n", - "| 466|362156|\n", - "| 67|362156|\n", - "| 594|362156|\n", - "| 531|362156|\n", - "| 60|362156|\n", - "| 589|362157|\n", - "| 583|362157|\n", - "| 587|362157|\n", - "| 328|362157|\n", - "| 453|362157|\n", - "| 588|362157|\n", - "| 533|362157|\n", - "| 452|362157|\n", - "| 585|362157|\n", - "| 560|362157|\n", - "| 70|362157|\n", - "| 360|362157|\n", - "| 364|362157|\n", - "| 455|362157|\n", - "| 591|362157|\n", - "| 554|362157|\n", - "| 366|362157|\n", - "| 536|362157|\n", - "| 541|362157|\n", - "| 106|362157|\n", - "| 68|362157|\n", - "| 329|362157|\n", - "| 59|362157|\n", - "| 55|362157|\n", - "| 365|362157|\n", - "| 523|362157|\n", - "| 529|362157|\n", - "| 584|362157|\n", - "| 451|362157|\n", - "| 468|362157|\n", - "| 517|362157|\n", - "| 537|362157|\n", - "| 586|362157|\n", - "| 540|362157|\n", - "| 54|362157|\n", - "| 71|362158|\n", - "| 331|362158|\n", - "| 458|362158|\n", - "| 69|362158|\n", - "| 359|362158|\n", - "| 543|362158|\n", + "| 42|362155|\n", + "| 17|362155|\n", + "| 18|362155|\n", + "| 41|362155|\n", + "| 165|362156|\n", + "| 162|362156|\n", + "| 331|362156|\n", + "| 12|362156|\n", + "| 156|362156|\n", + "| 78|362156|\n", + "| 79|362156|\n", + "| 13|362156|\n", + "| 161|362156|\n", + "| 157|362156|\n", + "| 3|362157|\n", + "| 166|362157|\n", + "| 598|362157|\n", + "| 11|362157|\n", + "| 1|362157|\n", + "| 2|362157|\n", + "| 597|362157|\n", + "| 589|362158|\n", + "| 583|362158|\n", + "| 332|362158|\n", + "| 585|362158|\n", + "| 599|362158|\n", + "| 582|362158|\n", + "| 167|362158|\n", + "| 80|362158|\n", + "| 336|362158|\n", + "| 587|362158|\n", + "| 322|362158|\n", + "| 588|362158|\n", + "| 152|362158|\n", + "| 324|362158|\n", + "| 586|362158|\n", + "| 562|362158|\n", + "| 0|362158|\n", + "| 584|362158|\n", + "| 320|362158|\n", + "| 323|362158|\n", + "| 535|362158|\n", + "| 14|362158|\n", + "| 4|362158|\n", + "| 321|362158|\n", "| 330|362158|\n", - "| 449|362158|\n", - "| 361|362158|\n", - "| 544|362158|\n", - "| 367|362158|\n", - "| 93|362158|\n", - "| 115|362158|\n", - "| 553|362158|\n", - "| 542|362158|\n", - "| 112|362158|\n", - "| 327|362158|\n", - "| 457|362158|\n", - "| 358|362158|\n", - "| 376|362158|\n", - "| 469|362158|\n", - "| 590|362158|\n", - "| 109|362158|\n", - "| 56|362158|\n", - "| 107|362158|\n", - "| 456|362158|\n", - "| 552|362158|\n", - "| 561|362158|\n", - "| 108|362158|\n", - "| 373|362158|\n", - "| 58|362158|\n", - "| 522|362158|\n", - "| 368|362158|\n", - "| 580|362158|\n", - "| 74|362158|\n", - "| 516|362158|\n", - "| 521|362158|\n", - "| 94|362158|\n", - "| 326|362158|\n", - "| 111|362158|\n", - "| 57|362158|\n", - "| 559|362159|\n", - "| 448|362159|\n", - "| 450|362159|\n", - "| 321|362159|\n", - "| 471|362159|\n", - "| 556|362159|\n", - "| 566|362159|\n", - "| 318|362159|\n", - "| 565|362159|\n", - "| 332|362159|\n", - "| 555|362159|\n", - "| 470|362159|\n", - "| 319|362159|\n", - "| 545|362159|\n", - "| 372|362159|\n", - "| 518|362159|\n", - "| 113|362159|\n", - "| 569|362159|\n", - "| 511|362159|\n", + "| 576|362159|\n", + "| 327|362159|\n", + "| 154|362159|\n", + "| 168|362159|\n", "| 333|362159|\n", - "| 447|362159|\n", - "| 73|362159|\n", - "| 563|362159|\n", - "| 335|362159|\n", - "| 362|362159|\n", - "| 513|362159|\n", - "| 322|362159|\n", - "| 515|362159|\n", - "| 75|362159|\n", + "| 81|362159|\n", + "| 190|362159|\n", "| 334|362159|\n", - "| 374|362159|\n", - "| 564|362159|\n", - "| 110|362159|\n", - "| 116|362159|\n", - "| 512|362159|\n", - "| 114|362159|\n", - "| 581|362159|\n", - "| 72|362159|\n", - "| 557|362159|\n", - "| 363|362159|\n", - "| 551|362159|\n", - "| 562|362159|\n", - "| 377|362159|\n", - "| 582|362159|\n", - "| 324|362159|\n", - "| 336|362159|\n", - "| 317|362159|\n", - "| 76|362160|\n", - "| 77|362160|\n", - "| 558|362160|\n", - "| 476|362160|\n", - "| 325|362160|\n", - "| 550|362160|\n", - "| 375|362160|\n", + "| 310|362159|\n", + "| 596|362159|\n", + "| 10|362159|\n", + "| 404|362159|\n", + "| 569|362159|\n", + "| 540|362159|\n", + "| 591|362159|\n", + "| 5|362159|\n", + "| 526|362159|\n", + "| 444|362159|\n", + "| 595|362159|\n", + "| 151|362159|\n", + "| 435|362159|\n", + "| 536|362159|\n", + "| 561|362159|\n", + "| 83|362159|\n", + "| 155|362159|\n", + "| 337|362159|\n", + "| 573|362159|\n", + "| 405|362159|\n", + "| 153|362159|\n", + "| 6|362159|\n", + "| 563|362159|\n", + "| 329|362159|\n", + "| 319|362159|\n", + "| 572|362159|\n", + "| 315|362159|\n", + "| 326|362159|\n", + "| 338|362159|\n", + "| 7|362160|\n", + "| 171|362160|\n", + "| 318|362160|\n", + "| 552|362160|\n", + "| 538|362160|\n", + "| 223|362160|\n", + "| 530|362160|\n", "| 339|362160|\n", - "| 547|362160|\n", - "| 475|362160|\n", - "| 315|362160|\n", - "| 87|362160|\n", - "| 79|362160|\n", - "| 519|362160|\n", - "| 96|362160|\n", - "| 91|362160|\n", - "| 92|362160|\n", - "| 356|362160|\n", - "| 510|362160|\n", - "| 95|362160|\n", - "| 338|362160|\n", - "| 78|362160|\n", - "| 378|362160|\n", - "| 370|362160|\n", - "| 105|362160|\n", - "| 369|362160|\n", - "| 508|362160|\n", + "| 232|362160|\n", + "| 577|362160|\n", + "| 84|362160|\n", + "| 443|362160|\n", + "| 532|362160|\n", + "| 169|362160|\n", + "| 568|362160|\n", + "| 314|362160|\n", + "| 436|362160|\n", + "| 527|362160|\n", + "| 539|362160|\n", + "| 82|362160|\n", + "| 427|362160|\n", + "| 195|362160|\n", + "| 560|362160|\n", + "| 221|362160|\n", + "| 554|362160|\n", + "| 574|362160|\n", + "| 170|362160|\n", + "| 312|362160|\n", "| 567|362160|\n", - "| 118|362160|\n", - "| 473|362160|\n", - "| 337|362160|\n", - "| 520|362160|\n", - "| 357|362160|\n", - "| 88|362160|\n", - "| 89|362160|\n", - "| 90|362160|\n", - "| 320|362160|\n", - "| 549|362160|\n", - "| 514|362160|\n", - "| 117|362161|\n", - "| 102|362161|\n", - "| 101|362161|\n", - "| 119|362161|\n", - "| 371|362161|\n", - "| 120|362161|\n", - "| 100|362161|\n", - "| 579|362161|\n", - "| 578|362161|\n", - "| 509|362161|\n", - "| 477|362161|\n", - "| 492|362161|\n", - "| 323|362161|\n", - "| 478|362161|\n", - "| 505|362161|\n", - "| 506|362161|\n", - "| 503|362161|\n", - "| 474|362161|\n", - "| 572|362161|\n", - "| 546|362161|\n", + "| 150|362160|\n", + "| 541|362160|\n", + "| 590|362160|\n", + "| 328|362160|\n", + "| 316|362160|\n", + "| 531|362160|\n", + "| 566|362160|\n", + "| 564|362160|\n", + "| 149|362160|\n", + "| 442|362160|\n", + "| 9|362160|\n", + "| 571|362160|\n", + "| 172|362160|\n", + "| 594|362160|\n", + "| 222|362160|\n", + "| 311|362160|\n", + "| 173|362160|\n", + "| 445|362160|\n", + "| 325|362160|\n", + "| 578|362160|\n", + "| 309|362160|\n", + "| 313|362160|\n", + "| 8|362160|\n", + "| 537|362160|\n", + "| 335|362160|\n", + "| 592|362161|\n", + "| 580|362161|\n", + "| 429|362161|\n", + "| 438|362161|\n", + "| 194|362161|\n", "| 570|362161|\n", + "| 518|362161|\n", + "| 452|362161|\n", + "| 593|362161|\n", + "| 219|362161|\n", + "| 258|362161|\n", + "| 192|362161|\n", + "| 575|362161|\n", + "| 189|362161|\n", + "| 409|362161|\n", + "| 236|362161|\n", + "| 449|362161|\n", + "| 193|362161|\n", "| 446|362161|\n", - "| 97|362161|\n", - "| 576|362161|\n", - "| 472|362161|\n", - "| 490|362161|\n", - "| 379|362161|\n", - "| 316|362161|\n", - "| 571|362161|\n", - "| 548|362161|\n", - "| 104|362161|\n", - "| 577|362161|\n", - "| 99|362161|\n", - "| 502|362162|\n", - "| 86|362162|\n", + "| 514|362161|\n", + "| 453|362161|\n", + "| 565|362161|\n", + "| 434|362161|\n", + "| 543|362161|\n", + "| 403|362161|\n", + "| 524|362161|\n", + "| 533|362161|\n", + "| 361|362161|\n", + "| 432|362161|\n", + "| 406|362161|\n", + "| 559|362161|\n", + "| 408|362161|\n", + "| 547|362161|\n", + "| 513|362161|\n", + "| 191|362161|\n", + "| 528|362161|\n", + "| 542|362161|\n", + "| 416|362161|\n", + "| 556|362161|\n", + "| 515|362161|\n", + "| 519|362161|\n", + "| 579|362161|\n", + "| 224|362161|\n", + "| 534|362161|\n", + "| 447|362161|\n", + "| 188|362161|\n", + "| 512|362161|\n", + "| 525|362161|\n", + "| 508|362161|\n", + "| 206|362161|\n", + "| 308|362161|\n", + "| 197|362161|\n", + "| 506|362161|\n", + "| 529|362161|\n", + "| 440|362161|\n", + "| 174|362161|\n", + "| 86|362161|\n", + "| 551|362161|\n", + "| 85|362161|\n", + "| 437|362161|\n", + "| 407|362161|\n", + "| 196|362161|\n", + "| 261|362161|\n", + "| 517|362161|\n", + "| 317|362161|\n", + "| 441|362161|\n", + "| 425|362161|\n", + "| 208|362161|\n", + "| 233|362161|\n", + "| 307|362161|\n", + "| 225|362162|\n", + "| 520|362162|\n", + "| 87|362162|\n", + "| 439|362162|\n", + "| 210|362162|\n", + "| 295|362162|\n", + "| 431|362162|\n", + "| 249|362162|\n", + "| 237|362162|\n", + "| 545|362162|\n", + "| 300|362162|\n", + "| 390|362162|\n", + "| 214|362162|\n", + "| 430|362162|\n", + "| 553|362162|\n", + "| 145|362162|\n", + "| 303|362162|\n", + "| 389|362162|\n", + "| 262|362162|\n", + "| 581|362162|\n", + "| 148|362162|\n", + "| 204|362162|\n", + "| 234|362162|\n", + "| 413|362162|\n", + "| 201|362162|\n", + "| 294|362162|\n", + "| 548|362162|\n", + "| 557|362162|\n", + "| 251|362162|\n", + "| 211|362162|\n", + "| 418|362162|\n", + "| 550|362162|\n", + "| 220|362162|\n", + "| 301|362162|\n", + "| 230|362162|\n", + "| 306|362162|\n", + "| 187|362162|\n", + "| 302|362162|\n", + "| 362|362162|\n", + "| 451|362162|\n", + "| 209|362162|\n", + "| 428|362162|\n", + "| 448|362162|\n", "| 340|362162|\n", - "| 479|362162|\n", - "| 346|362162|\n", - "| 121|362162|\n", - "| 352|362162|\n", - "| 353|362162|\n", - "| 83|362162|\n", - "| 313|362162|\n", - "| 499|362162|\n", - "| 208|362162|\n", - "| 493|362162|\n", + "| 250|362162|\n", + "| 235|362162|\n", "| 507|362162|\n", - "| 81|362162|\n", - "| 80|362162|\n", - "| 491|362162|\n", - "| 98|362162|\n", - "| 504|362162|\n", - "| 355|362162|\n", - "| 341|362162|\n", - "| 381|362162|\n", - "| 103|362162|\n", - "| 351|362162|\n", - "| 573|362162|\n", - "| 568|362162|\n", - "| 445|362162|\n", - "| 380|362162|\n", - "| 144|362162|\n", - "| 486|362162|\n", - "| 497|362162|\n", - "| 350|362162|\n", - "| 386|362163|\n", - "| 383|362163|\n", - "| 205|362163|\n", - "| 84|362163|\n", - "| 495|362163|\n", - "| 82|362163|\n", - "| 305|362163|\n", - "| 214|362163|\n", - "| 221|362163|\n", - "| 145|362163|\n", - "| 382|362163|\n", + "| 544|362162|\n", + "| 558|362162|\n", + "| 92|362162|\n", + "| 433|362162|\n", + "| 200|362162|\n", + "| 146|362162|\n", + "| 228|362162|\n", + "| 426|362162|\n", + "| 304|362162|\n", + "| 260|362162|\n", + "| 147|362162|\n", + "| 454|362162|\n", + "| 363|362162|\n", + "| 522|362162|\n", + "| 231|362162|\n", + "| 410|362162|\n", + "| 417|362162|\n", + "| 516|362162|\n", + "| 175|362162|\n", + "| 252|362162|\n", + "| 521|362162|\n", + "| 89|362162|\n", + "| 555|362162|\n", + "| 229|362162|\n", + "| 305|362162|\n", + "| 238|362163|\n", + "| 343|362163|\n", + "| 391|362163|\n", + "| 367|362163|\n", + "| 245|362163|\n", + "| 411|362163|\n", + "| 511|362163|\n", + "| 366|362163|\n", + "| 88|362163|\n", + "| 198|362163|\n", + "| 546|362163|\n", + "| 91|362163|\n", + "| 393|362163|\n", + "| 241|362163|\n", + "| 93|362163|\n", + "| 450|362163|\n", + "| 549|362163|\n", + "| 368|362163|\n", + "| 243|362163|\n", + "| 247|362163|\n", + "| 360|362163|\n", + "| 359|362163|\n", + "| 346|362163|\n", + "| 412|362163|\n", + "| 364|362163|\n", + "| 510|362163|\n", + "| 226|362163|\n", + "| 255|362163|\n", + "| 419|362163|\n", + "| 90|362163|\n", + "| 227|362163|\n", + "| 455|362163|\n", + "| 239|362163|\n", "| 207|362163|\n", - "| 122|362163|\n", - "| 498|362163|\n", - "| 206|362163|\n", - "| 309|362163|\n", - "| 342|362163|\n", - "| 220|362163|\n", - "| 127|362163|\n", - "| 85|362163|\n", - "| 314|362163|\n", - "| 501|362163|\n", - "| 300|362163|\n", - "| 308|362163|\n", - "| 500|362163|\n", - "| 312|362163|\n", - "| 394|362163|\n", - "| 146|362163|\n", - "| 489|362163|\n", - "| 228|362163|\n", - "| 304|362163|\n", - "| 354|362163|\n", - "| 574|362163|\n", - "| 480|362163|\n", - "| 349|362163|\n", - "| 494|362163|\n", + "| 199|362163|\n", + "| 392|362163|\n", + "| 248|362163|\n", + "| 365|362163|\n", + "| 263|362163|\n", + "| 202|362163|\n", + "| 388|362163|\n", "| 215|362163|\n", - "| 129|362164|\n", - "| 390|362164|\n", - "| 147|362164|\n", - "| 393|362164|\n", - "| 128|362164|\n", - "| 210|362164|\n", - "| 209|362164|\n", - "| 216|362164|\n", - "| 233|362164|\n", - "| 487|362164|\n", - "| 444|362164|\n", - "| 348|362164|\n", - "| 345|362164|\n", - "| 496|362164|\n", - "| 303|362164|\n", - "| 387|362164|\n", - "| 123|362164|\n", - "| 301|362164|\n", - "| 204|362164|\n", - "| 137|362164|\n", - "| 138|362164|\n", - "| 306|362164|\n", - "| 229|362164|\n", - "| 481|362164|\n", + "| 505|362163|\n", + "| 387|362163|\n", + "| 184|362163|\n", + "| 296|362163|\n", + "| 509|362163|\n", + "| 402|362163|\n", + "| 394|362163|\n", + "| 240|362163|\n", + "| 256|362163|\n", + "| 341|362163|\n", + "| 414|362163|\n", + "| 203|362163|\n", + "| 254|362163|\n", + "| 213|362163|\n", + "| 259|362163|\n", + "| 358|362163|\n", + "| 342|362163|\n", + "| 212|362163|\n", + "| 523|362163|\n", + "| 297|362163|\n", + "| 344|362163|\n", + "| 253|362163|\n", + "| 386|362163|\n", + "| 205|362163|\n", "| 385|362164|\n", + "| 246|362164|\n", + "| 350|362164|\n", + "| 94|362164|\n", + "| 397|362164|\n", + "| 357|362164|\n", + "| 176|362164|\n", "| 395|362164|\n", - "| 343|362164|\n", - "| 347|362164|\n", - "| 488|362164|\n", - "| 155|362164|\n", - "| 575|362164|\n", - "| 211|362164|\n", - "| 302|362164|\n", - "| 311|362164|\n", - "| 219|362164|\n", - "| 217|362164|\n", - "| 307|362164|\n", - "| 124|362165|\n", - "| 287|362165|\n", - "| 154|362165|\n", - "| 297|362165|\n", - "| 299|362165|\n", - "| 389|362165|\n", - "| 234|362165|\n", - "| 126|362165|\n", - "| 443|362165|\n", - "| 396|362165|\n", - "| 482|362165|\n", - "| 151|362165|\n", - "| 213|362165|\n", - "| 222|362165|\n", - "| 152|362165|\n", + "| 216|362164|\n", + "| 356|362164|\n", + "| 396|362164|\n", + "| 415|362164|\n", + "| 257|362164|\n", + "| 299|362164|\n", + "| 177|362164|\n", + "| 458|362164|\n", + "| 298|362164|\n", + "| 186|362164|\n", + "| 503|362164|\n", + "| 456|362164|\n", + "| 185|362164|\n", + "| 355|362164|\n", + "| 376|362164|\n", + "| 424|362164|\n", + "| 420|362164|\n", + "| 244|362164|\n", + "| 377|362165|\n", + "| 178|362165|\n", + "| 459|362165|\n", + "| 293|362165|\n", + "| 375|362165|\n", + "| 265|362165|\n", + "| 264|362165|\n", + "| 354|362165|\n", + "| 242|362165|\n", + "| 398|362165|\n", + "| 353|362165|\n", + "| 183|362165|\n", + "| 267|362165|\n", + "| 96|362165|\n", + "| 351|362165|\n", + "| 457|362165|\n", + "| 504|362165|\n", + "| 347|362165|\n", + "| 369|362165|\n", + "| 373|362165|\n", + "| 349|362165|\n", "| 218|362165|\n", - "| 156|362165|\n", - "| 344|362165|\n", - "| 310|362165|\n", - "| 485|362165|\n", - "| 140|362165|\n", - "| 153|362165|\n", - "| 388|362165|\n", - "| 230|362165|\n", - "| 392|362165|\n", - "| 223|362165|\n", - "| 232|362165|\n", - "| 125|362165|\n", - "| 212|362165|\n", - "| 148|362165|\n", - "| 131|362165|\n", - "| 149|362165|\n", - "| 384|362165|\n", - "| 441|362165|\n", - "| 130|362165|\n", - "| 400|362166|\n", - "| 225|362166|\n", - "| 132|362166|\n", - "| 136|362166|\n", - "| 235|362166|\n", - "| 227|362166|\n", - "| 157|362166|\n", - "| 397|362166|\n", - "| 133|362166|\n", - "| 163|362166|\n", + "| 401|362165|\n", + "| 217|362165|\n", + "| 372|362165|\n", + "| 400|362165|\n", + "| 95|362165|\n", + "| 345|362165|\n", + "| 502|362165|\n", + "| 348|362165|\n", + "| 266|362166|\n", + "| 381|362166|\n", + "| 290|362166|\n", + "| 495|362166|\n", + "| 97|362166|\n", + "| 462|362166|\n", + "| 98|362166|\n", + "| 371|362166|\n", + "| 382|362166|\n", + "| 370|362166|\n", + "| 383|362166|\n", + "| 423|362166|\n", + "| 144|362166|\n", + "| 399|362166|\n", + "| 422|362166|\n", + "| 292|362166|\n", "| 291|362166|\n", - "| 440|362166|\n", - "| 298|362166|\n", - "| 231|362166|\n", - "| 236|362166|\n", - "| 142|362166|\n", - "| 294|362166|\n", - "| 160|362166|\n", - "| 238|362166|\n", - "| 139|362166|\n", - "| 391|362166|\n", - "| 403|362166|\n", - "| 237|362166|\n", - "| 484|362166|\n", - "| 296|362166|\n", - "| 226|362166|\n", - "| 150|362166|\n", - "| 134|362166|\n", + "| 352|362166|\n", + "| 488|362166|\n", + "| 378|362166|\n", + "| 421|362166|\n", + "| 179|362166|\n", + "| 461|362166|\n", + "| 384|362166|\n", + "| 374|362166|\n", + "| 99|362166|\n", "| 483|362166|\n", - "| 224|362166|\n", - "| 401|362166|\n", - "| 164|362166|\n", - "| 166|362166|\n", - "| 398|362166|\n", - "| 292|362166|\n", - "| 442|362166|\n", - "| 165|362167|\n", - "| 295|362167|\n", - "| 438|362167|\n", - "| 239|362167|\n", - "| 167|362167|\n", - "| 265|362167|\n", - "| 141|362167|\n", - "| 402|362167|\n", - "| 161|362167|\n", - "| 431|362167|\n", + "| 460|362166|\n", + "| 100|362167|\n", + "| 464|362167|\n", + "| 494|362167|\n", + "| 501|362167|\n", "| 288|362167|\n", - "| 293|362167|\n", - "| 404|362167|\n", - "| 439|362167|\n", - "| 430|362167|\n", - "| 289|362167|\n", - "| 437|362167|\n", - "| 135|362167|\n", - "| 158|362167|\n", - "| 143|362167|\n", - "| 410|362167|\n", - "| 399|362167|\n", - "| 290|362167|\n", - "| 168|362167|\n", - "| 162|362167|\n", - "| 169|362168|\n", - "| 241|362168|\n", - "| 170|362168|\n", - "| 432|362168|\n", - "| 243|362168|\n", - "| 415|362168|\n", - "| 258|362168|\n", - "| 240|362168|\n", - "| 411|362168|\n", - "| 433|362168|\n", - "| 254|362168|\n", - "| 436|362168|\n", - "| 405|362168|\n", - "| 416|362168|\n", - "| 417|362168|\n", - "| 407|362168|\n", - "| 159|362168|\n", - "| 262|362168|\n", + "| 463|362167|\n", + "| 484|362167|\n", + "| 489|362167|\n", + "| 101|362167|\n", + "| 499|362167|\n", + "| 497|362167|\n", + "| 180|362167|\n", + "| 493|362167|\n", + "| 268|362167|\n", + "| 379|362167|\n", + "| 182|362167|\n", + "| 108|362167|\n", + "| 491|362168|\n", + "| 181|362168|\n", + "| 496|362168|\n", + "| 380|362168|\n", + "| 485|362168|\n", + "| 289|362168|\n", + "| 480|362168|\n", + "| 287|362168|\n", + "| 109|362168|\n", + "| 269|362168|\n", + "| 486|362168|\n", + "| 466|362168|\n", "| 285|362168|\n", - "| 203|362168|\n", - "| 414|362168|\n", - "| 286|362168|\n", - "| 242|362168|\n", - "| 406|362168|\n", - "| 264|362169|\n", - "| 255|362169|\n", - "| 412|362169|\n", - "| 266|362169|\n", - "| 409|362169|\n", - "| 434|362169|\n", - "| 419|362169|\n", - "| 263|362169|\n", - "| 178|362169|\n", - "| 267|362169|\n", - "| 171|362169|\n", - "| 249|362169|\n", - "| 245|362169|\n", - "| 256|362169|\n", - "| 413|362169|\n", - "| 175|362169|\n", - "| 260|362169|\n", - "| 253|362169|\n", - "| 284|362169|\n", - "| 250|362169|\n", - "| 418|362169|\n", - "| 261|362169|\n", - "| 173|362169|\n", - "| 435|362169|\n", - "| 246|362170|\n", - "| 280|362170|\n", - "| 408|362170|\n", - "| 179|362170|\n", - "| 202|362170|\n", - "| 259|362170|\n", + "| 500|362168|\n", + "| 143|362168|\n", + "| 490|362168|\n", + "| 105|362169|\n", + "| 286|362169|\n", + "| 465|362169|\n", + "| 102|362169|\n", + "| 481|362169|\n", + "| 471|362169|\n", + "| 487|362169|\n", + "| 473|362169|\n", + "| 106|362169|\n", + "| 270|362169|\n", + "| 498|362169|\n", + "| 468|362169|\n", + "| 103|362169|\n", + "| 492|362169|\n", + "| 467|362169|\n", + "| 479|362169|\n", + "| 482|362169|\n", + "| 111|362169|\n", + "| 107|362169|\n", + "| 271|362170|\n", "| 281|362170|\n", - "| 172|362170|\n", - "| 252|362170|\n", - "| 422|362170|\n", - "| 244|362170|\n", - "| 429|362170|\n", - "| 247|362170|\n", "| 278|362170|\n", - "| 248|362170|\n", - "| 199|362170|\n", + "| 142|362170|\n", + "| 277|362170|\n", + "| 476|362170|\n", + "| 469|362170|\n", + "| 272|362170|\n", + "| 110|362170|\n", + "| 284|362170|\n", + "| 274|362170|\n", + "| 275|362170|\n", + "| 112|362170|\n", "| 282|362170|\n", - "| 420|362170|\n", - "| 174|362170|\n", - "| 176|362171|\n", + "| 475|362170|\n", + "| 276|362170|\n", + "| 104|362170|\n", + "| 474|362171|\n", + "| 472|362171|\n", "| 283|362171|\n", - "| 182|362171|\n", - "| 269|362171|\n", - "| 279|362171|\n", - "| 423|362171|\n", - "| 251|362171|\n", - "| 177|362171|\n", - "| 197|362171|\n", - "| 180|362171|\n", - "| 257|362171|\n", - "| 198|362171|\n", - "| 428|362171|\n", - "| 427|362171|\n", - "| 268|362171|\n", - "| 200|362171|\n", - "| 425|362171|\n", - "| 276|362171|\n", - "| 274|362171|\n", - "| 270|362171|\n", - "| 275|362171|\n", - "| 271|362171|\n", - "| 201|362171|\n", - "| 421|362171|\n", - "| 277|362172|\n", - "| 181|362172|\n", - "| 424|362172|\n", - "| 426|362172|\n", - "| 183|362172|\n", - "| 272|362172|\n", - "| 184|362172|\n", - "| 186|362172|\n", - "| 273|362173|\n", - "| 190|362173|\n", - "| 185|362173|\n", - "| 192|362174|\n", - "| 187|362174|\n", - "| 191|362174|\n", - "| 193|362175|\n", - "| 194|362175|\n", - "| 189|362175|\n", - "| 196|362175|\n", - "| 188|362175|\n", - "| 195|362175|\n", + "| 113|362171|\n", + "| 478|362171|\n", + "| 470|362171|\n", + "| 273|362171|\n", + "| 477|362171|\n", + "| 136|362172|\n", + "| 135|362172|\n", + "| 114|362172|\n", + "| 115|362172|\n", + "| 138|362172|\n", + "| 279|362172|\n", + "| 280|362172|\n", + "| 133|362173|\n", + "| 128|362173|\n", + "| 129|362173|\n", + "| 139|362173|\n", + "| 132|362173|\n", + "| 134|362173|\n", + "| 137|362173|\n", + "| 141|362173|\n", + "| 116|362173|\n", + "| 140|362173|\n", + "| 131|362173|\n", + "| 117|362174|\n", + "| 127|362174|\n", + "| 119|362174|\n", + "| 130|362174|\n", + "| 118|362174|\n", + "| 126|362175|\n", + "| 120|362175|\n", + "| 121|362176|\n", + "| 125|362176|\n", + "| 123|362177|\n", + "| 124|362177|\n", + "| 122|362177|\n", "+-----------+------+\n", "\n" ] @@ -1936,7 +2185,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 8, "id": "3a1943f1-e3fd-44c6-a125-30268d45b7a6", "metadata": {}, "outputs": [], @@ -1946,7 +2195,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 9, "id": "04f8d859-1872-4496-b211-6f8a72469c2c", "metadata": {}, "outputs": [ @@ -1954,14 +2203,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 294:====================================================>(597 + 3) / 600]\r" + "23/11/24 23:44:31 WARN com.github.fommil.netlib.BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS\n", + "23/11/24 23:44:32 WARN com.github.fommil.netlib.BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS\n", + "[Stage 118:====================================================>(598 + 2) / 600]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Silhouette with squared euclidean distance = 0.6724456252751287\n" + "Silhouette with squared euclidean distance = 0.786760389836245\n" ] }, { @@ -2001,7 +2252,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 10, "id": "86b58900-850f-4a83-82be-91ffc6f62be7", "metadata": {}, "outputs": [ @@ -2010,9 +2261,9 @@ "output_type": "stream", "text": [ "Cluster Centers: \n", - "[ 1.80161164 869.01481622 -1588.97018395]\n", - "[ 4.18828318 871.29960529 -1593.60879614]\n", - "[ 5.21132153e-01 8.70095495e+02 -1.58910819e+03]\n" + "[ 1.13637426e+00 8.69572453e+02 -1.58903761e+03]\n", + "[ 4.19114642 871.29986067 -1593.60698384]\n", + "[ 4.19114642 871.2953187 -1593.62644601]\n" ] } ], @@ -2026,7 +2277,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 11, "id": "b7acfa60-e70a-4d9c-a120-a9e71c037148", "metadata": {}, "outputs": [], @@ -2037,7 +2288,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 12, "id": "ed1bed2e-98ab-4391-82a9-e98353394258", "metadata": {}, "outputs": [ @@ -2045,7 +2296,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 303:> (0 + 1) / 1]\r" + "[Stage 122:====================================================>(590 + 3) / 593]\r" ] }, { @@ -2055,20 +2306,20 @@ "+-----------+-------------+--------------+--------------------+--------------------+----------+\n", "|pickup_area| pickup_lat| pickup_lon| features| scaled_features|prediction|\n", "+-----------+-------------+--------------+--------------------+--------------------+----------+\n", - "| 41|41.8016710371|-87.5942656985|[41.0,41.80167103...|[2.26101319928869...| 0|\n", - "| 5|41.9448137543|-87.6907750098|[5.0,41.944813754...|[0.27573331698642...| 2|\n", - "| 6|41.9359889065|-87.6709663837|[6.0,41.935988906...|[0.33087998038371...| 2|\n", - "| 6| 41.936159071|-87.6612652184|[6.0,41.936159071...|[0.33087998038371...| 2|\n", - "| 23|41.9066839592|-87.7103539349|[23.0,41.90668395...|[1.26837325813756...| 2|\n", - "| 41|41.8012268363|-87.5853031602|[41.0,41.80122683...|[2.26101319928869...| 0|\n", - "| 6| 41.942577185|-87.6470785093|[6.0,41.942577185...|[0.33087998038371...| 2|\n", - "| 6| 41.942577185|-87.6470785093|[6.0,41.942577185...|[0.33087998038371...| 2|\n", - "| 7|41.9217781876|-87.6510618838|[7.0,41.921778187...|[0.38602664378099...| 2|\n", - "| 7|41.9217781876|-87.6510618838|[7.0,41.921778187...|[0.38602664378099...| 2|\n", - "| 7|41.9217781876|-87.6510618838|[7.0,41.921778187...|[0.38602664378099...| 2|\n", - "| 3|41.9724370811|-87.6711095263|[3.0,41.972437081...|[0.16543999019185...| 2|\n", - "| 28|41.8773834707|-87.6806541164|[28.0,41.87738347...|[1.54410657512398...| 0|\n", - "| 16|41.9595267906|-87.7020985409|[16.0,41.95952679...|[0.88234661435656...| 2|\n", + "| 1|42.0016981937|-87.6735740325|[1.0,42.001698193...|[0.05514666339728...| 0|\n", + "| 6| 41.936159071|-87.6612652184|[6.0,41.936159071...|[0.33087998038371...| 0|\n", + "| 23|41.9066839592|-87.7103539349|[23.0,41.90668395...|[1.26837325813755...| 0|\n", + "| 6| 41.942577185|-87.6470785093|[6.0,41.942577185...|[0.33087998038371...| 0|\n", + "| 7|41.9217781876|-87.6510618838|[7.0,41.921778187...|[0.38602664378099...| 0|\n", + "| 3|41.9724370811|-87.6711095263|[3.0,41.972437081...|[0.16543999019185...| 0|\n", + "| 31|41.8561441046|-87.6489783241|[31.0,41.85614410...|[1.70954656531584...| 0|\n", + "| 32|41.8809944707|-87.6327464887|[32.0,41.88099447...|[1.76469322871312...| 0|\n", + "| 32|41.8809944707|-87.6327464887|[32.0,41.88099447...|[1.76469322871312...| 0|\n", + "| 32|41.8809944707|-87.6327464887|[32.0,41.88099447...|[1.76469322871312...| 0|\n", + "| 32|41.8809944707|-87.6327464887|[32.0,41.88099447...|[1.76469322871312...| 0|\n", + "| 32|41.8809944707|-87.6327464887|[32.0,41.88099447...|[1.76469322871312...| 0|\n", + "| 32|41.8809944707|-87.6327464887|[32.0,41.88099447...|[1.76469322871312...| 0|\n", + "| 32|41.8809944707|-87.6327464887|[32.0,41.88099447...|[1.76469322871312...| 0|\n", "| 32|41.8809944707|-87.6327464887|[32.0,41.88099447...|[1.76469322871312...| 0|\n", "| 32|41.8809944707|-87.6327464887|[32.0,41.88099447...|[1.76469322871312...| 0|\n", "| 32|41.8809944707|-87.6327464887|[32.0,41.88099447...|[1.76469322871312...| 0|\n", @@ -2095,7 +2346,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "ebe1200e-f37a-47bd-a78a-a74fafc4145f", "metadata": {}, "outputs": [ @@ -2103,1324 +2354,35 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 307:> (0 + 0) / 534][Stage 308:> (0 + 0) / 534]\r" + " \r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(147900469, 6)\n" ] } ], "source": [ + "# We have 147 million rows after dropping the NAs:\n", "print((chicago_clustering.count(), len(chicago_clustering.columns)))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "1a2f3163-097d-4ca3-8105-d891019a7bad", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Exception in thread \"YARN application state monitor\" java.lang.OutOfMemoryError: GC overhead limit exceeded\n", - "\tat com.sun.proxy.$Proxy12.getApplicationReport(Unknown Source)\n", - "\tat org.apache.hadoop.yarn.api.impl.pb.client.ApplicationClientProtocolPBClientImpl.getApplicationReport(ApplicationClientProtocolPBClientImpl.java:256)\n", - "\tat sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source)\n", - "\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n", - "\tat java.lang.reflect.Method.invoke(Method.java:498)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)\n", - "\tat com.sun.proxy.$Proxy13.getApplicationReport(Unknown Source)\n", - "\tat org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.getApplicationReport(YarnClientImpl.java:535)\n", - "\tat org.apache.spark.deploy.yarn.Client.getApplicationReport(Client.scala:345)\n", - "\tat org.apache.spark.deploy.yarn.Client.monitorApplication(Client.scala:1066)\n", - "\tat org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend$MonitorThread.run(YarnClientSchedulerBackend.scala:117)\n", - "23/11/24 22:27:46 ERROR com.codahale.metrics.ScheduledReporter: Exception thrown from StatsdReporter#report. Exception was suppressed.\n", - "java.lang.OutOfMemoryError: GC overhead limit exceeded\n", - "\tat java.util.Collections$UnmodifiableMap$UnmodifiableEntrySet.iterator(Collections.java:1656)\n", - "\tat scala.collection.convert.Wrappers$JSetWrapper.iterator(Wrappers.scala:163)\n", - "\tat scala.collection.IterableLike.foreach(IterableLike.scala:74)\n", - "\tat scala.collection.IterableLike.foreach$(IterableLike.scala:73)\n", - "\tat scala.collection.AbstractIterable.foreach(Iterable.scala:56)\n", - "\tat org.apache.spark.metrics.sink.StatsdReporter.$anonfun$report$7(StatsdReporter.scala:76)\n", - "\tat org.apache.spark.metrics.sink.StatsdReporter$$Lambda$2237/1648211405.apply$mcV$sp(Unknown Source)\n", - "\tat scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\n", - "\tat scala.util.Try$.apply(Try.scala:213)\n", - "\tat org.apache.spark.metrics.sink.StatsdReporter.report(StatsdReporter.scala:75)\n", - "\tat com.codahale.metrics.ScheduledReporter.report(ScheduledReporter.java:237)\n", - "\tat com.codahale.metrics.ScheduledReporter.lambda$start$0(ScheduledReporter.java:177)\n", - "\tat com.codahale.metrics.ScheduledReporter$$Lambda$897/1780696444.run(Unknown Source)\n", - "\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n", - "\tat java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)\n", - "\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)\n", - "\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)\n", - "\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n", - "\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 WARN org.apache.hadoop.ipc.Client: Exception encountered while connecting to the server \n", - "org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): appattempt_1700850934132_0002_000001 not found in AMRMTokenSecretManager.\n", - "\tat org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:374)\n", - "\tat org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:623)\n", - "\tat org.apache.hadoop.ipc.Client$Connection.access$2300(Client.java:414)\n", - "\tat org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:842)\n", - "\tat org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:838)\n", - "\tat java.security.AccessController.doPrivileged(Native Method)\n", - "\tat javax.security.auth.Subject.doAs(Subject.java:422)\n", - "\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)\n", - "\tat org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:838)\n", - "\tat org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:414)\n", - "\tat org.apache.hadoop.ipc.Client.getConnection(Client.java:1654)\n", - "\tat org.apache.hadoop.ipc.Client.call(Client.java:1479)\n", - "\tat org.apache.hadoop.ipc.Client.call(Client.java:1432)\n", - "\tat org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:231)\n", - "\tat org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)\n", - "\tat com.sun.proxy.$Proxy20.allocate(Unknown Source)\n", - "\tat org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.allocate(ApplicationMasterProtocolPBClientImpl.java:77)\n", - "\tat sun.reflect.GeneratedMethodAccessor60.invoke(Unknown Source)\n", - "\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n", - "\tat java.lang.reflect.Method.invoke(Method.java:498)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)\n", - "\tat com.sun.proxy.$Proxy21.allocate(Unknown Source)\n", - "\tat org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl.allocate(AMRMClientImpl.java:325)\n", - "\tat org.apache.spark.deploy.yarn.YarnAllocator.allocateResources(YarnAllocator.scala:401)\n", - "\tat org.apache.spark.deploy.yarn.ApplicationMaster.org$apache$spark$deploy$yarn$ApplicationMaster$$allocationThreadImpl(ApplicationMaster.scala:576)\n", - "\tat org.apache.spark.deploy.yarn.ApplicationMaster$$anon$1.run(ApplicationMaster.scala:646)\n", - "23/11/24 22:27:46 WARN org.apache.spark.HeartbeatReceiver: Removing executor 41 with no recent heartbeats: 606566 ms exceeds timeout 120000 ms\n", - "23/11/24 22:27:46 WARN org.apache.spark.network.server.TransportChannelHandler: Exception in connection from /10.128.0.13:49396\n", - "java.lang.OutOfMemoryError: GC overhead limit exceeded\n", - "23/11/24 22:27:46 WARN org.apache.spark.network.server.TransportChannelHandler: Exception in connection from /10.128.0.13:46910\n", - "java.lang.OutOfMemoryError: GC overhead limit exceeded\n", - "23/11/24 22:27:46 WARN org.apache.spark.HeartbeatReceiver: Removing executor 44 with no recent heartbeats: 606187 ms exceeds timeout 120000 ms\n", - "23/11/24 22:27:46 WARN org.apache.spark.HeartbeatReceiver: Removing executor 43 with no recent heartbeats: 606566 ms exceeds timeout 120000 ms\n", - "23/11/24 22:27:46 WARN org.apache.spark.rpc.netty.NettyRpcEnv: Ignored message: true\n", - "23/11/24 22:27:46 WARN org.apache.spark.deploy.yarn.ApplicationMaster: Reporter thread fails 1 time(s) in a row.\n", - "org.apache.hadoop.security.token.SecretManager$InvalidToken: appattempt_1700850934132_0002_000001 not found in AMRMTokenSecretManager.\n", - "\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n", - "\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n", - "\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n", - "\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n", - "\tat org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53)\n", - "\tat org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80)\n", - "\tat org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119)\n", - "\tat org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.allocate(ApplicationMasterProtocolPBClientImpl.java:79)\n", - "\tat sun.reflect.GeneratedMethodAccessor60.invoke(Unknown Source)\n", - "\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n", - "\tat java.lang.reflect.Method.invoke(Method.java:498)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)\n", - "\tat com.sun.proxy.$Proxy21.allocate(Unknown Source)\n", - "\tat org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl.allocate(AMRMClientImpl.java:325)\n", - "\tat org.apache.spark.deploy.yarn.YarnAllocator.allocateResources(YarnAllocator.scala:401)\n", - "\tat org.apache.spark.deploy.yarn.ApplicationMaster.org$apache$spark$deploy$yarn$ApplicationMaster$$allocationThreadImpl(ApplicationMaster.scala:576)\n", - "\tat org.apache.spark.deploy.yarn.ApplicationMaster$$anon$1.run(ApplicationMaster.scala:646)\n", - "Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): appattempt_1700850934132_0002_000001 not found in AMRMTokenSecretManager.\n", - "\tat org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1589)\n", - "\tat org.apache.hadoop.ipc.Client.call(Client.java:1535)\n", - "\tat org.apache.hadoop.ipc.Client.call(Client.java:1432)\n", - "\tat org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:231)\n", - "\tat org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)\n", - "\tat com.sun.proxy.$Proxy20.allocate(Unknown Source)\n", - "\tat org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.allocate(ApplicationMasterProtocolPBClientImpl.java:77)\n", - "\t... 13 more\n", - "23/11/24 22:27:46 ERROR org.apache.spark.scheduler.cluster.YarnScheduler: Lost executor 43 on hub-msca-bdp-dphub-students-rohitk-w-2.c.msca-bdp-student-ap.internal: Executor heartbeat timed out after 606566 ms\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7062424549637989301,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6032750637647102155,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.13:49396; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5268706045776405251,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5591622169200431532,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5892056795421893606,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.13:49396; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7342795057508538874,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=9002533160262250437,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.13:49396; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7171009396498015550,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5123703343359460651,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.13:49396; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5030036934419348835,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8061176774465780970,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.13:49396; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6275514956338655765,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7828384139997219750,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.13:49396; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5234905306283778916,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7789772703993336541,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.13:49396; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6800628084432341581,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8213585359316955070,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.13:49396; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5131226224466442418,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8617269945376389868,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.13:49396; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8058121759705382362,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7416385524504093420,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.13:49396; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=4642252440098860807,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.13:49396; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6509882082312688553,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.13:49396; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7634120766875275351,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7758302538454388336,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7978209826066971467,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7089452522359168993,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5630724675740277167,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7760660057606289933,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8248972273521563715,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6446068527737886466,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7989704962497205119,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5067183351138927627,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5541095263609571894,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6969817484473126207,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5550732265229726279,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7190263940346000551,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8911400010236913338,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6527343649626953451,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6836995330930616345,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8530956988707859204,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8003062449364760209,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=4732309432786122124,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7337997448924489870,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7322390798941227700,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6992245375304881473,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8314434094591263209,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=4899434880742290427,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6029321391095087720,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8617063757393656819,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5993956207069172087,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6587574803061694452,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5865241218922505797,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=4771336953810389767,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7894267170970360490,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7701602773991044291,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8841052770018393551,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=9045520713432016446,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8438087716647885123,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=5324784523455447518,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7771201177250886556,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7993906302975017186,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8932539557589202258,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=4953530044374065889,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7270169998797811892,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8018870935869479309,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6637838094204597345,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8705192790647397879,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=8840047058856808637,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThre" - ] - }, - { - "ename": "Py4JJavaError", - "evalue": "An error occurred while calling o1622.collectToPython.\n: java.lang.OutOfMemoryError: GC overhead limit exceeded\n", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mPy4JJavaError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[29], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Convert Spark DataFrame to Pandas DataFrame\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m pandas_df \u001b[38;5;241m=\u001b[39m \u001b[43mchicago_clustering\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtoPandas\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# Plotting chicago_clustering:\u001b[39;00m\n\u001b[1;32m 5\u001b[0m gdf \u001b[38;5;241m=\u001b[39m gpd\u001b[38;5;241m.\u001b[39mGeoDataFrame(pandas_df, geometry\u001b[38;5;241m=\u001b[39mgpd\u001b[38;5;241m.\u001b[39mpoints_from_xy(pandas_df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpickup_lon\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mfloat\u001b[39m), pandas_df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpickup_lat\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mfloat\u001b[39m)))\n", - "File \u001b[0;32m/usr/lib/spark/python/pyspark/sql/pandas/conversion.py:141\u001b[0m, in \u001b[0;36mPandasConversionMixin.toPandas\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 138\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m\n\u001b[1;32m 140\u001b[0m \u001b[38;5;66;03m# Below is toPandas without Arrow optimization.\u001b[39;00m\n\u001b[0;32m--> 141\u001b[0m pdf \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mDataFrame\u001b[38;5;241m.\u001b[39mfrom_records(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m, columns\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcolumns)\n\u001b[1;32m 142\u001b[0m column_counter \u001b[38;5;241m=\u001b[39m Counter(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcolumns)\n\u001b[1;32m 144\u001b[0m dtype \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28;01mNone\u001b[39;00m] \u001b[38;5;241m*\u001b[39m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mschema)\n", - "File \u001b[0;32m/usr/lib/spark/python/pyspark/sql/dataframe.py:677\u001b[0m, in \u001b[0;36mDataFrame.collect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 667\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Returns all the records as a list of :class:`Row`.\u001b[39;00m\n\u001b[1;32m 668\u001b[0m \n\u001b[1;32m 669\u001b[0m \u001b[38;5;124;03m.. versionadded:: 1.3.0\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 674\u001b[0m \u001b[38;5;124;03m[Row(age=2, name='Alice'), Row(age=5, name='Bob')]\u001b[39;00m\n\u001b[1;32m 675\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 676\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m SCCallSiteSync(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sc) \u001b[38;5;28;01mas\u001b[39;00m css:\n\u001b[0;32m--> 677\u001b[0m sock_info \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_jdf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollectToPython\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 678\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mlist\u001b[39m(_load_from_socket(sock_info, BatchedSerializer(PickleSerializer())))\n", - "File \u001b[0;32m/opt/conda/miniconda3/lib/python3.8/site-packages/py4j/java_gateway.py:1304\u001b[0m, in \u001b[0;36mJavaMember.__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m 1298\u001b[0m command \u001b[38;5;241m=\u001b[39m proto\u001b[38;5;241m.\u001b[39mCALL_COMMAND_NAME \u001b[38;5;241m+\u001b[39m\\\n\u001b[1;32m 1299\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcommand_header \u001b[38;5;241m+\u001b[39m\\\n\u001b[1;32m 1300\u001b[0m args_command \u001b[38;5;241m+\u001b[39m\\\n\u001b[1;32m 1301\u001b[0m proto\u001b[38;5;241m.\u001b[39mEND_COMMAND_PART\n\u001b[1;32m 1303\u001b[0m answer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgateway_client\u001b[38;5;241m.\u001b[39msend_command(command)\n\u001b[0;32m-> 1304\u001b[0m return_value \u001b[38;5;241m=\u001b[39m \u001b[43mget_return_value\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1305\u001b[0m \u001b[43m \u001b[49m\u001b[43manswer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgateway_client\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtarget_id\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1307\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m temp_arg \u001b[38;5;129;01min\u001b[39;00m temp_args:\n\u001b[1;32m 1308\u001b[0m temp_arg\u001b[38;5;241m.\u001b[39m_detach()\n", - "File \u001b[0;32m/usr/lib/spark/python/pyspark/sql/utils.py:111\u001b[0m, in \u001b[0;36mcapture_sql_exception..deco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdeco\u001b[39m(\u001b[38;5;241m*\u001b[39ma, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw):\n\u001b[1;32m 110\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 111\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43ma\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m py4j\u001b[38;5;241m.\u001b[39mprotocol\u001b[38;5;241m.\u001b[39mPy4JJavaError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 113\u001b[0m converted \u001b[38;5;241m=\u001b[39m convert_exception(e\u001b[38;5;241m.\u001b[39mjava_exception)\n", - "File \u001b[0;32m/opt/conda/miniconda3/lib/python3.8/site-packages/py4j/protocol.py:326\u001b[0m, in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m 324\u001b[0m value \u001b[38;5;241m=\u001b[39m OUTPUT_CONVERTER[\u001b[38;5;28mtype\u001b[39m](answer[\u001b[38;5;241m2\u001b[39m:], gateway_client)\n\u001b[1;32m 325\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m answer[\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m==\u001b[39m REFERENCE_TYPE:\n\u001b[0;32m--> 326\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m Py4JJavaError(\n\u001b[1;32m 327\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn error occurred while calling \u001b[39m\u001b[38;5;132;01m{0}\u001b[39;00m\u001b[38;5;132;01m{1}\u001b[39;00m\u001b[38;5;132;01m{2}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39m\n\u001b[1;32m 328\u001b[0m \u001b[38;5;28mformat\u001b[39m(target_id, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m, name), value)\n\u001b[1;32m 329\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 330\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m Py4JError(\n\u001b[1;32m 331\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn error occurred while calling \u001b[39m\u001b[38;5;132;01m{0}\u001b[39;00m\u001b[38;5;132;01m{1}\u001b[39;00m\u001b[38;5;132;01m{2}\u001b[39;00m\u001b[38;5;124m. Trace:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{3}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39m\n\u001b[1;32m 332\u001b[0m \u001b[38;5;28mformat\u001b[39m(target_id, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m, name, value))\n", - "\u001b[0;31mPy4JJavaError\u001b[0m: An error occurred while calling o1622.collectToPython.\n: java.lang.OutOfMemoryError: GC overhead limit exceeded\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "adEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=6654704714059881652,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=7723646830402821403,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 ERROR org.apache.spark.network.server.TransportRequestHandler: Error sending result RpcResponse[requestId=4935209983121636070,body=NioManagedBuffer[buf=java.nio.HeapByteBuffer[pos=0 lim=81 cap=156]]] to /10.128.0.92:33856; closing connection\n", - "java.nio.channels.ClosedChannelException\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.newClosedChannelException(AbstractChannel.java:957)\n", - "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.write(AbstractChannel.java:865)\n", - "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.write(DefaultChannelPipeline.java:1367)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWrite0(AbstractChannelHandlerContext.java:717)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext.invokeWriteAndFlush(AbstractChannelHandlerContext.java:764)\n", - "\tat io.netty.channel.AbstractChannelHandlerContext$WriteTask.run(AbstractChannelHandlerContext.java:1071)\n", - "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n", - "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n", - "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n", - "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n", - "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n", - "\tat java.lang.Thread.run(Thread.java:750)\n", - "23/11/24 22:27:46 WARN org.apache.hadoop.ipc.Client: Exception encountered while connecting to the server \n", - "org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): appattempt_1700850934132_0002_000001 not found in AMRMTokenSecretManager.\n", - "\tat org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:374)\n", - "\tat org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:623)\n", - "\tat org.apache.hadoop.ipc.Client$Connection.access$2300(Client.java:414)\n", - "\tat org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:842)\n", - "\tat org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:838)\n", - "\tat java.security.AccessController.doPrivileged(Native Method)\n", - "\tat javax.security.auth.Subject.doAs(Subject.java:422)\n", - "\tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762)\n", - "\tat org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:838)\n", - "\tat org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:414)\n", - "\tat org.apache.hadoop.ipc.Client.getConnection(Client.java:1654)\n", - "\tat org.apache.hadoop.ipc.Client.call(Client.java:1479)\n", - "\tat org.apache.hadoop.ipc.Client.call(Client.java:1432)\n", - "\tat org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:231)\n", - "\tat org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)\n", - "\tat com.sun.proxy.$Proxy20.allocate(Unknown Source)\n", - "\tat org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.allocate(ApplicationMasterProtocolPBClientImpl.java:77)\n", - "\tat sun.reflect.GeneratedMethodAccessor60.invoke(Unknown Source)\n", - "\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n", - "\tat java.lang.reflect.Method.invoke(Method.java:498)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)\n", - "\tat com.sun.proxy.$Proxy21.allocate(Unknown Source)\n", - "\tat org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl.allocate(AMRMClientImpl.java:325)\n", - "\tat org.apache.spark.deploy.yarn.YarnAllocator.allocateResources(YarnAllocator.scala:401)\n", - "\tat org.apache.spark.deploy.yarn.ApplicationMaster.org$apache$spark$deploy$yarn$ApplicationMaster$$allocationThreadImpl(ApplicationMaster.scala:576)\n", - "\tat org.apache.spark.deploy.yarn.ApplicationMaster$$anon$1.run(ApplicationMaster.scala:646)\n", - "23/11/24 22:27:46 WARN org.apache.spark.deploy.yarn.ApplicationMaster: Reporter thread fails 2 time(s) in a row.\n", - "org.apache.hadoop.security.token.SecretManager$InvalidToken: appattempt_1700850934132_0002_000001 not found in AMRMTokenSecretManager.\n", - "\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n", - "\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n", - "\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n", - "\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n", - "\tat org.apache.hadoop.yarn.ipc.RPCUtil.instantiateException(RPCUtil.java:53)\n", - "\tat org.apache.hadoop.yarn.ipc.RPCUtil.instantiateIOException(RPCUtil.java:80)\n", - "\tat org.apache.hadoop.yarn.ipc.RPCUtil.unwrapAndThrowException(RPCUtil.java:119)\n", - "\tat org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.allocate(ApplicationMasterProtocolPBClientImpl.java:79)\n", - "\tat sun.reflect.GeneratedMethodAccessor60.invoke(Unknown Source)\n", - "\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n", - "\tat java.lang.reflect.Method.invoke(Method.java:498)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)\n", - "\tat org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)\n", - "\tat com.sun.proxy.$Proxy21.allocate(Unknown Source)\n", - "\tat org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl.allocate(AMRMClientImpl.java:325)\n", - "\tat org.apache.spark.deploy.yarn.YarnAllocator.allocateResources(YarnAllocator.scala:401)\n", - "\tat org.apache.spark.deploy.yarn.ApplicationMaster.org$apache$spark$deploy$yarn$ApplicationMaster$$allocationThreadImpl(ApplicationMaster.scala:576)\n", - "\tat org.apache.spark.deploy.yarn.ApplicationMaster$$anon$1.run(ApplicationMaster.scala:646)\n", - "Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): appattempt_1700850934132_0002_000001 not found in AMRMTokenSecretManager.\n", - "\tat org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1589)\n", - "\tat org.apache.hadoop.ipc.Client.call(Client.java:1535)\n", - "\tat org.apache.hadoop.ipc.Client.call(Client.java:1432)\n", - "\tat org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:231)\n", - "\tat org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118)\n", - "\tat com.sun.proxy.$Proxy20.allocate(Unknown Source)\n", - "\tat org.apache.hadoop.yarn.api.impl.pb.client.ApplicationMasterProtocolPBClientImpl.allocate(ApplicationMasterProtocolPBClientImpl.java:77)\n", - "\t... 13 more\n" - ] - } - ], + "outputs": [], "source": [ - "# Convert Spark DataFrame to Pandas DataFrame\n", - "pandas_df = chicago_clustering.toPandas()\n", - "\n", - "# Create GeoDataFrame with simplified geometry\n", - "gdf = gpd.GeoDataFrame(pandas_df, geometry=gpd.points_from_xy(pandas_df['pickup_lon'].astype(float), pandas_df['pickup_lat'].astype(float)))\n", - "gdf['geometry'] = gdf['geometry'].simplify(tolerance=0.001)\n", - "\n", - "# Plot the clusters\n", - "fig, ax = plt.subplots(figsize=(10, 8))\n", - "gdf.plot(ax=ax, column='prediction', legend=True, markersize=50, cmap='viridis', legend_kwds={'label': \"Cluster\"})\n", - "\n", - "# Add labels and title\n", - "plt.xlabel('Longitude')\n", - "plt.ylabel('Latitude')\n", - "plt.title('Clusters of Pickup Areas')\n", + "# TODO:\n", "\n", - "# Show the plot\n", - "plt.show()" + "# Define program area (from EDA)\n", + "# Run clustering on hyde park pre & post program\n", + "# grid search to find optimal ks (and features?)--Ridhi\n", + "# Run plots for all three" ] }, {