Skip to content

Commit

Permalink
resolve failures for dind
Browse files Browse the repository at this point in the history
  • Loading branch information
KiranVelumuri committed Feb 26, 2025
1 parent 92a5335 commit ea17bba
Show file tree
Hide file tree
Showing 19 changed files with 46 additions and 36 deletions.
9 changes: 7 additions & 2 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -135,16 +135,21 @@ def hdbPodTemplate(closure) {
resourceRequestMemory: '6400Mi',
resourceLimitMemory: '12000Mi',
envVars: [
envVar(key: 'DOCKER_HOST', value: 'tcp://localhost:2375')
envVar(key: 'DOCKER_TLS', value: '1'),
envVar(key: 'DOCKER_CERT_PATH', value: '/certs/client')
]
),
containerTemplate(name: 'dind', image: 'docker:18.05-dind',
containerTemplate(name: 'dind', image: 'docker:20.10-dind',
alwaysPullImage: true,
privileged: true,
envVars: [
envVar(key: 'DOCKER_TLS_CERTDIR', value: '/certs')
]
),
],
volumes: [
emptyDirVolume(mountPath: '/var/lib/docker', memory: false),
emptyDirVolume(mountPath: '/certs', memory: false)
], yaml:'''
spec:
securityContext:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,10 @@ private String[] buildLogCmd() {
return new String[] { "docker", "logs", getDockerContainerName() };
}

private String[] lsofCmd() {
return new String[] { "lsof", "-i", ":5432" };
}

private ProcessResults runCmd(String[] cmd, long secondsToWait)
throws IOException, InterruptedException {
LOG.info("Going to run: " + String.join(" ", cmd));
Expand Down Expand Up @@ -111,6 +115,7 @@ private ProcessResults runCmdAndPrintStreams(String[] cmd, long secondsToWait)


public void launchDockerContainer() throws Exception {
runCmdAndPrintStreams(lsofCmd(), 5);
runCmdAndPrintStreams(buildRmCmd(), 600);
if (runCmdAndPrintStreams(buildRunCmd(), 600).rc != 0) {
printDockerEvents();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public PostgresExternalDB() {
}

public String getJdbcUrl() {
return "jdbc:postgresql://" + getContainerHostAddress() + ":5432/" + dbName;
return "jdbc:postgresql://" + getContainerHostAddress() + ":5433/" + dbName;
}

public String getJdbcDriver() {
Expand All @@ -39,7 +39,7 @@ public String getDockerImageName() {
}

public String[] getDockerAdditionalArgs() {
return new String[] {"-p", "5432:5432",
return new String[] {"-p", "5433:5432",
"-e", "POSTGRES_PASSWORD=" + getRootPassword(),
"-e", "POSTGRES_USER=" + getRootUser(),
"-e", "POSTGRES_DB=" + dbName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "book");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ STORED BY
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "book",
Expand Down
4 changes: 2 additions & 2 deletions ql/src/test/queries/clientpositive/jdbc_project_pushdown.q
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ STORED BY
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "book"
Expand All @@ -26,7 +26,7 @@ STORED BY
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "author"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "country");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "country");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.schema" = "bob",
Expand All @@ -25,7 +25,7 @@ CREATE EXTERNAL TABLE country_1 (id int, name varchar(20))
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.schema" = "alice",
Expand All @@ -45,7 +45,7 @@ CREATE EXTERNAL TABLE country_2 (id int, name varchar(20))
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "greg",
"hive.sql.dbcp.password" = "GregPass123!$",
"hive.sql.table" = "country");
Expand Down
2 changes: 1 addition & 1 deletion ql/src/test/queries/clientpositive/qt_database_postgres.q
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ CREATE EXTERNAL TABLE country
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "country"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "book")
Expand All @@ -15,7 +15,7 @@ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "book")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ STORED BY
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "book",
Expand All @@ -30,7 +30,7 @@ STORED BY
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "book",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ STORED BY
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "book"
Expand All @@ -28,7 +28,7 @@ STORED BY
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "book"
Expand All @@ -45,7 +45,7 @@ STORED BY
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "author"
Expand All @@ -62,7 +62,7 @@ STORED BY
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "author"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "country")
Expand All @@ -15,7 +15,7 @@ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "country")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "country")
Expand All @@ -15,7 +15,7 @@ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "country")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.schema" = "bob",
Expand All @@ -16,7 +16,7 @@ STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.schema" = "bob",
Expand Down Expand Up @@ -51,7 +51,7 @@ PREHOOK: query: CREATE EXTERNAL TABLE country_1 (id int, name varchar(20))
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.schema" = "alice",
Expand All @@ -64,7 +64,7 @@ POSTHOOK: query: CREATE EXTERNAL TABLE country_1 (id int, name varchar(20))
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.schema" = "alice",
Expand Down Expand Up @@ -120,7 +120,7 @@ PREHOOK: query: CREATE EXTERNAL TABLE country_2 (id int, name varchar(20))
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "greg",
"hive.sql.dbcp.password" = "GregPass123!$",
"hive.sql.table" = "country")
Expand All @@ -132,7 +132,7 @@ POSTHOOK: query: CREATE EXTERNAL TABLE country_2 (id int, name varchar(20))
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "greg",
"hive.sql.dbcp.password" = "GregPass123!$",
"hive.sql.table" = "country")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ PREHOOK: query: CREATE EXTERNAL TABLE country
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "country"
Expand All @@ -24,7 +24,7 @@ POSTHOOK: query: CREATE EXTERNAL TABLE country
TBLPROPERTIES (
"hive.sql.database.type" = "POSTGRES",
"hive.sql.jdbc.driver" = "org.postgresql.Driver",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5432/qtestDB",
"hive.sql.jdbc.url" = "jdbc:postgresql://localhost:5433/qtestDB",
"hive.sql.dbcp.username" = "qtestuser",
"hive.sql.dbcp.password" = "qtestpassword",
"hive.sql.table" = "country"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ public List<String> filterDataConnectors(List<String> dcList) throws MetaExcepti
private static String mysql_type = "mysql";
private static String mysql_url = "jdbc:mysql://localhost:3306/hive";
private static String postgres_type = "postgres";
private static String postgres_url = "jdbc:postgresql://localhost:5432";
private static String postgres_url = "jdbc:postgresql://localhost:5433";


protected HiveMetaStoreClient createClient(Configuration metaStoreConf) throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3787,7 +3787,7 @@ public void testDataConnector() throws Throwable {
final String mysql_type = "mysql";
final String mysql_url = "jdbc:mysql://nightly1.apache.org:3306/hive1";
final String postgres_type = "postgres";
final String postgres_url = "jdbc:postgresql://localhost:5432";
final String postgres_url = "jdbc:postgresql://localhost:5433";

try {
DataConnector connector = new DataConnector(connector_name1, mysql_type, mysql_url);
Expand Down

0 comments on commit ea17bba

Please sign in to comment.