Merge branch 'REL24_10' into REL24_Q4

build.sh

@@ -340,6 +340,7 @@ initPG () {
     initC "lolor-pg$pgM"      "lolor"      "$lolorV"     "$outPlat" "postgres/lolor"     "" "" "nil"
     initC "snowflake-pg$pgM"  "snowflake"  "$snwflkV"    "$outPlat" "postgres/snowflake" "" "" "nil"
     initC "spock40-pg$pgM"    "spock40"    "$spock40V"   "$outPlat" "postgres/spock40"   "" "" "nil"
+    initC "spock40-pg$pgM"    "spock40"    "$spock40V1"  "$outPlat" "postgres/spock40"   "" "" "nil"
   fi
 
   if [ "$pgM" \> "13" ] && [ "$pgM" \< "17" ]; then
@@ -376,18 +377,18 @@ initPG () {
     initC "plprofiler-pg$pgM" "plprofiler" "$profV"      "$outPlat" "postgres/profiler"   "" "" "nil"
     initC "cron-pg$pgM"       "cron"       "$cronV"      "$outPlat" "postgres/cron"       "" "" "nil"
     initC "setuser-pg$pgM"    "setuser"    "$setuserV"   "$outPlat" "postgres/setuser"    "" "" "nil"
-    initC "orafce-pg$pgM"     "orafce"     "$orafceV"    "$outPlat" "postgres/orafce"     "" "" "nil"
-    initC "postgis-pg$pgM"    "postgis"    "$postgisV"   "$outPlat" "postgres/postgis"    "" "" "nil"
-    initC "plv8-pg$pgM"       "plv8"       "$v8V"        "$outPlat" "postgres/plv8"       "" "" "nil"
-    initC "vector-pg$pgM"     "vector"     "$vectorV"    "$outPlat" "postgres/vector"     "" "" "nil"
-    initC "permissions-pg$pgM" "permissions" "$permissionsV" "$outPlat" "postgres/permissions" "" "" "nil"
-    initC "timescaledb-pg$pgM" "timescaledb" "$timescaleV"   "$outPlat" "postgres/timescale"   "" "" "nil"
   fi
 
   if [ "$pgM" == "15" ] || [ "$pgM" == "16" ]; then
-    initC "wal2json-pg$pgM"    "wal2json"   "$wal2jV"     "$outPlat" "postgres/wal2json"   "" "" "nil"
-    initC "hypopg-pg$pgM"      "hypopg"     "$hypoV"      "$outPlat" "postgres/hypopg"     "" "" "nil"
-    initC "partman-pg$pgM"     "partman"    "$partmanV"   "$outPlat" "postgres/partman"    "" "" "nil"
+    initC "plv8-pg$pgM"       "plv8"       "$v8V"        "$outPlat" "postgres/plv8"       "" "" "nil"
+    initC "wal2json-pg$pgM"   "wal2json"   "$wal2jV"     "$outPlat" "postgres/wal2json"   "" "" "nil"
+    initC "hypopg-pg$pgM"     "hypopg"     "$hypoV"      "$outPlat" "postgres/hypopg"     "" "" "nil"
+    initC "orafce-pg$pgM"     "orafce"     "$orafceV"    "$outPlat" "postgres/orafce"     "" "" "nil"
+    initC "vector-pg$pgM"     "vector"     "$vectorV"    "$outPlat" "postgres/vector"     "" "" "nil"
+    initC "plprofiler-pg$pgM" "plprofiler" "$profV"      "$outPlat" "postgres/profiler"   "" "" "nil"
+    initC "postgis-pg$pgM"    "postgis"    "$postgisV"   "$outPlat" "postgres/postgis"    "" "" "nil"
+    initC "partman-pg$pgM"    "partman"    "$partmanV"   "$outPlat" "postgres/partman"    "" "" "nil"
+
     initC "citus-pg$pgM"       "citus"       "$citusV"       "$outPlat" "postgres/citus"       "" "" "nil"
   fi
 
@@ -437,7 +438,6 @@ do
             setupOutdir
             OS_TYPE="POSIX"
 
-
             cp $CLI/sh/cli.sh     ./pgedge
 
             if [ "$outDir" == "posix" ]; then

cli/scripts/check_pre_reqs.sh

@@ -16,103 +16,17 @@ echoX () {
 }
 
 
-compatLIBC () {
-  compat_libc=libc-${1,,}.so
-
-  if [ -f /lib64/libc-2.*.so ]; then
-    sys_libc=`basename /lib64/libc-2.*.so`
-  else
-    libdir=/lib/`arch`-linux-gnu
-    sys_libc=`basename $libdir/libc-2.*.so`
-  fi
-
-  if [[ "$compat_libc" < "$sys_libc" || "$compat_libc" == "$sys_libc" ]]; then
-    echoX "#    LIBC - OK ($sys_libc)"
-  else
-    echoX "ERROR: Incompatible LIBC library ($sys_libc).  [Linux version is too old]"
-    exit 1
-  fi
-
-  return
-}
-
-
 isEL () {
-  ELx=EL$1
-
-
-  grep "VERSION_ID=\"$1" /etc/redhat-release > /dev/null 2>&1
-  rc=$?
-  if [ "$rc" == "0" ]; then
-    echoX "#       $ELx - OK"
-
-    ## also make sure wget and python3.11 are installed
-    wget --version > /dev/null 2>&1
-    rc=$?
-    if [ ! "$rc" == "0" ]; then
-      sudo yum install -y wget python3.11 python3.11-pip
-    fi
 
-    return
+  if [ -f "/etc/redhat-release" ]; then
+      return
   fi
 
-  echoX "ERROR: must be Enterprise Linux 8+"
+  echoX "ERROR: must be Enterprise Linux"
   exit 1
 }
 
 
-installPERL () {
-  sudo yum install -y perl perl-devel perl-DBI
-  echoX "#    PERL - OK"
-  return
-}
-
-
-## install OPENJDK 8 or 11 (if needed) on EL7 or EL8
-installOPENJDK () {
-  OPENJDKx=OPENJDK$1
-  java -version > /dev/null 2>&1
-  rc=$?
-  if [ ! "$rc" == "0" ]; then
-    sudo yum install java-$1-openjdk-devel 
-    return
-  fi
-
-  java -version 2>&1 | grep "openjdk version \"$1." > /dev/null 2>&1
-  rc=$?
-  if [ ! "$rc" == "0" ]; then
-   sudo yum install openjdk$1
-   return
-  fi
-
-  echoX "# $OPENJDKx - OK"
-  return
-}
-
-
-installGCC () {
-  sudo yum groupinstall -y 'Development Tools'
-  echoX "#     GCC - OK"
-  return
-}
-
-
-## install PYTHON/PIP
-installPYTHON () {
-  PYTHONx=PYTHON$1
-
-  apt --version > /dev/null 2>&1
-  rc=$?
-  if [ $rc == "0" ]; then
-    sudo apt install -y python$1 python$1-pip
-  else
-    sudo yum install -y python$1 python$1-pip
-  fi
-
-  echoX "#   $PYTHONx - OK"
-  return
-}
-
 
 isAMD64 () {
   if [ `uname -m` == 'x86_64' ]; then
@@ -125,39 +39,6 @@ isAMD64 () {
 }
 
 
-isPGML () {
-  ver=22.04
-  cat /etc/os-release | grep Ubuntu > /dev/null
-  rc=$?
-  if [ $rc == "0" ]; then
-    cat /etc/os-release | grep VERSION_ID | grep $ver > /dev/null
-    rc=$?
-    if [ $rc == "0" ]; then
-      if [ `arch` == "x86_64" ]; then
-	rc=0
-        echoX '# UB22AMD - OK'
-      else
-	rc=1
-      fi
-    fi
-  fi
-
-  if [ ! "$rc" == "0" ]; then
-    echoX 'ERROR: PGML only presently supported on Ubuntu 22.04 x86_64' 
-    exit 1
-  fi
-
-  venv_test_dir=$HOME/venv/lib/python3.10/site-packages/xgboost
-  if [ -d "$venv_test_dir" ]; then
-    echoX '#  ~/VENV - OK'
-    return
-  else
-    echoX "ERROR: missing valid pgml virtual env dir - $venv_test_dir"
-    exit 1
-  fi
-}
-
-
 isUBU () {
   ver="$1.04"
   cat /etc/os-release | grep VERSION_ID | grep $ver > /dev/null
@@ -169,8 +50,6 @@ isUBU () {
 
   echoX "ERROR: only supported on Ubuntu $ver"
   exit 1
-
-
 }
 
 
@@ -188,23 +67,8 @@ do
   elif [ "${req:0:3}" == "UBU" ]; then
     ver=${req:3:2}
     isUBU $ver
-  elif [ "$req" == "PGML" ]; then
-    isPGML
   elif [ "$req" == "AMD64" ]; then
     isAMD64
-  elif [ "$req" == "PERL" ]; then
-    installPERL
-  elif [ "$req" == "GCC" ]; then
-    installGCC
-  elif [ "${req:0:6}" == "PYTHON" ]; then
-    ver=${req:6:1}
-    installPYTHON $ver
-  elif [ "${req:0:7}" == "OPENJDK" ]; then
-    ver=${req:7:2}
-    installOPENJDK $ver
-  elif [ "${req:0:5}" == "LIBC-" ]; then
-    ver=${req:5:4}
-    compatLIBC $ver
   else
     echoX "ERROR: invalid pre-req ($req)"
     exit 1

cli/scripts/cluster.py

@@ -282,7 +282,7 @@ def ssh_install(cluster_name, db, db_settings, db_user, db_passwd, n, install, p
         ndport = "5432"
 
     if REPO == "":
-        REPO = "https://pgedge-upstream.s3.amazonaws.com/REPO"
+        REPO = "https://pgedge-download.s3.amazonaws.com/REPO"
         os.environ["REPO"] = REPO
 
     verbose = db_settings.get("log_level", "info")

cli/scripts/get_old.py

@@ -0,0 +1,67 @@
+
+import os
+from pathlib import Path
+os.chdir(Path(__file__).parent)
+
+import sys, sqlite3, platform
+
+PROD_BUCKET="s3://pgedge-download/REPO"
+
+conf_dir="../../data/conf"
+meta_data_db=f"{conf_dir}/db_local.db"
+
+cache_dir=f"{os.getenv('HIST')}/out_old"
+os.system(f"rm -rf {cache_dir}")
+os.system(f"mkdir {cache_dir}")
+
+
+try:
+    con = sqlite3.connect("../../data/conf/db_local.db", check_same_thread=False)
+except Exception as e:
+    print(f"ERROR connecting to meta data: {e}")
+    sys.exit(1)
+
+arch="amd"
+if platform.machine() == "aarch64":
+    arch = "arm"
+
+
+data = []
+sql = '''
+SELECT component, version, platform
+  FROM v_versions 
+ WHERE stage = 'prod' 
+   AND is_current = 0;
+'''
+
+try:
+    c = con.cursor()
+    c.execute(sql)
+    data = c.fetchall()
+except Exception as e:
+    print(f"ERROR retrieving old meta data: {e}")
+    sys.exit(1)
+
+err_knt = 0
+
+for d in data:
+    component = str(d[0])
+    version = str(d[1])
+    platform = str(d[2])
+
+    file = f"{component}-{version}-{arch}.tgz"
+
+    cmd= f"aws s3 cp {PROD_BUCKET}/{file}  {cache_dir}/."
+    print(f"# {cmd}") 
+    rc = os.system(cmd)
+    if rc == 0:
+        os.system(f"aws s3 cp {PROD_BUCKET}/{file}.sha512 {cache_dir}/.")
+    else:
+        err_knt = err_knt + 1
+        print(f"#################### ERROR ({err_knt}) #########################")
+
+    print("")
+
+sys.exit(err_knt)
+
+

cli/scripts/meta.py

@@ -14,55 +14,6 @@
     pass
 
 
-def get_product(product, platf=None, pgv=None):
-    sql = f"""
-SELECT product, component, platform, pg_ver
-  FROM v_products
-"""
-
-    try:
-        c = con.cursor()
-        c.execute(sql)
-        data = c.fetchall()
-    except Exception as e:
-        fatal_error(e, sql, "get_product")
-
-    if len(data) == 0:
-        util.exit_message(f"Invalid product '{product}'")
-
-    for d in data:
-        if product:
-            if not product == str(d[0]):
-                continue
-
-        d_comp = str(d[1])
-        if platf:
-            d_platf = str(d[2])
-            if d_platf == "" or d_platf in platf:
-                pass
-            else:
-                util.exit_message(f"'{d_comp}' not available for {platf}")
-
-        if pgv:
-            d_pgv = str(d[3])
-            if d_pgv == "" or d_pgv == pg_ver:
-                pass
-            else:
-                util.exit_message(f"'{d_comp}' not available for {platf}")
-
-    return data
-
-
-def get_my_version():
-    data = exec_sql("SELECT v FROM hub")
-    return(str(data[0]))
-
-
-def get_my_codename():
-    data = exec_sql("SELECT c FROM hub")
-    return(str(data[0]))
-
-
 def pretty_sql(sql):
     data = []
 

cli/scripts/util.py

@@ -4,6 +4,9 @@
 import os
 import time
 
+MY_VERSION = "24.10.7"
+MY_CODENAME = "Constellation"
+
 DEFAULT_PG = "16"
 DEFAULT_SPOCK = "40"
 DEFAULT_SPOCK_17 = "40"
@@ -19,28 +22,10 @@
 MY_DATA = os.getenv("MY_DATA")
 
 MY_LIBS = f"{MY_HOME}/hub/scripts/lib"
-BACKUP_DIR = f"{MY_DATA}/conf/backup"
-TIME = time.strftime("%Y%m%d%H%M")
-BACKUP_TARGET_DIR = f"{BACKUP_DIR}/{TIME}"
-LOG_FILENAME = f"{MY_LOGS}/cli_log.out"
-
-################ Logging Configuration ############
-COMMAND = 15
-DEBUG = 10
-DEBUG2 = 9
-
-isDebug=0
-LOG_LEVEL = COMMAND
-pgeDebug = int(os.getenv('pgeDebug', '0'))
-if pgeDebug == 1:
-    LOG_LEVEL = DEBUG
-    isDebug = 1
-elif pgeDebug == 2:
-    LOG_LEVEL = DEBUG2
-    isDebug = 2
-
-if not os.path.isdir(MY_LOGS):
-    os.mkdir(MY_LOGS)
+MY_LITE = os.getenv("MY_LITE", None)
+BACKUP_DIR = os.path.join(MY_HOME, "data", "conf", "backup")
+BACKUP_TARGET_DIR = os.path.join(BACKUP_DIR, time.strftime("%Y%m%d%H%M"))
+VALID_PG = ["15", "16", "17"]
 
 import sys
 import socket