Ops fixes

.flake8

@@ -3,7 +3,6 @@ doctests = True
 filename =
         *.py
         *server/bin/pbench-index
-        *server/bin/pbench-reindex
 exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,*web-server/*,
         *agent/bench-scripts/test-bin/fio-histo-log-pctiles.py,
         *agent/bench-scripts/tests/pbench-trafficgen/test-39.trafficgen/*,

lib/pbench/cli/server/report.py

@@ -176,10 +176,11 @@ def report_cache(tree: CacheManager):
     lacks_metrics = 0
     bad_metrics = 0
     unpacked_count = 0
-    unpacked_times = 0
+    unpacked_multiple = 0
     stream_unpack_skipped = 0
     last_ref_errors = 0
     agecomp = Comparator("age", really_big=time.time() * 2.0)
+    unpackcomp = Comparator("unpack")
     sizecomp = Comparator("size")
     compcomp = Comparator("compression")
     speedcomp = Comparator("speed", really_big=GINORMOUS_FP)
@@ -254,7 +255,9 @@ def report_cache(tree: CacheManager):
         else:
             found_metrics = True
             unpacked_count += 1
-            unpacked_times += metrics["count"]
+            if metrics["count"] > 1:
+                unpacked_multiple += 1
+            unpackcomp.add(dsname, metrics["count"])
             speedcomp.add(dsname, metrics["min"], metrics["max"])
             if size:
                 stream_fast = size / metrics["min"] / MEGABYTE_FP
@@ -270,9 +273,14 @@ def report_cache(tree: CacheManager):
         f"{humanize.naturalsize(cached_size)}"
     )
     click.echo(
-        f"  {unpacked_count:,d} datasets have been unpacked a total of "
-        f"{unpacked_times:,d} times"
+        f"  {unpacked_count:,d} datasets have unpack metrics and "
+        f"{unpacked_multiple} have been unpacked more than once."
     )
+    if unpackcomp.max > 1 or verifier.verify:
+        click.echo(
+            f"  The most unpacked dataset has been unpacked {unpackcomp.max} times, "
+            f"{unpackcomp.max_name}"
+        )
     click.echo(
         "  The least recently used cache was referenced "
         f"{humanize.naturaldate(oldest)}, {agecomp.min_name}"
@@ -318,12 +326,9 @@ def report_cache(tree: CacheManager):
             f"{last_ref_errors:,d} are missing reference timestamps, "
             f"{bad_size:,d} have bad size metadata"
         )
-    if lacks_metrics or bad_metrics or verifier.verify:
-        click.echo(
-            f"  {lacks_metrics:,d} datasets are missing unpack metric data, "
-            f"{bad_metrics} have bad unpack metric data"
-        )
-    if lacks_tarpath:
+    if bad_metrics or verifier.verify:
+        click.echo(f"  {bad_metrics:,d} have bad unpack metric data")
+    if lacks_tarpath or verifier.verify:
         click.echo(
             f"  {lacks_tarpath} datasets are missing server.tarball-path metadata"
         )

lib/pbench/cli/server/shell.py

@@ -222,6 +222,7 @@ def main() -> int:
 
     try:
         return run_gunicorn(server_config, logger)
-    except Exception:
-        logger.exception("Unhandled exception running gunicorn")
+    except Exception as exc:
+        logger.exception("Unhandled exception running gunicorn: {!r}", str(exc))
+        print(exc, file=sys.stderr)
         return 1

lib/pbench/server/database/database.py

@@ -11,6 +11,10 @@
 from pbench.server import PbenchServerConfig
 
 
+class DatabaseInitError(Exception):
+    pass
+
+
 class Database:
     # Create declarative base model that our model can inherit from
     Base = declarative_base()
@@ -64,33 +68,38 @@ def init_db(server_config: PbenchServerConfig, logger: Logger):
         # metadata will not have any tables and create_all functionality will do
         # nothing.
 
-        engine = create_engine(db_uri)
-        db_url = urlparse(db_uri)
-        if db_url.scheme == "sqlite":
-            Database.Base.metadata.create_all(bind=engine)
-        else:
-            alembic_cfg = Config()
-            alembic_cfg.set_main_option(
-                "script_location", str(Path(__file__).parent / "alembic")
+        try:
+            engine = create_engine(db_uri)
+            db_url = urlparse(db_uri)
+            if db_url.scheme == "sqlite":
+                Database.Base.metadata.create_all(bind=engine)
+            else:
+                alembic_cfg = Config()
+                alembic_cfg.set_main_option(
+                    "script_location", str(Path(__file__).parent / "alembic")
+                )
+                alembic_cfg.set_main_option("prepend_sys_path", ".")
+                with engine.begin() as connection:
+                    alembic_cfg.attributes["connection"] = connection
+                    command.upgrade(alembic_cfg, "head")
+            Database.db_session = scoped_session(
+                sessionmaker(bind=engine, autocommit=False, autoflush=False)
+            )
+            Database.Base.query = Database.db_session.query_property()
+
+            # Although most of the Pbench Server currently works with the default
+            # SQLAlchemy transaction management, some parts rely on true atomic
+            # transactions and need a better isolation level.
+            # NOTE: In PostgreSQL we might use a slightly looser integrity level
+            # like "REPEATABLE READ", however as this isn't supported in sqlite3
+            # we're using the strictest "SERIALIZABLE" level.
+            Database.maker = sessionmaker(
+                bind=engine.execution_options(isolation_level="SERIALIZABLE")
             )
-            alembic_cfg.set_main_option("prepend_sys_path", ".")
-            with engine.begin() as connection:
-                alembic_cfg.attributes["connection"] = connection
-                command.upgrade(alembic_cfg, "head")
-        Database.db_session = scoped_session(
-            sessionmaker(bind=engine, autocommit=False, autoflush=False)
-        )
-        Database.Base.query = Database.db_session.query_property()
-
-        # Although most of the Pbench Server currently works with the default
-        # SQLAlchemy transaction management, some parts rely on true atomic
-        # transactions and need a better isolation level.
-        # NOTE: In PostgreSQL we might use a slightly looser integrity level
-        # like "REPEATABLE READ", however as this isn't supported in sqlite3
-        # we're using the strictest "SERIALIZABLE" level.
-        Database.maker = sessionmaker(
-            bind=engine.execution_options(isolation_level="SERIALIZABLE")
-        )
+        except Exception as e:
+            if logger:
+                logger.error("Unable to initialize database {}: {!r}", db_uri, str(e))
+            raise DatabaseInitError(str(e))
 
     @staticmethod
     def dump_query(query: Query, logger: Logger, level: int = DEBUG):

pyproject.toml

@@ -44,7 +44,6 @@ include = '''
 | ^/agent/util-scripts/validate-ipaddress$
 | ^/agent/util-scripts/pbench-tool-meister-start$
 | ^/server/bin/pbench-index$
-| ^/server/bin/pbench-reindex$
 | ^/utils/fetch-curr-commit-id$
 '''
 extend-exclude = '''

server/bin/pbench-index

@@ -204,4 +204,10 @@ if __name__ == "__main__":
         # exit status to 1.  All finally handlers would have been executed, no
         # need to report a SIGTERM again.
         status = 1
+    except Exception as e:
+        # Don't let an unhandled exception generate a potentially massive
+        # traceback that's unlikely to have much real value (and which will be
+        # echoed to syslogd by conmon, one line at a time).
+        print(f"Indexing failed with exception {str(e)!r}", file=sys.stderr)
+        status = 1
     sys.exit(status)