Improve docker-related query test failure avoidance

src/pgmon.py

@@ -656,15 +656,24 @@ def test_queries():
         # If the metric has arguments to use while testing, grab those
         args = metric.get("test_args", {})
         print("Testing {} [{}]".format(name, ", ".join(["{}={}".format(key, value) for key, value in args.items()])))
-        # Run the query with the ability to retry, mostly because the PostgreSQL
+        # When testing against a docker container, we may end up connecting
-        # docker image restarts during its initialization phase.  If the health
+        # before the service is truly up (it restarts during the initialization
-        # check passes during the first phase, the agent will start testing too
+        # phase).  To cope with this, we'll allow a few connection failures.
-        # early and will break when PostgreSQL restarts.
+        tries = 5
+        while True:
+            # Run the query without the ability to retry
             try:
-            res = sample_metric(dbname, name, args, retry=True)
+                res = sample_metric(dbname, name, args, retry=False)
+                break
             except MetricVersionError:
-            print("{} -> Unsupported for this version".format(name))
+                res = "Unsupported for this version"
-            continue
+                break
+            except psycopg2.OperationalError as e:
+                print("Error encountered, {} tries left: {}".format(tries, e))
+                if tries <= 0:
+                    raise
+                time.sleep(1)
+            tries -= 1
         # Compare the result to the provided sample results
         # TODO
         print("{} -> {}".format(name, res))

tests/run-tests.sh

@@ -41,14 +41,6 @@ do
     # Specify the version we're testing against
     export PGTAG="${images["$version"]}"
 
-    # Start the db container first and wait a moment for it to initialize
-    # This isn't perfect, but if the health check catches PostgreSQL when it's
-    # first being initialized, the agent can fail to connect.
-    # A better solution would probably be to make the agent retry more.
-    docker compose up -d db
-
-    sleep 2
-
     # Start the containers
     docker compose up --exit-code-from=agent agent
     rc=$?