Bump version to 1.0.2

* Modify Makefile to extract the version from the mainscript

* Bump version to 1.0.2

GENTOO/pgmon-0.1.0.ebuild

@@ -1,52 +0,0 @@
-# Copyright 2024 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-EAPI=8
-
-PYTHON_COMPAT=( python3_{6..12} )
-
-inherit git-r3 python-r1
-
-DESCRIPTION="PostgreSQL monitoring bridge"
-HOMEPAGE="None"
-
-LICENSE="BSD"
-SLOT="0"
-KEYWORDS="amd64"
-
-EGIT_REPO_URI="https://code2.shh-dot-com.org/james/pgmon.git"
-#EGIT_COMMIT=""
-
-DEPEND="
-	${PYTHON_DEPS}
-	dev-python/psycopg:3
-	dev-python/pyyaml
-	acct-user/zabbix
-	acct-group/zabbix
-	agent? ( net-analyzer/zabbix[agent] )
-	agent2? ( net-analyzer/zabbix[agent2] )
-	app-admin/logrotate
-	"
-RDEPEND="${DEPEND}"
-BDEPEND=""
-
-src_install() {
-	default
-
-	# Install init script
-	newinitd "${FILESDIR}/pgmon.openrc" pgmon
-
-	# Install script
-	exeinto /usr/bin
-	newexe "${S}/pgmon.py" pgmon
-
-	# Install default config
-	diropts -o root -g zabbix -m 0755
-	insinto /etc/pgmon
-	doins "${FILESDIR}/pgmon.yml"
-	doins "${S}/pgmon-metrics.yml"
-
-	# Install logrotate config
-	insinto /etc/logrotate.d
-	newins "${FILESDIR}/pgmon.logrotate" pgmon
-}

GENTOO/pgmon-1.0.1.ebuild

@@ -5,7 +5,7 @@ EAPI=8
 
 PYTHON_COMPAT=( python3_{6..13} )
 
-inherit git-r3 python-r1
+inherit python-r1
 
 DESCRIPTION="PostgreSQL monitoring bridge"
 HOMEPAGE="None"
@@ -14,7 +14,9 @@ LICENSE="BSD"
 SLOT="0"
 KEYWORDS="amd64"
 
-SRC_URI="https://code2.shh-dot-com.org/james/${PN}/archive/v${PV}.tar.gz -> ${P}.tar.gz"
+SRC_URI="https://code2.shh-dot-com.org/james/${PN}/archive/v${PV}.tar.bz2 -> ${P}.tar.bz2"
+
+IUSE="-systemd"
 
 DEPEND="
 	${PYTHON_DEPS}
@@ -25,21 +27,36 @@ DEPEND="
 RDEPEND="${DEPEND}"
 BDEPEND=""
 
-S="${WORKDIR}/${PN}"
+RESTRICT="fetch"
+
+#S="${WORKDIR}/${PN}"
+
+pkg_nofetch() {
+	einfo "Please download"
+	einfo "  - ${P}.tar.bz2"
+	einfo "from ${HOMEPAGE} and place it in your DISTDIR directory."
+	einfo "The file should be owned by portage:portage."
+}
+
+src_compile() {
+	true
+}
 
 src_install() {
-	default
-
 	# Install init script
-	newinitd "openrc/pgmon.initd" pgmon
-	newconfd "openrc/pgmon.confd" pgmon
+	if ! use systemd ; then
+		newinitd "openrc/pgmon.initd" pgmon
+		newconfd "openrc/pgmon.confd" pgmon
+	fi
 
 	# Install systemd unit
-	systemd_dounit "systemd/pgmon.service"
+	if use systemd ; then
+		systemd_dounit "systemd/pgmon.service"
+	fi
 
 	# Install script
 	exeinto /usr/bin
-	newexe "pgmon.py" pgmon
+	newexe "src/pgmon.py" pgmon
 
 	# Install default config
 	diropts -o root -g root -m 0755

GENTOO/pgmon-1.0.2.ebuild

@@ -0,0 +1,73 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+PYTHON_COMPAT=( python3_{6..13} )
+
+inherit python-r1
+
+DESCRIPTION="PostgreSQL monitoring bridge"
+HOMEPAGE="None"
+
+LICENSE="BSD"
+SLOT="0"
+KEYWORDS="amd64"
+
+SRC_URI="https://code2.shh-dot-com.org/james/${PN}/archive/v${PV}.tar.bz2 -> ${P}.tar.bz2"
+
+IUSE="-systemd"
+
+DEPEND="
+	${PYTHON_DEPS}
+	dev-python/psycopg:2
+	dev-python/pyyaml
+	app-admin/logrotate
+	"
+RDEPEND="${DEPEND}"
+BDEPEND=""
+
+RESTRICT="fetch"
+
+#S="${WORKDIR}/${PN}"
+
+pkg_nofetch() {
+	einfo "Please download"
+	einfo "  - ${P}.tar.bz2"
+	einfo "from ${HOMEPAGE} and place it in your DISTDIR directory."
+	einfo "The file should be owned by portage:portage."
+}
+
+src_compile() {
+	true
+}
+
+src_install() {
+	# Install init script
+	if ! use systemd ; then
+		newinitd "openrc/pgmon.initd" pgmon
+		newconfd "openrc/pgmon.confd" pgmon
+	fi
+
+	# Install systemd unit
+	if use systemd ; then
+		systemd_dounit "systemd/pgmon.service"
+	fi
+
+	# Install script
+	exeinto /usr/bin
+	newexe "src/pgmon.py" pgmon
+
+	# Install default config
+	diropts -o root -g root -m 0755
+	insinto /etc/pgmon
+	doins "sample-config/pgmon.yml"
+	doins "sample-config/pgmon-metrics.yml"
+
+	# Install logrotate config
+	insinto /etc/logrotate.d
+	newins "logrotate/pgmon.logrotate" pgmon
+
+	# Install man page
+	doman manpages/pgmon.1
+}

Makefile

@@ -1,9 +1,10 @@
 # Package details
 PACKAGE_NAME := pgmon
-VERSION := 1.0
 
 SCRIPT := src/$(PACKAGE_NAME).py
 
+VERSION := $(shell grep -m 1 '^VERSION = ' "$(SCRIPT)" | sed -ne 's/.*"\(.*\)".*/\1/p')
+
 # Where packages are built
 BUILD_DIR := build
 
@@ -15,19 +16,26 @@ SUPPORTED := ubuntu-20.04 \
 	     debian-11 \
 	     rockylinux-8 \
 	     rockylinux-9 \
-	     oraclelinux-7
+	     oraclelinux-7 \
+	     gentoo
 
 ##
 # These targets are the main ones to use for most things.
 ##
 
-.PHONY: all clean tgz test install
+.PHONY: all clean tgz test query-tests install
 
 
 # Build all packages
 .PHONY: package-all
 all: $(foreach distro_release, $(SUPPORTED), package-$(distro_release))
 
+# Gentoo package (tar.gz) creation
+.PHONY: package-gentoo
+package-gentoo:
+	mkdir -p $(BUILD_DIR)/gentoo
+	tar --transform "s,^\.,$(PACKAGE_NAME)-$(VERSION)," -acjf $(BUILD_DIR)/gentoo/$(PACKAGE_NAME)-$(VERSION).tar.bz2 --exclude $(BUILD_DIR) .
+
 
 # Create a deb package
 .PHONY: package-%
@@ -42,7 +50,6 @@ package-%:
 	    --user $(shell id -u):$(shell id -g) \
 	    "$(DISTRO)-packager:$(RELEASE)"
 
-
 # Create a tarball
 tgz:
 	rm -rf $(BUILD_DIR)/tgz/root
@@ -58,6 +65,10 @@ clean:
 test:
 	cd src ; python3 -m unittest
 
+# Run query tests
+query-tests:
+	cd tests ; ./run-tests.sh
+
 # Install the script at the specified base directory
 install:
 	# Set up directories
@@ -86,7 +97,7 @@ install:
 
 
 # Run all of the install tests
-.PHONY: install-tests debian-%-install-test rockylinux-%-install-test ubuntu-%-install-test
+.PHONY: install-tests debian-%-install-test rockylinux-%-install-test ubuntu-%-install-test gentoo-install-test
 install-tests: $(foreach distro_release, $(SUPPORTED), $(distro_release)-install-test)
 
 
@@ -118,6 +129,11 @@ oraclelinux-%-install-test:
 	     oraclelinux:7 \
 	     bash -c 'yum makecache && yum install -y /output/$(PACKAGE_NAME)-$(VERSION)-1.el7.noarch.rpm'
 
+# Run a Gentoo install test
+gentoo-install-test:
+	# May impliment this in the future, but would require additional headaches to set up a repo
+	true
+
 ##
 # Container targets
 #
@@ -158,7 +174,7 @@ actually-package-debian-%:
 # RedHat package creation
 actually-package-rockylinux-%:
 	mkdir -p /output/rockylinux-$*/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
-	cp RPM/$(PACKAGE_NAME).spec /output/rockylinux-$*/SPECS/
+	sed -e "s/@@VERSION@@/$(VERSION)/g" RPM/$(PACKAGE_NAME).spec > /output/rockylinux-$*/SPECS/$(PACKAGE_NAME).spec
 	rpmbuild --define '_topdir /output/rockylinux-$*' \
 	         --define 'version $(VERSION)' \
 	         -bb /output/rockylinux-$*/SPECS/$(PACKAGE_NAME).spec
@@ -173,7 +189,7 @@ actually-package-ubuntu-%:
 # OracleLinux package creation
 actually-package-oraclelinux-%:
 	mkdir -p /output/oraclelinux-$*/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
-	cp RPM/$(PACKAGE_NAME)-el7.spec /output/oraclelinux-$*/SPECS/$(PACKAGE_NAME).spec
+	sed -e "s/@@VERSION@@/$(VERSION)/g" RPM/$(PACKAGE_NAME)-el7.spec > /output/oraclelinux-$*/SPECS/$(PACKAGE_NAME).spec
 	rpmbuild --define '_topdir /output/oraclelinux-$*' \
 	         --define 'version $(VERSION)' \
 	         -bb /output/oraclelinux-$*/SPECS/$(PACKAGE_NAME).spec

RPM/pgmon-el7.spec

@@ -1,5 +1,5 @@
 Name: pgmon
-Version: 1.0
+Version: @@VERSION@@
 Release: 1%{?dist}
 Summary: A bridge to sit between monitoring tools and PostgreSQL
 

RPM/pgmon.spec

@@ -1,5 +1,5 @@
 Name: pgmon
-Version: 1.0
+Version: @@VERSION@@
 Release: 1%{?dist}
 Summary: A bridge to sit between monitoring tools and PostgreSQL
 

openrc/pgmon.initd

@@ -11,7 +11,14 @@ PGMON_USER="${PGMON_USER:-postgres}"
 PGMON_GROUP="${PGMON_GROUP:-$PGMON_USER}"
 CONFIG_FILE="/etc/pgmon/${agent_name}.yml"
 
+output_log=/var/log/pgmon/${SVCNAME}.log
+error_log=/var/log/pgmon/${SVCNAME}.err
+
+start_pre() {
+    checkpath -f -m 0644 -o "${PGMON_USER}:${PGMON_GROUP}" "${output_log}" "${error_log}"
+}
+
 command="/usr/bin/pgmon"
-command_args="'$CONFIG_FILE'"
+command_args="-c '$CONFIG_FILE'"
 command_background="true"
 command_user="${PGMON_USER}:${PGMON_GROUP}"

requirements-dev.yml

@@ -0,0 +1,4 @@
+-r requirements.txt
+testcontainers[postgresql]
+pytest
+black

sample-config/pgmon-metrics.yml

@@ -8,6 +8,11 @@ metrics:
     type: set
     query:
       0: SELECT client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') AS repid, client_addr, state FROM pg_stat_replication
+  discover_slots:
+    type: set
+    query:
+      90400: SELECT slot_name, plugin, slot_type, database, false as temporary, active FROM pg_replication_slots
+      100000: SELECT slot_name, plugin, slot_type, database, temporary, active FROM pg_replication_slots
 
   # cluster-wide metrics
   version:
@@ -15,9 +20,9 @@ metrics:
     query:
       0: SHOW server_version_num
   max_frozen_age:
-    type: value
+    type: row
     query:
-      0: SELECT max(age(datfrozenxid)) FROM pg_database
+      0: SELECT max(age(datfrozenxid)), max(mxid_age(datminmxid)) FROM pg_database
 
   # Per-database metrics
   db_stats:
@@ -25,6 +30,8 @@ metrics:
     query:
       0: SELECT numbackends, xact_commit, xact_rollback, blks_read, blks_hit, tup_returned, tup_fetched, tup_inserted, tup_updated, tup_deleted, conflicts, temp_files, temp_bytes, deadlocks, blk_read_time, blk_write_time, extract('epoch' from stats_reset)::float FROM pg_stat_database WHERE datname = %(dbname)s
       140000: SELECT numbackends, xact_commit, xact_rollback, blks_read, blks_hit, tup_returned, tup_fetched, tup_inserted, tup_updated, tup_deleted, conflicts, temp_files, temp_bytes, deadlocks, COALESCE(checksum_failures, 0) AS checksum_failures, blk_read_time, blk_write_time, session_time, active_time, idle_in_transaction_time, sessions, sessions_abandoned, sessions_fatal, sessions_killed, extract('epoch' from stats_reset)::float FROM pg_stat_database WHERE datname = %(dbname)s
+    test_args:
+      dbname: postgres
 
   # Debugging
   ntables:
@@ -36,10 +43,21 @@ metrics:
   rep_stats:
     type: row
     query:
-      0: SELECT * FROM pg_stat_database WHERE client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = '{repid}'
+      90400: SELECT * FROM pg_stat_replication WHERE client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = '{repid}'
+    test_args:
+      repid: 127.0.0.1_test_rep
 
   # Debugging
   sleep:
     type: value
     query:
       0: SELECT now(), pg_sleep(5);
+
+  # Per-slot metrics
+  slot_stats:
+    type: row
+    query:
+      90400: SELECT active_pid, xmin, pg_xlog_location_diff(pg_current_xlog_location(), restart_lsn) AS restart_bytes, pg_xlog_location_diff(pg_current_xlog_location(), confirmed_flush_lsn) AS confirmed_flush_bytes FROM pg_replication_slots WHERE slot_name = '{slot}'
+      100000: SELECT active_pid, xmin, pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) AS restart_bytes, pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn) AS confirmed_flush_bytes FROM pg_replication_slots WHERE slot_name = '{slot}'
+    test_args:
+      slot: test_slot

sample-config/pgmon.yml

@@ -1,3 +1,9 @@
+# The address the agent binds to
+#address: 127.0.0.1
+
+# The port the agent listens on for requests
+#port: 5400
+
 # Min PostgreSQL connection pool size (per database)
 #min_pool_size: 0
 
@@ -23,6 +29,9 @@
 # Default database to connect to when none is specified for a metric
 #dbname: 'postgres'
 
+# SSL connection mode
+#ssl_mode: require
+
 # Timeout for getting a connection slot from a pool
 #pool_slot_timeout: 5
 

src/pgmon.py

@@ -4,6 +4,7 @@ import yaml
 import json
 import time
 import os
+import sys
 
 import argparse
 import logging
@@ -11,7 +12,7 @@ import logging
 from datetime import datetime, timedelta
 
 import psycopg2
-from psycopg2.extras import DictCursor
+from psycopg2.extras import RealDictCursor
 from psycopg2.pool import ThreadedConnectionPool
 
 from contextlib import contextmanager
@@ -23,7 +24,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
 from http.server import ThreadingHTTPServer
 from urllib.parse import urlparse, parse_qs
 
-VERSION = '0.1.0'
+VERSION = "1.0.2"
 
 # Configuration
 config = {}
@@ -53,64 +54,73 @@ config_file = None
 
 # Configure logging
 log = logging.getLogger(__name__)
-formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s: %(funcName)s() line %(lineno)d: %(message)s')
+formatter = logging.Formatter(
+    "%(asctime)s - %(levelname)s - %(filename)s: %(funcName)s() line %(lineno)d: %(message)s"
+)
 console_log_handler = logging.StreamHandler()
 console_log_handler.setFormatter(formatter)
 log.addHandler(console_log_handler)
 
+
 # Error types
 class ConfigError(Exception):
     pass
+
+
 class DisconnectedError(Exception):
     pass
+
+
 class UnhappyDBError(Exception):
     pass
+
+
+class UnknownMetricError(Exception):
+    pass
+
+
 class MetricVersionError(Exception):
     pass
 
+
 # Default config settings
 default_config = {
+    # The address the agent binds to
+    "address": "127.0.0.1",
+    # The port the agent listens on for requests
+    "port": 5400,
     # Min PostgreSQL connection pool size (per database)
-    'min_pool_size': 0,
-
+    "min_pool_size": 0,
     # Max PostgreSQL connection pool size (per database)
-    'max_pool_size': 4,
-
+    "max_pool_size": 4,
     # How long a connection can sit idle in the pool before it's removed (seconds)
-    'max_idle_time': 30,
-
+    "max_idle_time": 30,
     # Log level for stderr logging
-    'log_level': 'error',
-
+    "log_level": "error",
     # Database user to connect as
-    'dbuser': 'postgres',
-
+    "dbuser": "postgres",
     # Database host
-    'dbhost': '/var/run/postgresql',
-
+    "dbhost": "/var/run/postgresql",
     # Database port
-    'dbport': 5432,
-
+    "dbport": 5432,
     # Default database to connect to when none is specified for a metric
-    'dbname': 'postgres',
-
+    "dbname": "postgres",
+    # SSL connection mode
+    "ssl_mode": "require",
     # Timeout for getting a connection slot from a pool
-    'pool_slot_timeout': 5,
-
+    "pool_slot_timeout": 5,
     # PostgreSQL connection timeout (seconds)
     # Note: It can actually be double this because of retries
-    'connect_timeout': 5,
-
+    "connect_timeout": 5,
     # Time to wait before trying to reconnect again after a reconnect failure (seconds)
-    'reconnect_cooldown': 30,
-
+    "reconnect_cooldown": 30,
     # How often to check the version of PostgreSQL (seconds)
-    'version_check_period': 300,
-
+    "version_check_period": 300,
     # Metrics
-    'metrics': {}
+    "metrics": {},
 }
 
+
 def update_deep(d1, d2):
     """
     Recursively update a dict, adding keys to dictionaries and appending to
@@ -124,24 +134,33 @@ def update_deep(d1, d2):
         The new d1
     """
     if not isinstance(d1, dict) or not isinstance(d2, dict):
-        raise TypeError('Both arguments to update_deep need to be dictionaries')
+        raise TypeError("Both arguments to update_deep need to be dictionaries")
 
     for k, v2 in d2.items():
         if isinstance(v2, dict):
             v1 = d1.get(k, {})
             if not isinstance(v1, dict):
-                raise TypeError('Type mismatch between dictionaries: {} is not a dict'.format(type(v1).__name__))
+                raise TypeError(
+                    "Type mismatch between dictionaries: {} is not a dict".format(
+                        type(v1).__name__
+                    )
+                )
             d1[k] = update_deep(v1, v2)
         elif isinstance(v2, list):
             v1 = d1.get(k, [])
             if not isinstance(v1, list):
-                raise TypeError('Type mismatch between dictionaries: {} is not a list'.format(type(v1).__name__))
+                raise TypeError(
+                    "Type mismatch between dictionaries: {} is not a list".format(
+                        type(v1).__name__
+                    )
+                )
             d1[k] = v1 + v2
         else:
             d1[k] = v2
     return d1
 
-def read_config(path, included = False):
+
+def read_config(path, included=False):
     """
     Read a config file.
 
@@ -151,7 +170,7 @@ def read_config(path, included = False):
     """
     # Read config file
     log.info("Reading log file: {}".format(path))
-    with open(path, 'r') as f:
+    with open(path, "r") as f:
         try:
             cfg = yaml.safe_load(f)
         except yaml.parser.ParserError as e:
@@ -161,42 +180,53 @@ def read_config(path, included = False):
     config_base = os.path.dirname(path)
 
     # Read any external queries and validate metric definitions
-    for name, metric in cfg.get('metrics', {}).items():
+    for name, metric in cfg.get("metrics", {}).items():
         # Validate return types
         try:
-            if metric['type'] not in ['value', 'row', 'column', 'set']:
-                raise ConfigError("Invalid return type: {} for metric {} in {}".format(metric['type'], name, path))
+            if metric["type"] not in ["value", "row", "column", "set"]:
+                raise ConfigError(
+                    "Invalid return type: {} for metric {} in {}".format(
+                        metric["type"], name, path
+                    )
+                )
         except KeyError:
-            raise ConfigError("No type specified for metric {} in {}".format(name, path))
+            raise ConfigError(
+                "No type specified for metric {} in {}".format(name, path)
+            )
 
         # Ensure queries exist
-        query_dict = metric.get('query', {})
+        query_dict = metric.get("query", {})
         if type(query_dict) is not dict:
-            raise ConfigError("Query definition should be a dictionary, got: {} for metric {} in {}".format(query_dict, name, path))
+            raise ConfigError(
+                "Query definition should be a dictionary, got: {} for metric {} in {}".format(
+                    query_dict, name, path
+                )
+            )
 
         if len(query_dict) == 0:
             raise ConfigError("Missing queries for metric {} in {}".format(name, path))
 
         # Read external sql files and validate version keys
-        for vers, query in metric['query'].items():
+        for vers, query in metric["query"].items():
             try:
                 int(vers)
             except:
-                raise ConfigError("Invalid version: {} for metric {} in {}".format(vers, name, path))
+                raise ConfigError(
+                    "Invalid version: {} for metric {} in {}".format(vers, name, path)
+                )
 
-            if query.startswith('file:'):
+            if query.startswith("file:"):
                 query_path = query[5:]
-                if not query_path.startswith('/'):
-                  query_path = os.path.join(config_base, query_path)
-                with open(query_path, 'r') as f:
-                    metric['query'][vers] = f.read()
-
+                if not query_path.startswith("/"):
+                    query_path = os.path.join(config_base, query_path)
+                with open(query_path, "r") as f:
+                    metric["query"][vers] = f.read()
 
     # Read any included config files
-    for inc in cfg.get('include', []):
+    for inc in cfg.get("include", []):
         # Prefix relative paths with the directory from the current config
-        if not inc.startswith('/'):
-          inc = os.path.join(config_base, inc)
+        if not inc.startswith("/"):
+            inc = os.path.join(config_base, inc)
         update_deep(cfg, read_config(inc, included=True))
 
     # Return the config we read if this is an include, otherwise set the final
@@ -209,19 +239,26 @@ def read_config(path, included = False):
         update_deep(new_config, cfg)
 
         # Minor sanity checks
-        if len(new_config['metrics']) == 0:
+        if len(new_config["metrics"]) == 0:
             log.error("No metrics are defined")
             raise ConfigError("No metrics defined")
 
         # Validate the new log level before changing the config
-        if new_config['log_level'].upper() not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
-            raise ConfigError("Invalid log level: {}".format(new_config['log_level']))
+        if new_config["log_level"].upper() not in [
+            "DEBUG",
+            "INFO",
+            "WARNING",
+            "ERROR",
+            "CRITICAL",
+        ]:
+            raise ConfigError("Invalid log level: {}".format(new_config["log_level"]))
 
         global config
         config = new_config
 
         # Apply changes to log level
-        log.setLevel(logging.getLevelName(config['log_level'].upper()))
+        log.setLevel(logging.getLevelName(config["log_level"].upper()))
+
 
 def signal_handler(sig, frame):
     """
@@ -233,7 +270,7 @@ def signal_handler(sig, frame):
     signal.signal(signal.SIGINT, signal.default_int_handler)
 
     # Signal everything to shut down
-    if sig in [ signal.SIGINT, signal.SIGTERM, signal.SIGQUIT ]:
+    if sig in [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]:
         log.info("Shutting down ...")
         global running
         running = False
@@ -245,11 +282,12 @@ def signal_handler(sig, frame):
         log.warning("Received config reload signal")
         read_config(config_file)
 
+
 class ConnectionPool(ThreadedConnectionPool):
     def __init__(self, dbname, minconn, maxconn, *args, **kwargs):
         # Make sure dbname isn't different in the kwargs
-        kwargs['dbname'] = dbname
-        
+        kwargs["dbname"] = dbname
+
         super().__init__(minconn, maxconn, *args, **kwargs)
         self.name = dbname
 
@@ -270,7 +308,10 @@ class ConnectionPool(ThreadedConnectionPool):
             except psycopg2.pool.PoolError:
                 # If we failed to get the connection slot, wait a bit and try again
                 time.sleep(0.1)
-        raise TimeoutError("Timed out waiting for an available connection to {}".format(self.name))
+        raise TimeoutError(
+            "Timed out waiting for an available connection to {}".format(self.name)
+        )
+
 
 def get_pool(dbname):
     """
@@ -288,26 +329,32 @@ def get_pool(dbname):
             # lock
             if dbname not in connections:
                 log.info("Creating connection pool for: {}".format(dbname))
+                # Actually create the connection pool
                 connections[dbname] = ConnectionPool(
                     dbname,
-                    int(config['min_pool_size']),
-                    int(config['max_pool_size']),
-                    application_name='pgmon',
-                    host=config['dbhost'],
-                    port=config['dbport'],
-                    user=config['dbuser'],
-                    connect_timeout=float(config['connect_timeout']),
-                    sslmode='require')
+                    int(config["min_pool_size"]),
+                    int(config["max_pool_size"]),
+                    application_name="pgmon",
+                    host=config["dbhost"],
+                    port=config["dbport"],
+                    user=config["dbuser"],
+                    connect_timeout=int(config["connect_timeout"]),
+                    sslmode=config["ssl_mode"],
+                )
             # Clear the unhappy indicator if present
             unhappy_cooldown.pop(dbname, None)
     return connections[dbname]
 
+
 def handle_connect_failure(pool):
     """
     Mark the database as being unhappy so we can leave it alone for a while
     """
     dbname = pool.name
-    unhappy_cooldown[dbname] = datetime.now() + timedelta(seconds=int(config['reconnect_cooldown']))
+    unhappy_cooldown[dbname] = datetime.now() + timedelta(
+        seconds=int(config["reconnect_cooldown"])
+    )
+
 
 def get_query(metric, version):
     """
@@ -318,42 +365,51 @@ def get_query(metric, version):
         version: The PostgreSQL version number, as given by server_version_num
     """
     # Select the correct query
-    for v in reversed(sorted(metric['query'].keys())):
+    for v in reversed(sorted(metric["query"].keys())):
         if version >= v:
-            if len(metric['query'][v].strip()) == 0:
-                raise MetricVersionError("Metric no longer applies to PostgreSQL {}".format(version))
-            return metric['query'][v]
+            if len(metric["query"][v].strip()) == 0:
+                raise MetricVersionError(
+                    "Metric no longer applies to PostgreSQL {}".format(version)
+                )
+            return metric["query"][v]
 
-    raise MetricVersionError('Missing metric query for PostgreSQL {}'.format(version))
+    raise MetricVersionError("Missing metric query for PostgreSQL {}".format(version))
 
 
 def run_query_no_retry(pool, return_type, query, args):
     """
     Run the query with no explicit retry code
     """
-    with pool.connection(float(config['connect_timeout'])) as conn:
+    with pool.connection(float(config["connect_timeout"])) as conn:
         try:
-            with conn.cursor(cursor_factory=DictCursor) as curs:
+            with conn.cursor(cursor_factory=RealDictCursor) as curs:
                 curs.execute(query, args)
                 res = curs.fetchall()
 
-                if return_type == 'value':
+                if return_type == "value":
+                    if len(res) == 0:
+                        return ""
                     return str(list(res[0].values())[0])
-                elif return_type == 'row':
+                elif return_type == "row":
+                    if len(res) == 0:
+                        return "[]"
                     return json.dumps(res[0])
-                elif return_type == 'column':
+                elif return_type == "column":
+                    if len(res) == 0:
+                        return "[]"
                     return json.dumps([list(r.values())[0] for r in res])
-                elif return_type == 'set':
+                elif return_type == "set":
                     return json.dumps(res)
         except:
             dbname = pool.name
             if dbname in unhappy_cooldown:
                 raise UnhappyDBError()
-            elif conn.broken:
+            elif conn.closed != 0:
                 raise DisconnectedError()
             else:
                 raise
 
+
 def run_query(pool, return_type, query, args):
     """
     Run the query, and if we find upon the first attempt that the connection
@@ -384,6 +440,7 @@ def run_query(pool, return_type, query, args):
             handle_connect_failure(pool)
             raise UnhappyDBError()
 
+
 def get_cluster_version():
     """
     Get the PostgreSQL version if we don't already know it, or if it's been
@@ -395,26 +452,90 @@ def get_cluster_version():
     # If we don't know the version or it's past the recheck time, get the
     # version from the database.  Only one thread needs to do this, so they all
     # try to grab the lock, and then make sure nobody else beat them to it.
-    if cluster_version is None or cluster_version_next_check is None or cluster_version_next_check < datetime.now():
+    if (
+        cluster_version is None
+        or cluster_version_next_check is None
+        or cluster_version_next_check < datetime.now()
+    ):
         with cluster_version_lock:
             # Only check if nobody already got the version before us
-            if cluster_version is None or cluster_version_next_check is None or cluster_version_next_check < datetime.now():
-                log.info('Checking PostgreSQL cluster version')
-                pool = get_pool(config['dbname'])
-                cluster_version = int(run_query(pool, 'value', 'SHOW server_version_num', None))
-                cluster_version_next_check = datetime.now() + timedelta(seconds=int(config['version_check_period']))
+            if (
+                cluster_version is None
+                or cluster_version_next_check is None
+                or cluster_version_next_check < datetime.now()
+            ):
+                log.info("Checking PostgreSQL cluster version")
+                pool = get_pool(config["dbname"])
+                cluster_version = int(
+                    run_query(pool, "value", "SHOW server_version_num", None)
+                )
+                cluster_version_next_check = datetime.now() + timedelta(
+                    seconds=int(config["version_check_period"])
+                )
                 log.info("Got PostgreSQL cluster version: {}".format(cluster_version))
-                log.debug("Next PostgreSQL cluster version check will be after: {}".format(cluster_version_next_check))
+                log.debug(
+                    "Next PostgreSQL cluster version check will be after: {}".format(
+                        cluster_version_next_check
+                    )
+                )
 
     return cluster_version
 
+
+def sample_metric(dbname, metric_name, args, retry=True):
+    """
+    Run the appropriate query for the named metric against the specified database
+    """
+    # Get the metric definition
+    try:
+        metric = config["metrics"][metric_name]
+    except KeyError:
+        raise UnknownMetricError("Unknown metric: {}".format(metric_name))
+
+    # Get the connection pool for the database, or create one if it doesn't
+    # already exist.
+    pool = get_pool(dbname)
+
+    # Identify the PostgreSQL version
+    version = get_cluster_version()
+
+    # Get the query version
+    query = get_query(metric, version)
+
+    # Execute the quert
+    if retry:
+        return run_query(pool, metric["type"], query, args)
+    else:
+        return run_query_no_retry(pool, metric["type"], query, args)
+
+
+def test_queries():
+    """
+    Run all of the metric queries against a database and check the results
+    """
+    # We just use the default db for tests
+    dbname = config["dbname"]
+    # Loop through all defined metrics.
+    for name, metric in config["metrics"].items():
+        # If the metric has arguments to use while testing, grab those
+        args = metric.get("test_args", {})
+        # Run the query without the ability to retry.
+        res = sample_metric(dbname, name, args, retry=False)
+        # Compare the result to the provided sample results
+        # TODO
+        print("{} -> {}".format(name, res))
+    # Return the number of errors
+    # TODO
+    return 0
+
+
 class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
     """
     This is our request handling server.  It is responsible for listening for
     requests, processing them, and responding.
     """
 
-    def log_request(self, code='-', size='-'):
+    def log_request(self, code="-", size="-"):
         """
         Override to suppress standard request logging
         """
@@ -436,10 +557,10 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
         """
         # Parse the URL
         parsed_path = urlparse(self.path)
-        name = parsed_path.path.strip('/')
+        metric_name = parsed_path.path.strip("/")
         parsed_query = parse_qs(parsed_path.query)
 
-        if name == 'agent_version':
+        if metric_name == "agent_version":
             self._reply(200, VERSION)
             return
 
@@ -447,60 +568,31 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
         #       single values, just grab the first from each.
         args = {key: values[0] for key, values in parsed_query.items()}
 
-        # Get the metric definition
-        try:
-            metric = config['metrics'][name]
-        except KeyError:
-            log.error("Unknown metric: {}".format(name))
-            self._reply(404, 'Unknown metric')
-            return
-
         # Get the dbname.  If none was provided, use the default from the
         # config.
-        dbname = args.get('dbname', config['dbname'])
+        dbname = args.get("dbname", config["dbname"])
 
-        # Get the connection pool for the database, or create one if it doesn't
-        # already exist.
+        # Sample the metric
         try:
-            pool = get_pool(dbname)
-        except UnhappyDBError:
+            self._reply(200, sample_metric(dbname, metric_name, args))
+            return
+        except UnknownMetricError as e:
+            log.error("Unknown metric: {}".format(metric_name))
+            self._reply(404, "Unknown metric")
+            return
+        except MetricVersionError as e:
+            log.error(
+                "Failed to find a version of {} for {}".format(metric_name, version)
+            )
+            self._reply(404, "Unsupported version")
+            return
+        except UnhappyDBError as e:
             log.info("Database {} is unhappy, please be patient".format(dbname))
-            self._reply(503, 'Database unavailable')
-            return
-
-        # Identify the PostgreSQL version
-        try:
-            version = get_cluster_version()
-        except UnhappyDBError:
+            self._reply(503, "Database unavailable")
             return
         except Exception as e:
-            if dbname in unhappy_cooldown:
-                log.info("Database {} is unhappy, please be patient".format(dbname))
-                self._reply(503, 'Database unavailable')
-            else:
-                log.error("Failed to get PostgreSQL version: {}".format(e))
-                self._reply(500, 'Error getting DB version')
-            return
-
-        # Get the query version
-        try:
-            query = get_query(metric, version)
-        except KeyError:
-            log.error("Failed to find a version of {} for {}".format(name, version))
-            self._reply(404, 'Unsupported version')
-            return
-
-        # Execute the quert
-        try:
-            self._reply(200, run_query(pool, metric['type'], query, args))
-            return
-        except Exception as e:
-            if dbname in unhappy_cooldown:
-                log.info("Database {} is unhappy, please be patient".format(dbname))
-                self._reply(503, 'Database unavailable')
-            else:
-                log.error("Error running query: {}".format(e))
-                self._reply(500, "Error running query")
+            log.error("Error running query: {}".format(e))
+            self._reply(500, "Unexpected error: {}".format(e))
             return
 
     def _reply(self, code, content):
@@ -508,19 +600,29 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
         Send a reply to the client
         """
         self.send_response(code)
-        self.send_header('Content-type', 'application/json')
+        self.send_header("Content-type", "application/json")
         self.end_headers()
 
-        self.wfile.write(bytes(content, 'utf-8'))
+        self.wfile.write(bytes(content, "utf-8"))
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     # Handle cli args
     parser = argparse.ArgumentParser(
-                prog = 'pgmon',
-                description='A PostgreSQL monitoring agent')
+        prog="pgmon", description="A PostgreSQL monitoring agent"
+    )
 
-    parser.add_argument('config_file', default='pgmon.yml', nargs='?',
-                        help='The config file to read (default: %(default)s)')
+    parser.add_argument(
+        "-c",
+        "--config_file",
+        default="pgmon.yml",
+        nargs="?",
+        help="The config file to read (default: %(default)s)",
+    )
+
+    parser.add_argument(
+        "-t", "--test", action="store_true", help="Run query tests and exit"
+    )
 
     args = parser.parse_args()
 
@@ -530,8 +632,16 @@ if __name__ == '__main__':
     # Read the config file
     read_config(config_file)
 
+    # Run query tests and exit if test mode is enabled
+    if args.test:
+        errors = test_queries()
+        if errors > 0:
+            sys.exit(1)
+        else:
+            sys.exit(0)
+
     # Set up the http server to receive requests
-    server_address = ('127.0.0.1', config['port'])
+    server_address = (config["address"], config["port"])
     httpd = ThreadingHTTPServer(server_address, SimpleHTTPRequestHandler)
 
     # Set up the signal handler
@@ -539,7 +649,7 @@ if __name__ == '__main__':
     signal.signal(signal.SIGHUP, signal_handler)
 
     # Handle requests.
-    log.info("Listening on port {}...".format(config['port']))
+    log.info("Listening on port {}...".format(config["port"]))
     while running:
         httpd.handle_request()
 

src/test_pgmon.py

@@ -10,246 +10,225 @@ import pgmon
 # Silence most logging output
 logging.disable(logging.CRITICAL)
 
+
 class TestPgmonMethods(unittest.TestCase):
-  ##
-  # update_deep
-  ##
-  def test_update_deep__empty_cases(self):
-    # Test empty dict cases
-    d1 = {}
-    d2 = {}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, {})
-    self.assertEqual(d2, {})
+    ##
+    # update_deep
+    ##
+    def test_update_deep__empty_cases(self):
+        # Test empty dict cases
+        d1 = {}
+        d2 = {}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(d1, {})
+        self.assertEqual(d2, {})
 
-    d1 = {'a': 1}
-    d2 = {}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, { 'a': 1 })
-    self.assertEqual(d2, {})
+        d1 = {"a": 1}
+        d2 = {}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(d1, {"a": 1})
+        self.assertEqual(d2, {})
 
-    d1 = {}
-    d2 = {'a': 1}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, { 'a': 1 })
-    self.assertEqual(d2, d1)
+        d1 = {}
+        d2 = {"a": 1}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(d1, {"a": 1})
+        self.assertEqual(d2, d1)
 
-  def test_update_deep__scalars(self):
-    # Test adding/updating scalar values
-    d1 = {'foo': 1, 'bar': "text", 'hello': "world"}
-    d2 = {'foo': 2, 'baz': "blah"}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, {'foo': 2, 'bar': "text", 'baz': "blah", 'hello': "world"})
-    self.assertEqual(d2, {'foo': 2, 'baz': "blah"})
+    def test_update_deep__scalars(self):
+        # Test adding/updating scalar values
+        d1 = {"foo": 1, "bar": "text", "hello": "world"}
+        d2 = {"foo": 2, "baz": "blah"}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(d1, {"foo": 2, "bar": "text", "baz": "blah", "hello": "world"})
+        self.assertEqual(d2, {"foo": 2, "baz": "blah"})
 
-  def test_update_deep__lists(self):
-    # Test adding to lists
-    d1 = {'lst1': []}
-    d2 = {'lst1': [1, 2]}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, {'lst1': [1, 2]})
-    self.assertEqual(d2, d1)
+    def test_update_deep__lists(self):
+        # Test adding to lists
+        d1 = {"lst1": []}
+        d2 = {"lst1": [1, 2]}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(d1, {"lst1": [1, 2]})
+        self.assertEqual(d2, d1)
 
-    d1 = {'lst1': [1, 2]}
-    d2 = {'lst1': []}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, {'lst1': [1, 2]})
-    self.assertEqual(d2, {'lst1': []})
+        d1 = {"lst1": [1, 2]}
+        d2 = {"lst1": []}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(d1, {"lst1": [1, 2]})
+        self.assertEqual(d2, {"lst1": []})
 
-    d1 = {'lst1': [1, 2, 3]}
-    d2 = {'lst1': [3, 4]}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, {'lst1': [1, 2, 3, 3, 4]})
-    self.assertEqual(d2, {'lst1': [3, 4]})
+        d1 = {"lst1": [1, 2, 3]}
+        d2 = {"lst1": [3, 4]}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(d1, {"lst1": [1, 2, 3, 3, 4]})
+        self.assertEqual(d2, {"lst1": [3, 4]})
 
-    # Lists of objects
-    d1 = {'lst1': [{'id': 1}, {'id': 2}, {'id': 3}]}
-    d2 = {'lst1': [{'id': 3}, {'id': 4}]}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, {'lst1': [{'id': 1}, {'id': 2}, {'id': 3}, {'id': 3}, {'id': 4}]})
-    self.assertEqual(d2, {'lst1': [{'id': 3}, {'id': 4}]})
+        # Lists of objects
+        d1 = {"lst1": [{"id": 1}, {"id": 2}, {"id": 3}]}
+        d2 = {"lst1": [{"id": 3}, {"id": 4}]}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(
+            d1, {"lst1": [{"id": 1}, {"id": 2}, {"id": 3}, {"id": 3}, {"id": 4}]}
+        )
+        self.assertEqual(d2, {"lst1": [{"id": 3}, {"id": 4}]})
 
-    # Nested lists
-    d1 = {'obj1': {'l1': [1, 2]}}
-    d2 = {'obj1': {'l1': [3, 4]}}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, {'obj1': {'l1': [1, 2, 3, 4]}})
-    self.assertEqual(d2, {'obj1': {'l1': [3, 4]}})
+        # Nested lists
+        d1 = {"obj1": {"l1": [1, 2]}}
+        d2 = {"obj1": {"l1": [3, 4]}}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(d1, {"obj1": {"l1": [1, 2, 3, 4]}})
+        self.assertEqual(d2, {"obj1": {"l1": [3, 4]}})
 
-  def test_update_deep__dicts(self):
-    # Test adding to lists
-    d1 = {'obj1': {}}
-    d2 = {'obj1': {'a': 1, 'b': 2}}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, {'obj1': {'a': 1, 'b': 2}})
-    self.assertEqual(d2, d1)
+    def test_update_deep__dicts(self):
+        # Test adding to lists
+        d1 = {"obj1": {}}
+        d2 = {"obj1": {"a": 1, "b": 2}}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(d1, {"obj1": {"a": 1, "b": 2}})
+        self.assertEqual(d2, d1)
 
-    d1 = {'obj1': {'a': 1, 'b': 2}}
-    d2 = {'obj1': {}}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, {'obj1': {'a': 1, 'b': 2}})
-    self.assertEqual(d2, {'obj1': {}})
+        d1 = {"obj1": {"a": 1, "b": 2}}
+        d2 = {"obj1": {}}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(d1, {"obj1": {"a": 1, "b": 2}})
+        self.assertEqual(d2, {"obj1": {}})
 
-    d1 = {'obj1': {'a': 1, 'b': 2}}
-    d2 = {'obj1': {'a': 5, 'c': 12}}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, {'obj1': {'a': 5, 'b': 2, 'c': 12}})
-    self.assertEqual(d2, {'obj1': {'a': 5, 'c': 12}})
+        d1 = {"obj1": {"a": 1, "b": 2}}
+        d2 = {"obj1": {"a": 5, "c": 12}}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(d1, {"obj1": {"a": 5, "b": 2, "c": 12}})
+        self.assertEqual(d2, {"obj1": {"a": 5, "c": 12}})
 
-    # Nested dicts
-    d1 = {'obj1': {'d1': {'a': 1, 'b': 2}}}
-    d2 = {'obj1': {'d1': {'a': 5, 'c': 12}}}
-    pgmon.update_deep(d1, d2)
-    self.assertEqual(d1, {'obj1': {'d1': {'a': 5, 'b': 2, 'c': 12}}})
-    self.assertEqual(d2, {'obj1': {'d1': {'a': 5, 'c': 12}}})
+        # Nested dicts
+        d1 = {"obj1": {"d1": {"a": 1, "b": 2}}}
+        d2 = {"obj1": {"d1": {"a": 5, "c": 12}}}
+        pgmon.update_deep(d1, d2)
+        self.assertEqual(d1, {"obj1": {"d1": {"a": 5, "b": 2, "c": 12}}})
+        self.assertEqual(d2, {"obj1": {"d1": {"a": 5, "c": 12}}})
 
-  def test_update_deep__types(self):
-    # Test mismatched types
-    d1 = {'foo': 5}
-    d2 = None
-    self.assertRaises(TypeError, pgmon.update_deep, d1, d2)
+    def test_update_deep__types(self):
+        # Test mismatched types
+        d1 = {"foo": 5}
+        d2 = None
+        self.assertRaises(TypeError, pgmon.update_deep, d1, d2)
 
-    d1 = None
-    d2 = {'foo': 5}
-    self.assertRaises(TypeError, pgmon.update_deep, d1, d2)
+        d1 = None
+        d2 = {"foo": 5}
+        self.assertRaises(TypeError, pgmon.update_deep, d1, d2)
 
-    # Nested mismatched types
-    d1 = {'foo': [1, 2]}
-    d2 = {'foo': {'a': 7}}
-    self.assertRaises(TypeError, pgmon.update_deep, d1, d2)
+        # Nested mismatched types
+        d1 = {"foo": [1, 2]}
+        d2 = {"foo": {"a": 7}}
+        self.assertRaises(TypeError, pgmon.update_deep, d1, d2)
 
+    ##
+    # get_pool
+    ##
 
-  ##
-  # get_pool
-  ##
+    def test_get_pool__simple(self):
+        # Just get a pool in a normal case
+        pgmon.config.update(pgmon.default_config)
+        pool = pgmon.get_pool("postgres")
+        self.assertIsNotNone(pool)
 
-  def test_get_pool__simple(self):
-    # Just get a pool in a normal case
-    pgmon.config.update(pgmon.default_config)
-    pool = pgmon.get_pool('postgres')
-    self.assertIsNotNone(pool)
+    def test_get_pool__unhappy(self):
+        # Test getting an unhappy database pool
+        pgmon.config.update(pgmon.default_config)
+        pgmon.unhappy_cooldown["postgres"] = datetime.now() + timedelta(60)
+        self.assertRaises(pgmon.UnhappyDBError, pgmon.get_pool, "postgres")
 
-  def test_get_pool__unhappy(self):
-    # Test getting an unhappy database pool
-    pgmon.config.update(pgmon.default_config)
-    pgmon.unhappy_cooldown['postgres'] = datetime.now() + timedelta(60)
-    self.assertRaises(pgmon.UnhappyDBError, pgmon.get_pool, 'postgres')
+        # Test getting a different database when there's an unhappy one
+        pool = pgmon.get_pool("template0")
+        self.assertIsNotNone(pool)
 
-    # Test getting a different database when there's an unhappy one
-    pool = pgmon.get_pool('template0')
-    self.assertIsNotNone(pool)
+    ##
+    # handle_connect_failure
+    ##
 
+    def test_handle_connect_failure__simple(self):
+        # Test adding to an empty unhappy list
+        pgmon.config.update(pgmon.default_config)
+        pgmon.unhappy_cooldown = {}
+        pool = pgmon.get_pool("postgres")
+        pgmon.handle_connect_failure(pool)
+        self.assertGreater(pgmon.unhappy_cooldown["postgres"], datetime.now())
 
-  ##
-  # handle_connect_failure
-  ##
+        # Test adding another database
+        pool = pgmon.get_pool("template0")
+        pgmon.handle_connect_failure(pool)
+        self.assertGreater(pgmon.unhappy_cooldown["postgres"], datetime.now())
+        self.assertGreater(pgmon.unhappy_cooldown["template0"], datetime.now())
+        self.assertEqual(len(pgmon.unhappy_cooldown), 2)
 
-  def test_handle_connect_failure__simple(self):
-    # Test adding to an empty unhappy list
-    pgmon.config.update(pgmon.default_config)
-    pgmon.unhappy_cooldown = {}
-    pool = pgmon.get_pool('postgres')
-    pgmon.handle_connect_failure(pool)
-    self.assertGreater(pgmon.unhappy_cooldown['postgres'], datetime.now())
+    ##
+    # get_query
+    ##
 
-    # Test adding another database
-    pool = pgmon.get_pool('template0')
-    pgmon.handle_connect_failure(pool)
-    self.assertGreater(pgmon.unhappy_cooldown['postgres'], datetime.now())
-    self.assertGreater(pgmon.unhappy_cooldown['template0'], datetime.now())
-    self.assertEqual(len(pgmon.unhappy_cooldown), 2)
+    def test_get_query__basic(self):
+        # Test getting a query with one version
+        metric = {"type": "value", "query": {0: "DEFAULT"}}
+        self.assertEqual(pgmon.get_query(metric, 100000), "DEFAULT")
 
+    def test_get_query__versions(self):
+        metric = {"type": "value", "query": {0: "DEFAULT", 110000: "NEW"}}
 
-  ##
-  # get_query
-  ##
+        # Test getting the default version of a query with no lower bound and a newer version
+        self.assertEqual(pgmon.get_query(metric, 100000), "DEFAULT")
 
-  def test_get_query__basic(self):
-    # Test getting a query with one version
-    metric = {
-      'type': 'value',
-      'query': {
-        0: 'DEFAULT'
-      }
-    }
-    self.assertEqual(pgmon.get_query(metric, 100000), 'DEFAULT')
+        # Test getting the newer version of a query with no lower bound and a newer version for the newer version
+        self.assertEqual(pgmon.get_query(metric, 110000), "NEW")
 
-  def test_get_query__versions(self):
-    metric = {
-      'type': 'value',
-      'query': {
-        0: 'DEFAULT',
-        110000: 'NEW'
-      }
-    }
+        # Test getting the newer version of a query with no lower bound and a newer version for an even newer version
+        self.assertEqual(pgmon.get_query(metric, 160000), "NEW")
 
-    # Test getting the default version of a query with no lower bound and a newer version
-    self.assertEqual(pgmon.get_query(metric, 100000), 'DEFAULT')
+        # Test getting a version in bwtween two other versions
+        metric = {"type": "value", "query": {0: "DEFAULT", 96000: "OLD", 110000: "NEW"}}
+        self.assertEqual(pgmon.get_query(metric, 100000), "OLD")
 
-    # Test getting the newer version of a query with no lower bound and a newer version for the newer version
-    self.assertEqual(pgmon.get_query(metric, 110000), 'NEW')
+    def test_get_query__missing_version(self):
+        metric = {"type": "value", "query": {96000: "OLD", 110000: "NEW", 150000: ""}}
 
-    # Test getting the newer version of a query with no lower bound and a newer version for an even newer version
-    self.assertEqual(pgmon.get_query(metric, 160000), 'NEW')
+        # Test getting a metric that only exists for newer versions
+        self.assertRaises(pgmon.MetricVersionError, pgmon.get_query, metric, 80000)
 
-    # Test getting a version in bwtween two other versions
-    metric = {
-      'type': 'value',
-      'query': {
-        0: 'DEFAULT',
-        96000: 'OLD',
-        110000: 'NEW'
-      }
-    }
-    self.assertEqual(pgmon.get_query(metric, 100000), 'OLD')
+        # Test getting a metric that only exists for older versions
+        self.assertRaises(pgmon.MetricVersionError, pgmon.get_query, metric, 160000)
 
-  def test_get_query__missing_version(self):
-    metric = {
-      'type': 'value',
-      'query': {
-        96000: 'OLD',
-        110000: 'NEW',
-        150000: ''
-      }
-    }
+    ##
+    # read_config
+    ##
 
-    # Test getting a metric that only exists for newer versions
-    self.assertRaises(pgmon.MetricVersionError, pgmon.get_query, metric, 80000)
+    def test_read_config__simple(self):
+        pgmon.config = {}
 
-    # Test getting a metric that only exists for older versions
-    self.assertRaises(pgmon.MetricVersionError, pgmon.get_query, metric, 160000)
-
-
-  ##
-  # read_config
-  ##
-
-  def test_read_config__simple(self):
-    pgmon.config = {}
-
-    # Test reading just a metric and using the defaults for everything else
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test reading just a metric and using the defaults for everything else
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 # This is a comment!
 metrics:
   test1:
     type: value
     query:
       0: TEST1
-""")
+"""
+                )
 
-      pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(f"{tmpdirname}/config.yml")
 
-    self.assertEqual(pgmon.config['max_pool_size'], pgmon.default_config['max_pool_size'])
-    self.assertEqual(pgmon.config['dbuser'], pgmon.default_config['dbuser'])
+        self.assertEqual(
+            pgmon.config["max_pool_size"], pgmon.default_config["max_pool_size"]
+        )
+        self.assertEqual(pgmon.config["dbuser"], pgmon.default_config["dbuser"])
 
-    pgmon.config = {}
+        pgmon.config = {}
 
-    # Test reading a basic config
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test reading a basic config
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 # This is a comment!
 min_pool_size: 1
 max_pool_size: 2
@@ -280,22 +259,24 @@ metrics:
     type: column
     query:
       0: TEST4
-""")
+"""
+                )
 
-      pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(f"{tmpdirname}/config.yml")
 
-    self.assertEqual(pgmon.config['dbuser'], 'someone')
-    self.assertEqual(pgmon.config['metrics']['test1']['type'], 'value')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
-    self.assertEqual(pgmon.config['metrics']['test2']['query'][0], 'TEST2')
+        self.assertEqual(pgmon.config["dbuser"], "someone")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["type"], "value")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+        self.assertEqual(pgmon.config["metrics"]["test2"]["query"][0], "TEST2")
 
-  def test_read_config__include(self):
-    pgmon.config = {}
+    def test_read_config__include(self):
+        pgmon.config = {}
 
-    # Test reading a config that includes other files (absolute and relative paths, multiple levels)
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write(f"""---
+        # Test reading a config that includes other files (absolute and relative paths, multiple levels)
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    f"""---
 # This is a comment!
 min_pool_size: 1
 max_pool_size: 2
@@ -308,18 +289,22 @@ version_check_period: 3600
 include:
   - dbsettings.yml
   - {tmpdirname}/metrics.yml
-""")
-    
-      with open(f"{tmpdirname}/dbsettings.yml", 'w') as f:
-        f.write(f"""---
+"""
+                )
+
+            with open(f"{tmpdirname}/dbsettings.yml", "w") as f:
+                f.write(
+                    f"""---
 dbuser: someone
 dbhost: localhost
 dbport: 5555
 dbname: template0
-""")
+"""
+                )
 
-      with open(f"{tmpdirname}/metrics.yml", 'w') as f:
-        f.write(f"""---
+            with open(f"{tmpdirname}/metrics.yml", "w") as f:
+                f.write(
+                    f"""---
 metrics:
   test1:
     type: value
@@ -331,31 +316,35 @@ metrics:
       0: TEST2
 include:
   - more_metrics.yml
-""")
+"""
+                )
 
-      with open(f"{tmpdirname}/more_metrics.yml", 'w') as f:
-        f.write(f"""---
+            with open(f"{tmpdirname}/more_metrics.yml", "w") as f:
+                f.write(
+                    f"""---
 metrics:
   test3:
     type: value
     query:
       0: TEST3
-""")
-      pgmon.read_config(f"{tmpdirname}/config.yml")
-    
-    self.assertEqual(pgmon.config['max_idle_time'], 10)
-    self.assertEqual(pgmon.config['dbuser'], 'someone')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
-    self.assertEqual(pgmon.config['metrics']['test2']['query'][0], 'TEST2')
-    self.assertEqual(pgmon.config['metrics']['test3']['query'][0], 'TEST3')
+"""
+                )
+            pgmon.read_config(f"{tmpdirname}/config.yml")
 
-  def test_read_config__reload(self):
-    pgmon.config = {}
+        self.assertEqual(pgmon.config["max_idle_time"], 10)
+        self.assertEqual(pgmon.config["dbuser"], "someone")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+        self.assertEqual(pgmon.config["metrics"]["test2"]["query"][0], "TEST2")
+        self.assertEqual(pgmon.config["metrics"]["test3"]["query"][0], "TEST3")
 
-    # Test rereading a config to update an existing config
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+    def test_read_config__reload(self):
+        pgmon.config = {}
+
+        # Test rereading a config to update an existing config
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 # This is a comment!
 min_pool_size: 1
 max_pool_size: 2
@@ -378,15 +367,17 @@ metrics:
     type: value
     query:
       0: TEST2
-""")
+"""
+                )
 
-      pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(f"{tmpdirname}/config.yml")
 
-      # Just make sure the first config was read
-      self.assertEqual(len(pgmon.config['metrics']), 2)
+            # Just make sure the first config was read
+            self.assertEqual(len(pgmon.config["metrics"]), 2)
 
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 # This is a comment!
 min_pool_size: 7
 metrics:
@@ -394,73 +385,87 @@ metrics:
     type: value
     query:
       0: NEW1
-""")
+"""
+                )
 
-      pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(f"{tmpdirname}/config.yml")
 
-    self.assertEqual(pgmon.config['min_pool_size'], 7)
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'NEW1')
-    self.assertEqual(len(pgmon.config['metrics']), 1)
+        self.assertEqual(pgmon.config["min_pool_size"], 7)
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "NEW1")
+        self.assertEqual(len(pgmon.config["metrics"]), 1)
 
-  def test_read_config__query_file(self):
-    pgmon.config = {}
+    def test_read_config__query_file(self):
+        pgmon.config = {}
 
-    # Read a config file that reads a query from a file
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Read a config file that reads a query from a file
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 metrics:
   test1:
     type: value
     query:
       0: file:some_query.sql
-""")
+"""
+                )
 
-      with open(f"{tmpdirname}/some_query.sql", 'w') as f:
-        f.write("This is a query")
+            with open(f"{tmpdirname}/some_query.sql", "w") as f:
+                f.write("This is a query")
 
-      pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(f"{tmpdirname}/config.yml")
 
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'This is a query')
+        self.assertEqual(
+            pgmon.config["metrics"]["test1"]["query"][0], "This is a query"
+        )
 
-  def test_read_config__invalid(self):
-    pgmon.config = {}
+    def test_read_config__invalid(self):
+        pgmon.config = {}
 
-    # For all of these tests, we start with a valid config and also ensure that
-    #  it is not modified when a new config read fails
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # For all of these tests, we start with a valid config and also ensure that
+        #  it is not modified when a new config read fails
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 metrics:
   test1:
     type: value
     query:
       0: TEST1
-""")
+"""
+                )
 
-      pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(f"{tmpdirname}/config.yml")
 
-    # Just make sure the config was read
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
+        # Just make sure the config was read
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
 
-    # Test reading a nonexistant config file
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      self.assertRaises(FileNotFoundError, pgmon.read_config, f'{tmpdirname}/missing.yml')
+        # Test reading a nonexistant config file
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            self.assertRaises(
+                FileNotFoundError, pgmon.read_config, f"{tmpdirname}/missing.yml"
+            )
 
-    # Test reading an invalid config file
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""[default]
+        # Test reading an invalid config file
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """[default]
 This looks a lot like an ini file to me
 
 Or maybe a TOML?
-""")
-      self.assertRaises(pgmon.ConfigError, pgmon.read_config, f'{tmpdirname}/config.yml')
+"""
+                )
+            self.assertRaises(
+                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+            )
 
-    # Test reading a config that includes an invalid file
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test reading a config that includes an invalid file
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 dbuser: evil
 metrics:
   test1:
@@ -469,15 +474,19 @@ metrics:
       0: EVIL1
 include:
   - missing_file.yml
-""")
-      self.assertRaises(FileNotFoundError, pgmon.read_config, f'{tmpdirname}/config.yml')
-    self.assertEqual(pgmon.config['dbuser'], 'postgres')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
+"""
+                )
+            self.assertRaises(
+                FileNotFoundError, pgmon.read_config, f"{tmpdirname}/config.yml"
+            )
+        self.assertEqual(pgmon.config["dbuser"], "postgres")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
 
-    # Test invalid log level
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test invalid log level
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 log_level: noisy
 dbuser: evil
 metrics:
@@ -485,132 +494,170 @@ metrics:
     type: value
     query:
       0: EVIL1
-""")
-      self.assertRaises(pgmon.ConfigError, pgmon.read_config, f'{tmpdirname}/config.yml')
-    self.assertEqual(pgmon.config['dbuser'], 'postgres')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
+"""
+                )
+            self.assertRaises(
+                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+            )
+        self.assertEqual(pgmon.config["dbuser"], "postgres")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
 
-    # Test invalid query return type
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test invalid query return type
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 dbuser: evil
 metrics:
   test1:
     type: lots_of_data
     query:
       0: EVIL1
-""")
-      self.assertRaises(pgmon.ConfigError, pgmon.read_config, f'{tmpdirname}/config.yml')
-    self.assertEqual(pgmon.config['dbuser'], 'postgres')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
+"""
+                )
+            self.assertRaises(
+                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+            )
+        self.assertEqual(pgmon.config["dbuser"], "postgres")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
 
-    # Test invalid query dict type
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test invalid query dict type
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 dbuser: evil
 metrics:
   test1:
     type: lots_of_data
     query: EVIL1
-""")
-      self.assertRaises(pgmon.ConfigError, pgmon.read_config, f'{tmpdirname}/config.yml')
-    self.assertEqual(pgmon.config['dbuser'], 'postgres')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
+"""
+                )
+            self.assertRaises(
+                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+            )
+        self.assertEqual(pgmon.config["dbuser"], "postgres")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
 
-    # Test incomplete metric: missing type
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test incomplete metric: missing type
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 dbuser: evil
 metrics:
   test1:
     query:
       0: EVIL1
-""")
-      self.assertRaises(pgmon.ConfigError, pgmon.read_config, f'{tmpdirname}/config.yml')
-    self.assertEqual(pgmon.config['dbuser'], 'postgres')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
+"""
+                )
+            self.assertRaises(
+                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+            )
+        self.assertEqual(pgmon.config["dbuser"], "postgres")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
 
-    # Test incomplete metric: missing queries
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test incomplete metric: missing queries
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 dbuser: evil
 metrics:
   test1:
     type: value
-""")
-      self.assertRaises(pgmon.ConfigError, pgmon.read_config, f'{tmpdirname}/config.yml')
-    self.assertEqual(pgmon.config['dbuser'], 'postgres')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
+"""
+                )
+            self.assertRaises(
+                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+            )
+        self.assertEqual(pgmon.config["dbuser"], "postgres")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
 
-    # Test incomplete metric: empty queries
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test incomplete metric: empty queries
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 dbuser: evil
 metrics:
   test1:
     type: value
     query: {}
-""")
-      self.assertRaises(pgmon.ConfigError, pgmon.read_config, f'{tmpdirname}/config.yml')
-    self.assertEqual(pgmon.config['dbuser'], 'postgres')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
+"""
+                )
+            self.assertRaises(
+                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+            )
+        self.assertEqual(pgmon.config["dbuser"], "postgres")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
 
-    # Test incomplete metric: query dict is None
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test incomplete metric: query dict is None
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 dbuser: evil
 metrics:
   test1:
     type: value
     query:
-""")
-      self.assertRaises(pgmon.ConfigError, pgmon.read_config, f'{tmpdirname}/config.yml')
-    self.assertEqual(pgmon.config['dbuser'], 'postgres')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
+"""
+                )
+            self.assertRaises(
+                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+            )
+        self.assertEqual(pgmon.config["dbuser"], "postgres")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
 
-    # Test reading a config with no metrics
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test reading a config with no metrics
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 dbuser: evil
-""")
-      self.assertRaises(pgmon.ConfigError, pgmon.read_config, f'{tmpdirname}/config.yml')
-    self.assertEqual(pgmon.config['dbuser'], 'postgres')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
+"""
+                )
+            self.assertRaises(
+                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+            )
+        self.assertEqual(pgmon.config["dbuser"], "postgres")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
 
-    # Test reading a query defined in a file but the file is missing
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test reading a query defined in a file but the file is missing
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 dbuser: evil
 metrics:
   test1:
     type: value
     query: 
       0: file:missing.sql
-""")
-      self.assertRaises(FileNotFoundError, pgmon.read_config, f'{tmpdirname}/config.yml')
-    self.assertEqual(pgmon.config['dbuser'], 'postgres')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
+"""
+                )
+            self.assertRaises(
+                FileNotFoundError, pgmon.read_config, f"{tmpdirname}/config.yml"
+            )
+        self.assertEqual(pgmon.config["dbuser"], "postgres")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
 
-    # Test invalid query versions
-    with tempfile.TemporaryDirectory() as tmpdirname:
-      with open(f"{tmpdirname}/config.yml", 'w') as f:
-        f.write("""---
+        # Test invalid query versions
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            with open(f"{tmpdirname}/config.yml", "w") as f:
+                f.write(
+                    """---
 dbuser: evil
 metrics:
   test1:
     type: value
     query: 
       default: EVIL1
-""")
-      self.assertRaises(pgmon.ConfigError, pgmon.read_config, f'{tmpdirname}/config.yml')
-    self.assertEqual(pgmon.config['dbuser'], 'postgres')
-    self.assertEqual(pgmon.config['metrics']['test1']['query'][0], 'TEST1')
-
+"""
+                )
+            self.assertRaises(
+                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+            )
+        self.assertEqual(pgmon.config["dbuser"], "postgres")
+        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")

systemd/pgmon@.service

@@ -7,7 +7,7 @@ After=network.target
 [Service]
 EnvironmentFile=/etc/pgmon/%i-service.conf
 User=${SERVICE_USER:-postgres}
-ExecStart=/usr/local/bin/pgmon /etc/pgmon/%i.yml
+ExecStart=/usr/local/bin/pgmon -c /etc/pgmon/%i.yml
 ExecReload=kill -HUP $MAINPID
 Restart=on-failure
 Type=exec

tests/Dockerfile

@@ -0,0 +1,22 @@
+FROM alpine:3.21
+
+RUN apk update && \
+    apk add py3-psycopg2 \
+            py3-yaml \
+            tini
+
+WORKDIR /app
+
+COPY src/pgmon.py /app/
+
+COPY sample-config/pgmon-metrics.yml /app/
+
+COPY tests/test-config.yml /app/
+
+COPY --chmod=0600 --chown=postgres:postgres tests/pgpass /root/.pgpass
+
+ENTRYPOINT ["tini", "--"]
+
+EXPOSE 5400
+
+CMD ["/app/pgmon.py", "-c", "/app/test-config.yml", "--test"]

tests/docker-compose.yml

@@ -0,0 +1,32 @@
+---
+
+services:
+  agent:
+    image: pgmon
+    build:
+      context: ..
+      dockerfile: tests/Dockerfile
+    ports:
+      - :5400
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    image: "postgres:${PGTAG:-17-bookworm}"
+    ports:
+      - :5432
+    environment:
+      POSTGRES_PASSWORD: secret
+    healthcheck:
+      #test: [ "CMD", "pg_isready", "-U", "postgres" ]
+      test: [ "CMD-SHELL", "pg_controldata /var/lib/postgresql/data/ | grep -q 'in production'" ]
+      interval: 5s
+      timeout: 2s
+      retries: 20
+    command: >
+      postgres -c ssl=on
+               -c ssl_cert_file='/etc/ssl/certs/ssl-cert-snakeoil.pem'
+               -c ssl_key_file='/etc/ssl/private/ssl-cert-snakeoil.key'
+               -c listen_addresses='*'
+

tests/pgpass

@@ -0,0 +1 @@
+db:5432:*:postgres:secret

tests/run-tests.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Versions to test
+versions=( $@ )
+
+# If we weren't given any versions, test them all
+if [ ${#versions[@]} -eq 0 ]
+then
+    versions=( 9.2 9.6 10 11 12 13 14 15 16 17 )
+fi
+
+# Image tags to use
+declare -A images=()
+images["9.2"]='9.2'
+images["9.6"]='9.6-bullseye'
+images["10"]='10-bullseye'
+images["11"]='11-bookworm'
+images["12"]='12-bookworm'
+images["13"]='13-bookworm'
+images["14"]='14-bookworm'
+images["15"]='15-bookworm'
+images["16"]='16-bookworm'
+images["17"]='17-bookworm'
+
+declare -A results=()
+
+# Make sure everything's down to start with
+docker compose down
+
+# Make sure our agent container is up to date
+docker compose build agent
+
+for version in "${versions[@]}"
+do
+    echo
+    echo "Testing: PostgreSQL ${version}"
+
+    # Specify the version we're testing against
+    export PGTAG="${images["$version"]}"
+
+    # Start the containers
+    docker compose up --exit-code-from=agent agent
+    rc=$?
+
+    results["$version"]=$rc
+
+    # Destroy the containers
+    docker compose down
+done
+
+echo
+echo
+for v in "${versions[@]}"
+do
+    case "${results["$v"]}" in
+        0) msg="OK" ;;
+        1) msg="Query failure detected" ;;
+        18) msg="Docker image error: 18" ;;
+        *) msg="Unexpected error: ${results["$v"]}" ;;
+    esac
+    echo "$v -> $msg"
+done

tests/test-config.yml

@@ -0,0 +1,17 @@
+---
+
+# Bind to all interfaces so we can submit requests from outside the test container
+address: 0.0.0.0
+
+# We always just connect to the db container
+dbhost: db
+dbport: 5432
+dbuser: postgres
+
+# The SSL cipher parameters are too old in the 9.2 container, so we allow the tests
+# to be run without encryption
+ssl_mode: prefer
+
+# Pull in the standard metrics
+include:
+  - pgmon-metrics.yml

zabbix_templates/pgmon_templates.yaml

@@ -13,58 +13,58 @@ zabbix_export:
       items:
         - uuid: 763920af8da84db8a9a2667d9653cb21
           name: 'PostgreSQL Agent Version'
-          key: 'web.page.get[localhost,/agent_version,{$AGENT_PORT}]'
+          type: HTTP_AGENT
+          key: 'pgmon[agent_version]'
           delay: 1h
           history: 90d
           value_type: TEXT
+          trends: '0'
           description: 'PostgreSQL monitoring agent version number'
-          preprocessing:
-            - type: REGEX
-              parameters:
-                - '\n\s?\n([\s\S]*)'
-                - \1
+          url: 'http://localhost:{$AGENT_PORT}/agent_version'
           tags:
             - tag: Application
               value: PostgreSQL
         - uuid: 8706eccb7edc4fa394f552fc31f401a9
           name: 'Max Frozen XID Age'
-          key: 'web.page.get[localhost,/max_frozen_age,{$AGENT_PORT}]'
+          type: HTTP_AGENT
+          key: 'pgmon[max_frozen_age]'
           history: 90d
-          trends: '0'
           description: 'Maximum age of any frozen XID in any database'
           preprocessing:
-            - type: REGEX
-              parameters:
-                - '\n\s?\n([\s\S]*)'
-                - \1
             - type: MATCHES_REGEX
               parameters:
                 - '^[0-9]+$'
+          url: 'http://localhost:{$AGENT_PORT}/max_frozen_age'
           tags:
             - tag: Application
               value: PostgreSQL
         - uuid: ee88f5f4d2384f97946d049af5af4502
           name: 'PostgreSQL version'
-          key: 'web.page.get[localhost,/version,{$AGENT_PORT}]'
+          type: HTTP_AGENT
+          key: 'pgmon[version]'
           delay: 1h
           history: 90d
           description: 'PostgreSQL Server version number'
           preprocessing:
-            - type: REGEX
-              parameters:
-                - '\n\s?\n([\s\S]*)'
-                - \1
             - type: MATCHES_REGEX
               parameters:
                 - '^[0-9]+$'
+          url: 'http://localhost:{$AGENT_PORT}/version'
           tags:
             - tag: Application
               value: PostgreSQL
       discovery_rules:
         - uuid: 085de335305e435dbb4439bd52e0d35d
           name: 'Discover Databases'
-          key: 'web.page.get[localhost,/discover_dbs,{$AGENT_PORT}]'
+          type: HTTP_AGENT
+          key: pgmon_discover_dbs
           delay: 10m
+          filter:
+            conditions:
+              - macro: '{#DBNAME}'
+                value: ^template0$
+                operator: NOT_MATCHES_REGEX
+                formulaid: A
           lifetime: 30d
           enabled_lifetime_type: DISABLE_NEVER
           item_prototypes:
@@ -72,6 +72,7 @@ zabbix_export:
               name: 'Time spent executing statements on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[active_time,{#DBNAME}]'
+              delay: '0'
               history: 90d
               value_type: FLOAT
               units: s
@@ -84,7 +85,7 @@ zabbix_export:
                   parameters:
                     - '0.001'
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -94,6 +95,7 @@ zabbix_export:
               name: 'Number of backends on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[backends,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of backends currently connected to this database, or NULL for shared objects. This is the only column in this view that returns a value reflecting current state; all other columns return the accumulated values since the last reset.'
               preprocessing:
@@ -101,7 +103,7 @@ zabbix_export:
                   parameters:
                     - $.numbackends
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -111,6 +113,7 @@ zabbix_export:
               name: 'Blocks hit on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[blks_hit,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of times disk blocks were found already in the buffer cache, so that a read was not necessary (this only includes hits in the PostgreSQL buffer cache, not the operating system''s file system cache)'
               preprocessing:
@@ -118,7 +121,7 @@ zabbix_export:
                   parameters:
                     - $.blks_hit
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -128,6 +131,7 @@ zabbix_export:
               name: 'Blocks read on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[blks_read,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of disk blocks read in this database'
               preprocessing:
@@ -135,7 +139,7 @@ zabbix_export:
                   parameters:
                     - $.blks_read
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -145,6 +149,7 @@ zabbix_export:
               name: 'Time spent reading blocks on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[blk_read_time,{#DBNAME}]'
+              delay: '0'
               history: 90d
               value_type: FLOAT
               units: s
@@ -157,7 +162,7 @@ zabbix_export:
                   parameters:
                     - '0.001'
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -167,6 +172,7 @@ zabbix_export:
               name: 'Time spent writing blocks on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[blk_write_time,{#DBNAME}]'
+              delay: '0'
               history: 90d
               value_type: FLOAT
               units: s
@@ -179,7 +185,7 @@ zabbix_export:
                   parameters:
                     - '0.001'
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -189,6 +195,7 @@ zabbix_export:
               name: 'Total number of checksum failures on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[checksum_failures,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of data page checksum failures detected in this database (or on a shared object), or NULL if data checksums are not enabled.'
               preprocessing:
@@ -201,7 +208,7 @@ zabbix_export:
                   error_handler: CUSTOM_VALUE
                   error_handler_params: '0'
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -211,6 +218,7 @@ zabbix_export:
               name: 'Total number of conflicts on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[conflicts,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of queries canceled due to conflicts with recovery in this database. (Conflicts occur only on standby servers; see pg_stat_database_conflicts for details.)'
               preprocessing:
@@ -218,7 +226,7 @@ zabbix_export:
                   parameters:
                     - $.conflicts
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -228,6 +236,7 @@ zabbix_export:
               name: 'Total number of deadlocks on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[deadlocks,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of deadlocks detected in this database'
               preprocessing:
@@ -235,7 +244,7 @@ zabbix_export:
                   parameters:
                     - $.deadlocks
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -245,6 +254,7 @@ zabbix_export:
               name: 'Time spent in idle transactions on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[idle_in_transaction_time,{#DBNAME}]'
+              delay: '0'
               history: 90d
               value_type: FLOAT
               units: s
@@ -257,7 +267,7 @@ zabbix_export:
                   parameters:
                     - '0.001'
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -267,6 +277,7 @@ zabbix_export:
               name: 'Total number of sessions on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[sessions,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Total number of sessions established to this database'
               preprocessing:
@@ -274,7 +285,7 @@ zabbix_export:
                   parameters:
                     - $.sessions
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -284,6 +295,7 @@ zabbix_export:
               name: 'Total number of abandoned sessions on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[sessions_abandoned,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of database sessions to this database that were terminated because connection to the client was lost'
               preprocessing:
@@ -291,7 +303,7 @@ zabbix_export:
                   parameters:
                     - $.sessions_abandoned
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -301,6 +313,7 @@ zabbix_export:
               name: 'Total number of fatal sessions on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[sessions_fatal,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of database sessions to this database that were terminated by fatal errors'
               preprocessing:
@@ -308,7 +321,7 @@ zabbix_export:
                   parameters:
                     - $.sessions_fatal
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -318,6 +331,7 @@ zabbix_export:
               name: 'Total number of terminated sessions on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[sessions_killed,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of database sessions to this database that were terminated by operator intervention'
               preprocessing:
@@ -325,7 +339,7 @@ zabbix_export:
                   parameters:
                     - $.sessions_killed
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -335,6 +349,7 @@ zabbix_export:
               name: 'Total temp file size on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[temp_bytes,{#DBNAME}]'
+              delay: '0'
               history: 90d
               units: b
               description: 'Total amount of data written to temporary files by queries in this database. All temporary files are counted, regardless of why the temporary file was created, and regardless of the log_temp_files setting.'
@@ -343,7 +358,7 @@ zabbix_export:
                   parameters:
                     - $.temp_bytes
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -353,6 +368,7 @@ zabbix_export:
               name: 'Total number of temp files on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[temp_files,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of temporary files created by queries in this database. All temporary files are counted, regardless of why the temporary file was created (e.g., sorting or hashing), and regardless of the log_temp_files setting.'
               preprocessing:
@@ -360,7 +376,7 @@ zabbix_export:
                   parameters:
                     - $.temp_files
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -370,6 +386,7 @@ zabbix_export:
               name: 'Tuples deleted on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[tup_deleted,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of rows deleted by queries in this database'
               preprocessing:
@@ -377,7 +394,7 @@ zabbix_export:
                   parameters:
                     - $.tup_deleted
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -387,6 +404,7 @@ zabbix_export:
               name: 'Tuples fetched by index scans on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[tup_fetched,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of live rows fetched by index scans in this database'
               preprocessing:
@@ -394,7 +412,7 @@ zabbix_export:
                   parameters:
                     - $.tup_fetched
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -404,6 +422,7 @@ zabbix_export:
               name: 'Tuples inserted on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[tup_inserted,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of rows inserted by queries in this database'
               preprocessing:
@@ -411,7 +430,7 @@ zabbix_export:
                   parameters:
                     - $.tup_inserted
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -421,6 +440,7 @@ zabbix_export:
               name: 'Tuples returned by sequential scans on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[tup_returned,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of live rows fetched by sequential scans and index entries returned by index scans in this database'
               preprocessing:
@@ -428,7 +448,7 @@ zabbix_export:
                   parameters:
                     - $.tup_returned
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -438,6 +458,7 @@ zabbix_export:
               name: 'Tuples updated on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[tup_updated,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of rows updated by queries in this database'
               preprocessing:
@@ -445,7 +466,7 @@ zabbix_export:
                   parameters:
                     - $.tup_updated
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -455,6 +476,7 @@ zabbix_export:
               name: 'Total number of commits on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[xact_commit,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of transactions in this database that have been committed'
               preprocessing:
@@ -462,7 +484,7 @@ zabbix_export:
                   parameters:
                     - $.xact_commit
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -472,6 +494,7 @@ zabbix_export:
               name: 'Total number of rollbacks on {#DBNAME}'
               type: DEPENDENT
               key: 'pgmon_db[xact_rollback,{#DBNAME}]'
+              delay: '0'
               history: 90d
               description: 'Number of transactions in this database that have been rolled back'
               preprocessing:
@@ -479,7 +502,7 @@ zabbix_export:
                   parameters:
                     - $.xact_rollback
               master_item:
-                key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+                key: 'pgmon_db_stats[{#DBNAME}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -497,6 +520,8 @@ zabbix_export:
                   value: PostgreSQL
                 - tag: Database
                   value: '{#DBNAME}'
+                - tag: Type
+                  value: Calculated
             - uuid: a81ba5a7c96d40bca9cc0861da574b49
               name: 'Tuples fetched by index scans on {#DBNAME} - 1m delta'
               type: CALCULATED
@@ -508,6 +533,8 @@ zabbix_export:
                   value: PostgreSQL
                 - tag: Database
                   value: '{#DBNAME}'
+                - tag: Type
+                  value: Calculated
             - uuid: 3bfa9b7db9394b6d9e0cac6255524f50
               name: 'Tuples fetched by index scans on {#DBNAME} - 5m delta'
               type: CALCULATED
@@ -519,6 +546,8 @@ zabbix_export:
                   value: PostgreSQL
                 - tag: Database
                   value: '{#DBNAME}'
+                - tag: Type
+                  value: Calculated
             - uuid: 261e83e1e87c42d587a9409a8a26f971
               name: 'Tuples returned by sequential scans on {#DBNAME} - 1h delta'
               type: CALCULATED
@@ -531,6 +560,8 @@ zabbix_export:
                   value: PostgreSQL
                 - tag: Database
                   value: '{#DBNAME}'
+                - tag: Type
+                  value: Calculated
             - uuid: e46c4ec2a21d48288337b90549fbf757
               name: 'Tuples returned by sequential scans on {#DBNAME} - 1m delta'
               type: CALCULATED
@@ -543,6 +574,8 @@ zabbix_export:
                   value: PostgreSQL
                 - tag: Database
                   value: '{#DBNAME}'
+                - tag: Type
+                  value: Calculated
             - uuid: 26a42b55d7f949588f0739388ff52831
               name: 'Tuples returned by sequential scans on {#DBNAME} - 5m delta'
               type: CALCULATED
@@ -555,24 +588,88 @@ zabbix_export:
                   value: PostgreSQL
                 - tag: Database
                   value: '{#DBNAME}'
+                - tag: Type
+                  value: Calculated
             - uuid: 492b3cac15f348c2b85f97b69c114d1b
               name: 'Database Stats for {#DBNAME}'
-              key: 'web.page.get[localhost,/db_stats?dbname={#DBNAME},{$AGENT_PORT}]'
+              type: HTTP_AGENT
+              key: 'pgmon_db_stats[{#DBNAME}]'
               history: '0'
               value_type: TEXT
-              preprocessing:
-                - type: REGEX
-                  parameters:
-                    - '\n\s?\n([\s\S]*)'
-                    - \1
+              trends: '0'
+              url: 'http://localhost:{$AGENT_PORT}/db_stats'
+              query_fields:
+                - name: dbname
+                  value: '{#DBNAME}'
               tags:
                 - tag: Application
                   value: PostgreSQL
                 - tag: Database
                   value: '{#DBNAME}'
           graph_prototypes:
+            - uuid: 1f7de43b77714f819e61c31273712b70
+              name: 'DML Totals for {#DBNAME}'
+              graph_items:
+                - color: 199C0D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_db[tup_deleted,{#DBNAME}]'
+                - sortorder: '1'
+                  color: F63100
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_db[tup_inserted,{#DBNAME}]'
+                - sortorder: '2'
+                  color: 2774A4
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_db[tup_updated,{#DBNAME}]'
+            - uuid: aaec1de6f2314cbd875980fc1be3a2db
+              name: 'Sessions for {#DBNAME}'
+              graph_items:
+                - color: 199C0D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_db[sessions_fatal,{#DBNAME}]'
+                - sortorder: '1'
+                  color: F63100
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_db[xact_rollback,{#DBNAME}]'
+                - sortorder: '2'
+                  color: 2774A4
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_db[sessions,{#DBNAME}]'
+                - sortorder: '3'
+                  color: F7941D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_db[sessions_killed,{#DBNAME}]'
+            - uuid: a08d60bd0ffb4f90b5411d30a057a85e
+              name: 'Temp Files for {#DBNAME}'
+              graph_items:
+                - color: 199C0D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_db[temp_files,{#DBNAME}]'
+                - sortorder: '1'
+                  color: F63100
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_db[temp_bytes,{#DBNAME}]'
             - uuid: 4949cdfbda614af796a2856fdfa9ac3f
-              name: 'Time breakdown for {#DBNAME} on {#CLUSTER}'
+              name: 'Time breakdown for {#DBNAME}'
               graph_items:
                 - color: 199C0D
                   calc_fnc: ALL
@@ -597,17 +694,14 @@ zabbix_export:
                   item:
                     host: 'PostgreSQL by pgmon'
                     key: 'pgmon_db[blk_write_time,{#DBNAME}]'
+          url: 'http://localhost:{$AGENT_PORT}/discover_dbs'
           lld_macro_paths:
             - lld_macro: '{#DBNAME}'
               path: $.dbname
-          preprocessing:
-            - type: REGEX
-              parameters:
-                - '\n\s?\n([\s\S]*)'
-                - \1
         - uuid: 8ec029d577ae4872858e2e5cfd1cc40e
           name: 'Discover Replication'
-          key: 'web.page.get[localhost,/discover_rep,{$AGENT_PORT}]'
+          type: HTTP_AGENT
+          key: pgmon_discover_rep
           delay: 10m
           lifetime: 30d
           enabled_lifetime_type: DISABLE_NEVER
@@ -616,6 +710,7 @@ zabbix_export:
               name: 'Flush lag for {#REPID}'
               type: DEPENDENT
               key: 'pgmon_rep[flush_lag,repid={#REPID}]'
+              delay: '0'
               history: 90d
               value_type: FLOAT
               description: 'Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written and flushed it (but not yet applied it). This can be used to gauge the delay that synchronous_commit level on incurred while committing if this server was configured as a synchronous standby.'
@@ -624,7 +719,7 @@ zabbix_export:
                   parameters:
                     - $.flush_lag
               master_item:
-                key: 'web.page.get[localhost,/rep_stats?repid={#REPID},{$AGENT_PORT}]'
+                key: 'pgmon_rep_stats[{#REPID}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -636,15 +731,17 @@ zabbix_export:
               name: 'Last flush LSN for {#REPID}'
               type: DEPENDENT
               key: 'pgmon_rep[flush_lsn,repid={#REPID}]'
+              delay: '0'
               history: 90d
               value_type: TEXT
+              trends: '0'
               description: 'Last write-ahead log location flushed to disk by this standby server'
               preprocessing:
                 - type: JSONPATH
                   parameters:
                     - $.flush_lsn
               master_item:
-                key: 'web.page.get[localhost,/rep_stats?repid={#REPID},{$AGENT_PORT}]'
+                key: 'pgmon_rep_stats[{#REPID}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -656,6 +753,7 @@ zabbix_export:
               name: 'Replay lag for {#REPID}'
               type: DEPENDENT
               key: 'pgmon_rep[replay_lag,repid={#REPID}]'
+              delay: '0'
               history: 90d
               value_type: FLOAT
               description: 'Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written, flushed and applied it. This can be used to gauge the delay that synchronous_commit level remote_apply incurred while committing if this server was configured as a synchronous standby.'
@@ -664,7 +762,7 @@ zabbix_export:
                   parameters:
                     - $.replay_lag
               master_item:
-                key: 'web.page.get[localhost,/rep_stats?repid={#REPID},{$AGENT_PORT}]'
+                key: 'pgmon_rep_stats[{#REPID}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -676,15 +774,17 @@ zabbix_export:
               name: 'Last replay LSN for {#REPID}'
               type: DEPENDENT
               key: 'pgmon_rep[replay_lsn,repid={#REPID}]'
+              delay: '0'
               history: 90d
               value_type: TEXT
+              trends: '0'
               description: 'Last write-ahead log location replayed into the database on this standby server'
               preprocessing:
                 - type: JSONPATH
                   parameters:
                     - $.replay_lsn
               master_item:
-                key: 'web.page.get[localhost,/rep_stats?repid={#REPID},{$AGENT_PORT}]'
+                key: 'pgmon_rep_stats[{#REPID}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -696,15 +796,17 @@ zabbix_export:
               name: 'Last sent LSN for {#REPID}'
               type: DEPENDENT
               key: 'pgmon_rep[sent_lsn,repid={#REPID}]'
+              delay: '0'
               history: 90d
               value_type: TEXT
+              trends: '0'
               description: 'Last write-ahead log location sent on this connection'
               preprocessing:
                 - type: JSONPATH
                   parameters:
                     - $.sent_lsn
               master_item:
-                key: 'web.page.get[localhost,/rep_stats?repid={#REPID},{$AGENT_PORT}]'
+                key: 'pgmon_rep_stats[{#REPID}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -716,8 +818,10 @@ zabbix_export:
               name: 'Replication state for {#REPID}'
               type: DEPENDENT
               key: 'pgmon_rep[state,repid={#REPID}]'
+              delay: '0'
               history: 90d
               value_type: TEXT
+              trends: '0'
               description: |
                 Current WAL sender state. Possible values are:
                 * startup: This WAL sender is starting up.
@@ -730,7 +834,7 @@ zabbix_export:
                   parameters:
                     - $.state
               master_item:
-                key: 'web.page.get[localhost,/rep_stats?repid={#REPID},{$AGENT_PORT}]'
+                key: 'pgmon_rep_stats[{#REPID}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -742,6 +846,7 @@ zabbix_export:
               name: 'Write lag for {#REPID}'
               type: DEPENDENT
               key: 'pgmon_rep[write_lag,repid={#REPID}]'
+              delay: '0'
               history: 90d
               value_type: FLOAT
               description: 'Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written it (but not yet flushed it or applied it). This can be used to gauge the delay that synchronous_commit level remote_write incurred while committing if this server was configured as a synchronous standby.'
@@ -750,7 +855,7 @@ zabbix_export:
                   parameters:
                     - $.write_lag
               master_item:
-                key: 'web.page.get[localhost,/rep_stats?repid={#REPID},{$AGENT_PORT}]'
+                key: 'pgmon_rep_stats[{#REPID}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -762,15 +867,17 @@ zabbix_export:
               name: 'Last write LSN for {#REPID}'
               type: DEPENDENT
               key: 'pgmon_rep[write_lsn,repid={#REPID}]'
+              delay: '0'
               history: 90d
               value_type: TEXT
+              trends: '0'
               description: 'Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written it (but not yet flushed it or applied it). This can be used to gauge the delay that synchronous_commit level remote_write incurred while committing if this server was configured as a synchronous standby.'
               preprocessing:
                 - type: JSONPATH
                   parameters:
                     - $.write_lsn
               master_item:
-                key: 'web.page.get[localhost,/rep_stats?repid={#REPID},{$AGENT_PORT}]'
+                key: 'pgmon_rep_stats[{#REPID}]'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -780,14 +887,15 @@ zabbix_export:
                   value: '{#DBNAME}'
             - uuid: efbe11f37c2f499488bdc5853c3d89e6
               name: 'Replication Stats for {#REPID}'
-              key: 'web.page.get[localhost,/rep_stats?repid={#REPID},{$AGENT_PORT}]'
+              type: HTTP_AGENT
+              key: 'pgmon_rep_stats[{#REPID}]'
               history: '0'
               value_type: TEXT
-              preprocessing:
-                - type: REGEX
-                  parameters:
-                    - '\n\s?\n([\s\S]*)'
-                    - \1
+              trends: '0'
+              url: 'http://localhost:{$AGENT_PORT}/rep_stats'
+              query_fields:
+                - name: repid
+                  value: '{#REPID}'
               tags:
                 - tag: Application
                   value: PostgreSQL
@@ -795,6 +903,7 @@ zabbix_export:
                   value: Replication
                 - tag: Database
                   value: '{#DBNAME}'
+          url: 'http://localhost:{$AGENT_PORT}/discover_rep'
           lld_macro_paths:
             - lld_macro: '{#CLIENT_ADDR}'
               path: $.client_addr
@@ -802,12 +911,80 @@ zabbix_export:
               path: $.repid
             - lld_macro: '{#STATE}'
               path: $.state
-          preprocessing:
-            - type: REGEX
-              parameters:
-                - '\n\s?\n([\s\S]*)'
-                - \1
       macros:
         - macro: '{$AGENT_PORT}'
           value: '5400'
           description: 'The port the agent listens on'
+      dashboards:
+        - uuid: a818cfb97d654c75a3d70ae7f942bb89
+          name: 'PostgreSQL - Overview'
+          pages:
+            - widgets:
+                - type: graphprototype
+                  name: 'Time Breakdown'
+                  width: '39'
+                  height: '5'
+                  fields:
+                    - type: INTEGER
+                      name: columns
+                      value: '1'
+                    - type: GRAPH_PROTOTYPE
+                      name: graphid.0
+                      value:
+                        host: 'PostgreSQL by pgmon'
+                        name: 'Time breakdown for {#DBNAME}'
+                    - type: STRING
+                      name: reference
+                      value: ZAWWD
+                - type: graphprototype
+                  name: 'Session Breakdown'
+                  'y': '5'
+                  width: '39'
+                  height: '6'
+                  fields:
+                    - type: INTEGER
+                      name: columns
+                      value: '1'
+                    - type: GRAPH_PROTOTYPE
+                      name: graphid.0
+                      value:
+                        host: 'PostgreSQL by pgmon'
+                        name: 'Sessions for {#DBNAME}'
+                    - type: STRING
+                      name: reference
+                      value: WBVFA
+                - type: graphprototype
+                  name: 'Temp Files'
+                  x: '39'
+                  width: '33'
+                  height: '5'
+                  fields:
+                    - type: INTEGER
+                      name: columns
+                      value: '1'
+                    - type: GRAPH_PROTOTYPE
+                      name: graphid.0
+                      value:
+                        host: 'PostgreSQL by pgmon'
+                        name: 'Temp Files for {#DBNAME}'
+                    - type: STRING
+                      name: reference
+                      value: CFQAN
+                - type: graphprototype
+                  name: 'DML Totals'
+                  x: '39'
+                  'y': '5'
+                  width: '33'
+                  height: '6'
+                  fields:
+                    - type: INTEGER
+                      name: columns
+                      value: '1'
+                    - type: GRAPH_PROTOTYPE
+                      name: graphid.0
+                      value:
+                        host: 'PostgreSQL by pgmon'
+                        name: 'DML Totals for {#DBNAME}'
+                    - type: STRING
+                      name: reference
+                      value: XSCMZ