Add a metric for tracking the number of granted locks

* Slots are crated for each table during the initial sync, which only
  live for the duration of the copy for that table.

* The initial sync backend workers' names are also based on the table
  being copied.

* Filter out the above in Zabbix discovery based on application_name and
  slot_name.

DEBIAN/control

@@ -3,7 +3,7 @@ Version: 1.0
 Section: utils
 Priority: optional
 Architecture: all
-Depends: logrotate, python3 (>= 3.6), python3-psycopg2, python3-yaml, systemd
+Depends: logrotate, python3 (>= 3.6), python3-psycopg2, python3-requests, python3-yaml, systemd
 Maintainer: James Campbell <james@commandprompt.com>
 Homepage: https://www.commandprompt.com
 Description: A bridge to sit between monitoring tools and PostgreSQL

GENTOO/pgmon-1.0.2.ebuild

@@ -0,0 +1,73 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+PYTHON_COMPAT=( python3_{6..13} )
+
+inherit python-r1
+
+DESCRIPTION="PostgreSQL monitoring bridge"
+HOMEPAGE="None"
+
+LICENSE="BSD"
+SLOT="0"
+KEYWORDS="amd64"
+
+SRC_URI="https://code2.shh-dot-com.org/james/${PN}/archive/v${PV}.tar.bz2 -> ${P}.tar.bz2"
+
+IUSE="-systemd"
+
+DEPEND="
+	${PYTHON_DEPS}
+	dev-python/psycopg:2
+	dev-python/pyyaml
+	app-admin/logrotate
+	"
+RDEPEND="${DEPEND}"
+BDEPEND=""
+
+RESTRICT="fetch"
+
+#S="${WORKDIR}/${PN}"
+
+pkg_nofetch() {
+	einfo "Please download"
+	einfo "  - ${P}.tar.bz2"
+	einfo "from ${HOMEPAGE} and place it in your DISTDIR directory."
+	einfo "The file should be owned by portage:portage."
+}
+
+src_compile() {
+	true
+}
+
+src_install() {
+	# Install init script
+	if ! use systemd ; then
+		newinitd "openrc/pgmon.initd" pgmon
+		newconfd "openrc/pgmon.confd" pgmon
+	fi
+
+	# Install systemd unit
+	if use systemd ; then
+		systemd_dounit "systemd/pgmon.service"
+	fi
+
+	# Install script
+	exeinto /usr/bin
+	newexe "src/pgmon.py" pgmon
+
+	# Install default config
+	diropts -o root -g root -m 0755
+	insinto /etc/pgmon
+	doins "sample-config/pgmon.yml"
+	doins "sample-config/pgmon-metrics.yml"
+
+	# Install logrotate config
+	insinto /etc/logrotate.d
+	newins "logrotate/pgmon.logrotate" pgmon
+
+	# Install man page
+	doman manpages/pgmon.1
+}

GENTOO/pgmon-1.0.3.ebuild

@@ -0,0 +1,74 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+PYTHON_COMPAT=( python3_{6..13} )
+
+inherit python-r1 systemd
+
+DESCRIPTION="PostgreSQL monitoring bridge"
+HOMEPAGE="None"
+
+LICENSE="BSD"
+SLOT="0"
+KEYWORDS="amd64"
+
+SRC_URI="https://code2.shh-dot-com.org/james/${PN}/releases/download/v${PV}/${P}.tar.bz2"
+
+IUSE="-systemd"
+
+DEPEND="
+	${PYTHON_DEPS}
+	dev-python/psycopg:2
+	dev-python/pyyaml
+	dev-python/requests
+	app-admin/logrotate
+	"
+RDEPEND="${DEPEND}"
+BDEPEND=""
+
+#RESTRICT="fetch"
+
+#S="${WORKDIR}/${PN}"
+
+#pkg_nofetch() {
+#	einfo "Please download"
+#	einfo "  - ${P}.tar.bz2"
+#	einfo "from ${HOMEPAGE} and place it in your DISTDIR directory."
+#	einfo "The file should be owned by portage:portage."
+#}
+
+src_compile() {
+	true
+}
+
+src_install() {
+	# Install init script
+	if ! use systemd ; then
+		newinitd "openrc/pgmon.initd" pgmon
+		newconfd "openrc/pgmon.confd" pgmon
+	fi
+
+	# Install systemd unit
+	if use systemd ; then
+		systemd_dounit "systemd/pgmon.service"
+	fi
+
+	# Install script
+	exeinto /usr/bin
+	newexe "src/pgmon.py" pgmon
+
+	# Install default config
+	diropts -o root -g root -m 0755
+	insinto /etc/pgmon
+	doins "sample-config/pgmon.yml"
+	doins "sample-config/pgmon-metrics.yml"
+
+	# Install logrotate config
+	insinto /etc/logrotate.d
+	newins "logrotate/pgmon.logrotate" pgmon
+
+	# Install man page
+	doman manpages/pgmon.1
+}

GENTOO/pgmon-1.0.4.ebuild

@@ -0,0 +1,74 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+PYTHON_COMPAT=( python3_{6..13} )
+
+inherit python-r1 systemd
+
+DESCRIPTION="PostgreSQL monitoring bridge"
+HOMEPAGE="None"
+
+LICENSE="BSD"
+SLOT="0"
+KEYWORDS="amd64"
+
+SRC_URI="https://code2.shh-dot-com.org/james/${PN}/releases/download/v${PV}/${P}.tar.bz2"
+
+IUSE="-systemd"
+
+DEPEND="
+	${PYTHON_DEPS}
+	dev-python/psycopg:2
+	dev-python/pyyaml
+	dev-python/requests
+	app-admin/logrotate
+	"
+RDEPEND="${DEPEND}"
+BDEPEND=""
+
+#RESTRICT="fetch"
+
+#S="${WORKDIR}/${PN}"
+
+#pkg_nofetch() {
+#	einfo "Please download"
+#	einfo "  - ${P}.tar.bz2"
+#	einfo "from ${HOMEPAGE} and place it in your DISTDIR directory."
+#	einfo "The file should be owned by portage:portage."
+#}
+
+src_compile() {
+	true
+}
+
+src_install() {
+	# Install init script
+	if ! use systemd ; then
+		newinitd "openrc/pgmon.initd" pgmon
+		newconfd "openrc/pgmon.confd" pgmon
+	fi
+
+	# Install systemd unit
+	if use systemd ; then
+		systemd_dounit "systemd/pgmon.service"
+	fi
+
+	# Install script
+	exeinto /usr/bin
+	newexe "src/pgmon.py" pgmon
+
+	# Install default config
+	diropts -o root -g root -m 0755
+	insinto /etc/pgmon
+	doins "sample-config/pgmon.yml"
+	doins "sample-config/pgmon-metrics.yml"
+
+	# Install logrotate config
+	insinto /etc/logrotate.d
+	newins "logrotate/pgmon.logrotate" pgmon
+
+	# Install man page
+	doman manpages/pgmon.1
+}

Makefile

@@ -1,9 +1,25 @@
 # Package details
 PACKAGE_NAME := pgmon
-VERSION := 1.0.1
 
 SCRIPT := src/$(PACKAGE_NAME).py
 
+# Figure out the version components
+# Note: The release is for RPM packages, where prerelease releases are written as 0.<release>
+FULL_VERSION := $(shell grep -m 1 '^VERSION = ' "$(SCRIPT)" | sed -ne 's/.*"\(.*\)".*/\1/p')
+VERSION := $(shell echo $(FULL_VERSION) | sed -n 's/\(.*\)\(-rc.*\|$$\)/\1/p')
+RELEASE := $(shell echo $(FULL_VERSION) | sed -n 's/.*-rc\([0-9]\+\)$$/\1/p')
+
+ifeq ($(RELEASE),)
+RPM_RELEASE := 1
+RPM_VERSION := $(VERSION)-$(RPM_RELEASE)
+DEB_VERSION := $(VERSION)
+else
+RPM_RELEASE := 0.$(RELEASE)
+RPM_VERSION := $(VERSION)-$(RPM_RELEASE)
+DEB_VERSION := $(VERSION)~rc$(RELEASE)
+endif
+
+
 # Where packages are built
 BUILD_DIR := build
 
@@ -22,18 +38,22 @@ SUPPORTED := ubuntu-20.04 \
 # These targets are the main ones to use for most things.
 ##
 
-.PHONY: all clean tgz test install
+.PHONY: all clean tgz lint format test query-tests install-common install-openrc install-systemd
 
+all: package-all
+
+version:
+	@echo "full version=$(FULL_VERSION) version=$(VERSION) rel=$(RELEASE) rpm=$(RPM_VERSION) deb=$(DEB_VERSION)"
 
 # Build all packages
 .PHONY: package-all
-all: $(foreach distro_release, $(SUPPORTED), package-$(distro_release))
+package-all: $(foreach distro_release, $(SUPPORTED), package-$(distro_release))
 
 # Gentoo package (tar.gz) creation
 .PHONY: package-gentoo
 package-gentoo:
 	mkdir -p $(BUILD_DIR)/gentoo
-	tar --transform "s,^\.,$(PACKAGE_NAME)-$(VERSION)," -acjf $(BUILD_DIR)/gentoo/$(PACKAGE_NAME)-$(VERSION).tar.bz2 --exclude $(BUILD_DIR) .
+	tar --transform "s,^,$(PACKAGE_NAME)-$(FULL_VERSION)/," -acjf $(BUILD_DIR)/gentoo/$(PACKAGE_NAME)-$(FULL_VERSION).tar.bz2 --exclude .gitignore $(shell git ls-tree --full-tree --name-only -r HEAD)
 
 
 # Create a deb package
@@ -53,29 +73,44 @@ package-%:
 tgz:
 	rm -rf $(BUILD_DIR)/tgz/root
 	mkdir -p $(BUILD_DIR)/tgz/root
-	$(MAKE) install DESTDIR=$(BUILD_DIR)/tgz/root
-	tar -cz -f $(BUILD_DIR)/tgz/$(PACKAGE_NAME)-$(VERSION).tgz -C $(BUILD_DIR)/tgz/root .
+	$(MAKE) install-openrc DESTDIR=$(BUILD_DIR)/tgz/root
+	tar -cz -f $(BUILD_DIR)/tgz/$(PACKAGE_NAME)-$(FULL_VERSION).tgz -C $(BUILD_DIR)/tgz/root .
 
 # Clean up the build directory
 clean:
 	rm -rf $(BUILD_DIR)
 
+# Check for lint
+lint:
+	pylint src/pgmon.py
+	pylint src/test_pgmon.py
+	black --check --diff src/pgmon.py
+	black --check --diff src/test_pgmon.py
+
+# Format the code using black
+format:
+	black src/pgmon.py
+	black src/test_pylint.py
+
 # Run unit tests for the script
 test:
 	cd src ; python3 -m unittest
 
-# Install the script at the specified base directory
-install:
+# Run query tests
+query-tests:
+	cd tests ; ./run-tests.sh
+
+# Install the script at the specified base directory (common components)
+install-common:
 	# Set up directories
 	mkdir -p $(DESTDIR)/etc/$(PACKAGE_NAME)
 	mkdir -p ${DESTDIR}/etc/logrotate.d
-	mkdir -p $(DESTDIR)/lib/systemd/system
-	mkdir -p $(DESTDIR)/usr/local/bin
+	mkdir -p $(DESTDIR)/usr/bin
 	mkdir -p $(DESTDIR)/usr/share/man/man1
 	
 	# Install script
-	cp $(SCRIPT) $(DESTDIR)/usr/local/bin/$(PACKAGE_NAME)
-	chmod 755 $(DESTDIR)/usr/local/bin/$(PACKAGE_NAME)
+	cp $(SCRIPT) $(DESTDIR)/usr/bin/$(PACKAGE_NAME)
+	chmod 755 $(DESTDIR)/usr/bin/$(PACKAGE_NAME)
 	
 	# Install manpage
 	cp manpages/* $(DESTDIR)/usr/share/man/man1/
@@ -84,15 +119,39 @@ install:
 	# Install sample config
 	cp sample-config/* $(DESTDIR)/etc/$(PACKAGE_NAME)/
 	
-	# Install systemd unit files
-	cp systemd/* $(DESTDIR)/lib/systemd/system/
-	
 	# Install logrotate config
 	cp logrotate/${PACKAGE_NAME}.logrotate ${DESTDIR}/etc/logrotate.d/${PACKAGE_NAME}
 
+# Install for systemd
+install-systemd:
+	# Install the common stuff
+	$(MAKE) install-common
+	
+	# Set up directories
+	mkdir -p $(DESTDIR)/lib/systemd/system
+	
+	# Install systemd unit files
+	cp systemd/* $(DESTDIR)/lib/systemd/system/
+
+# Install for open-rc
+install-openrc:
+	# Install the common stuff
+	$(MAKE) install-common
+	
+	# Set up directories
+	mkdir -p $(DESTDIR)/etc/init.d
+	mkdir -p $(DESTDIR)/etc/conf.d
+	
+	# Install init script
+	cp openrc/pgmon.initd $(DESTDIR)/etc/init.d/pgmon
+	chmod 755 $(DESTDIR)/etc/init.d/pgmon
+	
+	# Install init script config file
+	cp openrc/pgmon.confd $(DESTDIR)/etc/conf.d/pgmon
+
 
 # Run all of the install tests
-.PHONY: install-tests debian-%-install-test rockylinux-%-install-test ubuntu-%-install-test
+.PHONY: install-tests debian-%-install-test rockylinux-%-install-test ubuntu-%-install-test gentoo-install-test
 install-tests: $(foreach distro_release, $(SUPPORTED), $(distro_release)-install-test)
 
 
@@ -101,28 +160,33 @@ debian-%-install-test:
 	docker run --rm \
 	     -v ./$(BUILD_DIR):/output \
 	     debian:$* \
-	     bash -c 'apt-get update && apt-get install -y /output/$(PACKAGE_NAME)-$(VERSION)-debian-$*.deb'
+	     bash -c 'apt-get update && apt-get install -y /output/$(PACKAGE_NAME)-$(DEB_VERSION)-debian-$*.deb'
 
 # Run a RedHat install test
 rockylinux-%-install-test:
 	docker run --rm \
 	     -v ./$(BUILD_DIR):/output \
 	     rockylinux:$* \
-	     bash -c 'dnf makecache && dnf install -y /output/$(PACKAGE_NAME)-$(VERSION)-1.el$*.noarch.rpm'
+	     bash -c 'dnf makecache && dnf install -y /output/$(PACKAGE_NAME)-$(RPM_VERSION).el$*.noarch.rpm'
 
 # Run an Ubuntu install test
 ubuntu-%-install-test:
 	docker run --rm \
 	     -v ./$(BUILD_DIR):/output \
 	     ubuntu:$* \
-	     bash -c 'apt-get update && apt-get install -y /output/$(PACKAGE_NAME)-$(VERSION)-ubuntu-$*.deb'
+	     bash -c 'apt-get update && apt-get install -y /output/$(PACKAGE_NAME)-$(DEB_VERSION)-ubuntu-$*.deb'
 
 # Run an OracleLinux install test (this is for EL7 since CentOS7 images no longer exist)
 oraclelinux-%-install-test:
 	docker run --rm \
 	     -v ./$(BUILD_DIR):/output \
 	     oraclelinux:7 \
-	     bash -c 'yum makecache && yum install -y /output/$(PACKAGE_NAME)-$(VERSION)-1.el7.noarch.rpm'
+	     bash -c 'yum makecache && yum install -y /output/$(PACKAGE_NAME)-$(RPM_VERSION).el7.noarch.rpm'
+
+# Run a Gentoo install test
+gentoo-install-test:
+	# May impliment this in the future, but would require additional headaches to set up a repo
+	true
 
 ##
 # Container targets
@@ -157,30 +221,30 @@ package-image-%:
 
 # Debian package creation
 actually-package-debian-%:
-	$(MAKE) install DESTDIR=/output/debian-$*
+	$(MAKE) install-systemd DESTDIR=/output/debian-$*
 	cp -r --preserve=mode DEBIAN /output/debian-$*/
-	dpkg-deb -Zgzip --build /output/debian-$* "/output/$(PACKAGE_NAME)-$(VERSION)-debian-$*.deb"
+	dpkg-deb -Zgzip --build /output/debian-$* "/output/$(PACKAGE_NAME)-$(DEB_VERSION)-debian-$*.deb"
 
 # RedHat package creation
 actually-package-rockylinux-%:
 	mkdir -p /output/rockylinux-$*/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
-	sed -e "s/@@VERSION@@/$(VERSION)/g" RPM/$(PACKAGE_NAME).spec > /output/rockylinux-$*/SPECS/$(PACKAGE_NAME).spec
+	sed -e "s/@@VERSION@@/$(VERSION)/g" -e "s/@@RELEASE@@/$(RPM_RELEASE)/g" RPM/$(PACKAGE_NAME).spec > /output/rockylinux-$*/SPECS/$(PACKAGE_NAME).spec
 	rpmbuild --define '_topdir /output/rockylinux-$*' \
-	         --define 'version $(VERSION)' \
+	         --define 'version $(RPM_VERSION)' \
 	         -bb /output/rockylinux-$*/SPECS/$(PACKAGE_NAME).spec
-	cp /output/rockylinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(VERSION)-1.el$*.noarch.rpm /output/
+	cp /output/rockylinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(RPM_VERSION).el$*.noarch.rpm /output/
 
 # Ubuntu package creation
 actually-package-ubuntu-%:
-	$(MAKE) install DESTDIR=/output/ubuntu-$*
+	$(MAKE) install-systemd DESTDIR=/output/ubuntu-$*
 	cp -r --preserve=mode DEBIAN /output/ubuntu-$*/
-	dpkg-deb -Zgzip --build /output/ubuntu-$* "/output/$(PACKAGE_NAME)-$(VERSION)-ubuntu-$*.deb"
+	dpkg-deb -Zgzip --build /output/ubuntu-$* "/output/$(PACKAGE_NAME)-$(DEB_VERSION)-ubuntu-$*.deb"
 
 # OracleLinux package creation
 actually-package-oraclelinux-%:
 	mkdir -p /output/oraclelinux-$*/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
-	sed -e "s/@@VERSION@@/$(VERSION)/g" RPM/$(PACKAGE_NAME)-el7.spec > /output/oraclelinux-$*/SPECS/$(PACKAGE_NAME).spec
+	sed -e "s/@@VERSION@@/$(VERSION)/g" -e "s/@@RELEASE@@/$(RPM_RELEASE)/g" RPM/$(PACKAGE_NAME)-el7.spec > /output/oraclelinux-$*/SPECS/$(PACKAGE_NAME).spec
 	rpmbuild --define '_topdir /output/oraclelinux-$*' \
-	         --define 'version $(VERSION)' \
+	         --define 'version $(RPM_VERSION)' \
 	         -bb /output/oraclelinux-$*/SPECS/$(PACKAGE_NAME).spec
-	cp /output/oraclelinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(VERSION)-1.el$*.noarch.rpm /output/
+	cp /output/oraclelinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(RPM_VERSION).el$*.noarch.rpm /output/

RPM/pgmon-el7.spec

@@ -1,13 +1,13 @@
 Name: pgmon
 Version: @@VERSION@@
-Release: 1%{?dist}
+Release: @@RELEASE@@%{?dist}
 Summary: A bridge to sit between monitoring tools and PostgreSQL
 
 License: MIT
 URL: https://www.commandprompt.com
 
 BuildArch: noarch
-Requires: logrotate, python, python-psycopg2, PyYAML, systemd
+Requires: logrotate, python, python-psycopg2, PyYAML, python-requests, systemd
 
 %description
 A bridge to sit between monitoring tools and PostgreSQL
@@ -19,7 +19,7 @@ A bridge to sit between monitoring tools and PostgreSQL
 # Do nothing since we have nothing to build
 
 %install
-make -C /src install DESTDIR=%{buildroot}
+make -C /src install-systemd DESTDIR=%{buildroot}
 
 %files
 /etc/logrotate.d/pgmon
@@ -28,7 +28,7 @@ make -C /src install DESTDIR=%{buildroot}
 /etc/pgmon/pgmon-service.conf
 /lib/systemd/system/pgmon.service
 /lib/systemd/system/pgmon@.service
-/usr/local/bin/pgmon
+/usr/bin/pgmon
 /usr/share/man/man1/pgmon.1.gz
 
 %post

RPM/pgmon.spec

@@ -1,13 +1,13 @@
 Name: pgmon
 Version: @@VERSION@@
-Release: 1%{?dist}
+Release: @@RELEASE@@%{?dist}
 Summary: A bridge to sit between monitoring tools and PostgreSQL
 
 License: MIT
 URL: https://www.commandprompt.com
 
 BuildArch: noarch
-Requires: logrotate, python3, python3-psycopg2, python3-pyyaml, systemd
+Requires: logrotate, python3, python3-psycopg2, python3-pyyaml, python3-requests, systemd
 
 %description
 A bridge to sit between monitoring tools and PostgreSQL
@@ -19,7 +19,7 @@ A bridge to sit between monitoring tools and PostgreSQL
 # Do nothing since we have nothing to build
 
 %install
-make -C /src install DESTDIR=%{buildroot}
+make -C /src install-systemd DESTDIR=%{buildroot}
 
 %files
 /etc/logrotate.d/pgmon
@@ -28,7 +28,7 @@ make -C /src install DESTDIR=%{buildroot}
 /etc/pgmon/pgmon-service.conf
 /lib/systemd/system/pgmon.service
 /lib/systemd/system/pgmon@.service
-/usr/local/bin/pgmon
+/usr/bin/pgmon
 /usr/share/man/man1/pgmon.1.gz
 
 %post

openrc/pgmon.initd

@@ -19,6 +19,6 @@ start_pre() {
 }
 
 command="/usr/bin/pgmon"
-command_args="'$CONFIG_FILE'"
+command_args="-c '$CONFIG_FILE'"
 command_background="true"
 command_user="${PGMON_USER}:${PGMON_GROUP}"

pylintrc

@@ -0,0 +1,4 @@
+[MASTER]
+py-version=3.5
+
+disable=fixme

requirements-dev.yml

@@ -0,0 +1,4 @@
+-r requirements.txt
+testcontainers[postgresql]
+pytest
+black

sample-config/pgmon-metrics.yml

@@ -1,55 +1,320 @@
 metrics:
+  ##
   # Discovery metrics
+  ##
   discover_dbs:
     type: set
     query:
-      0: SELECT datname AS dbname FROM pg_database
+      0: >
+        SELECT datname AS dbname
+        FROM pg_database
+
+  # Note: If the user lacks sufficient privileges, these fields will be NULL.
+  #       The WHERE clause is intended to prevent Zabbix from discovering a
+  #       connection it cannot monitor.  Ideally this would generate an error
+  #       instead.
   discover_rep:
     type: set
     query:
-      0: SELECT client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') AS repid, client_addr, state FROM pg_stat_replication
+      0: >
+        SELECT host(client_addr) || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') AS repid,
+               application_name,
+               client_addr,
+               state
+        FROM pg_stat_replication
+        WHERE state IS NOT NULL
+
   discover_slots:
     type: set
     query:
-      0: SELECT slot_name, plugin, slot_type, database, temporary, active FROM pg_replication_slots
+      90400: >
+        SELECT slot_name,
+               plugin,
+               slot_type,
+               database,
+               false as temporary,
+               active
+        FROM pg_replication_slots
+      100000: >
+        SELECT slot_name,
+               plugin,
+               slot_type,
+               database,
+               temporary,
+               active
+        FROM pg_replication_slots
 
+
+  ##
   # cluster-wide metrics
+  ##
   version:
     type: value
     query:
       0: SHOW server_version_num
-  max_frozen_age:
-    type: value
-    query:
-      0: SELECT max(age(datfrozenxid)) FROM pg_database
 
+  max_frozen_age:
+    type: row
+    query:
+      0: >
+        SELECT max(age(datfrozenxid)) AS xid_age,
+               NULL AS mxid_age
+        FROM pg_database
+      90600: >
+        SELECT max(age(datfrozenxid)) AS xid_age,
+               max(mxid_age(datminmxid)) AS mxid_age
+        FROM pg_database
+
+  bgwriter:
+    type: row
+    query:
+      0: >
+        SELECT checkpoints_timed,
+               checkpoints_req,
+               checkpoint_write_time,
+               checkpoint_sync_time,
+               buffers_checkpoint,
+               buffers_clean,
+               maxwritten_clean,
+               buffers_backend,
+               buffers_backend_fsync,
+               buffers_alloc
+        FROM pg_stat_bgwriter
+      170000: >
+        SELECT cp.num_timed AS checkpoints_timed,
+               cp.num_requested AS checkpoints_req,
+               cp.write_time AS checkpoint_write_time,
+               cp.sync_time AS checkpoint_sync_time,
+               cp.buffers_written AS buffers_checkpoint,
+               bg.buffers_clean AS buffers_clean,
+               bg.maxwritten_clean AS maxwritten_clean,
+               NULL AS buffers_backend,
+               NULL AS buffers_backend_fsync,
+               bg.buffers_alloc AS buffers_alloc
+        FROM pg_stat_bgwriter bg
+        CROSS JOIN pg_stat_checkpointer cp
+
+  io_per_backend:
+    type: set
+    query:
+      160000: >
+        SELECT backend_type,
+               COALESCE(SUM(reads * op_bytes), 0) AS reads,
+               COALESCE(SUM(read_time), 0) AS read_time,
+               COALESCE(SUM(writes * op_bytes), 0) AS writes,
+               COALESCE(SUM(write_time), 0) AS write_time,
+               COALESCE(SUM(writebacks * op_bytes), 0) AS writebacks,
+               COALESCE(SUM(writeback_time), 0) AS writeback_time,
+               COALESCE(SUM(extends * op_bytes), 0) AS extends,
+               COALESCE(SUM(extend_time), 0) AS extend_time,
+               COALESCE(SUM(op_bytes), 0) AS op_bytes,
+               COALESCE(SUM(hits), 0) AS hits,
+               COALESCE(SUM(evictions), 0) AS evictions,
+               COALESCE(SUM(reuses), 0) AS reuses,
+               COALESCE(SUM(fsyncs), 0) AS fsyncs,
+               COALESCE(SUM(fsync_time), 0) AS fsync_time
+        FROM pg_stat_io
+        GROUP BY backend_type
+
+
+  ##
   # Per-database metrics
+  ##
   db_stats:
     type: row
     query:
-      0: SELECT numbackends, xact_commit, xact_rollback, blks_read, blks_hit, tup_returned, tup_fetched, tup_inserted, tup_updated, tup_deleted, conflicts, temp_files, temp_bytes, deadlocks, blk_read_time, blk_write_time, extract('epoch' from stats_reset)::float FROM pg_stat_database WHERE datname = %(dbname)s
-      140000: SELECT numbackends, xact_commit, xact_rollback, blks_read, blks_hit, tup_returned, tup_fetched, tup_inserted, tup_updated, tup_deleted, conflicts, temp_files, temp_bytes, deadlocks, COALESCE(checksum_failures, 0) AS checksum_failures, blk_read_time, blk_write_time, session_time, active_time, idle_in_transaction_time, sessions, sessions_abandoned, sessions_fatal, sessions_killed, extract('epoch' from stats_reset)::float FROM pg_stat_database WHERE datname = %(dbname)s
+      0: >
+        SELECT numbackends,
+               xact_commit,
+               xact_rollback,
+               blks_read,
+               blks_hit,
+               tup_returned,
+               tup_fetched,
+               tup_inserted,
+               tup_updated,
+               tup_deleted,
+               conflicts,
+               temp_files,
+               temp_bytes,
+               deadlocks,
+               NULL AS checksum_failures,
+               blk_read_time,
+               blk_write_time,
+               NULL AS session_time,
+               NULL AS active_time,
+               NULL AS idle_in_transaction_time,
+               NULL AS sessions,
+               NULL AS sessions_abandoned,
+               NULL AS sessions_fatal,
+               NULL AS sessions_killed,
+               extract('epoch' from stats_reset) AS stats_reset
+        FROM pg_stat_database WHERE datname = %(dbname)s
+      140000: >
+        SELECT numbackends,
+               xact_commit,
+               xact_rollback,
+               blks_read,
+               blks_hit,
+               tup_returned,
+               tup_fetched,
+               tup_inserted,
+               tup_updated,
+               tup_deleted,
+               conflicts,
+               temp_files,
+               temp_bytes,
+               deadlocks,
+               COALESCE(checksum_failures, 0) AS checksum_failures,
+               blk_read_time,
+               blk_write_time,
+               session_time,
+               active_time,
+               idle_in_transaction_time,
+               sessions,
+               sessions_abandoned,
+               sessions_fatal,
+               sessions_killed,
+               extract('epoch' from stats_reset) AS stats_reset
+        FROM pg_stat_database WHERE datname = %(dbname)s
+    test_args:
+      dbname: postgres
 
+  hit_ratios:
+    type: row
+    query:
+      0: >
+        SELECT sum(heap_blks_read)::float / NULLIF(sum(heap_blks_read + heap_blks_hit), 0) AS avg_heap_hit_ratio,
+               sum(idx_blks_hit)::float / NULLIF(sum(idx_blks_read + idx_blks_hit), 0) AS avg_idx_hit_ratio,
+               sum(toast_blks_hit)::float / NULLIF(sum(toast_blks_read + toast_blks_hit), 0) AS avg_toast_hit_ratio,
+               sum(tidx_blks_hit)::float / NULLIF(sum(tidx_blks_read + tidx_blks_hit), 0) AS avg_tidx_hit_ratio
+        FROM pg_statio_all_tables
+    test_args:
+      dbname: postgres
+
+  activity:
+    type: set
+    query:
+      0: >
+        SELECT state,
+               count(*) AS backend_count,
+               COALESCE(EXTRACT(EPOCH FROM max(now() - state_change)), 0) AS max_state_time
+        FROM pg_stat_activity
+        WHERE datname = %(dbname)s
+        GROUP BY state
+    test_args:
+      dbname: postgres
+
+  sequence_usage:
+    type: value
+    query:
+# 9.2 lacks lateral joins, the pg_sequence_last_value function, and the pg_sequences view
+#      0: >
+#        SELECT COALESCE(MAX(pg_sequence_last_value(c.oid)::float / (pg_sequence_parameters(oid)).maximum_value), 0) AS max_usage
+#        FROM pg_class c
+#        WHERE c.relkind = 'S'
+# 9.3 - 9.6 lacks the pg_sequence_last_value function, and pg_sequences view
+#      90300: >
+#        SELECT COALESCE(MAX(pg_sequence_last_value(c.oid)::float / s.maximum_value), 0) AS max_usage
+#        FROM pg_class c
+#        CROSS JOIN LATERAL pg_sequence_parameters(c.oid) AS s
+#        WHERE c.relkind = 'S'
+      100000: SELECT COALESCE(MAX(last_value::float / max_value), 0) AS max_usage FROM pg_sequences;
+    test_args:
+      dbname: postgres
+
+  sequence_visibility:
+    type: row
+    query:
+      100000: >
+        SELECT COUNT(*) FILTER (WHERE has_sequence_privilege(c.oid, 'SELECT,USAGE')) AS visible_sequences,
+               COUNT(*) AS total_sequences
+        FROM pg_class AS c
+        WHERE relkind = 'S'
+
+  locks:
+    type: row
+    query:
+      0:
+        SELECT COUNT(*) AS total,
+               SUM(CASE WHEN granted THEN 1 ELSE 0 END) AS granted
+        FROM pg_locks
+      90400: >
+        SELECT COUNT(*) AS total,
+               COUNT(*) FILTER (WHERE granted) AS granted
+        FROM pg_locks
+
+
+  ##
+  # Per-replication metrics
+  ##
+  rep_stats:
+    type: row
+    query:
+      90400: >
+        SELECT pid, usename,
+               EXTRACT(EPOCH FROM backend_start) AS backend_start,
+               state,
+               pg_xlog_location_diff(pg_current_xlog_location(), sent_location) AS sent_lsn,
+               pg_xlog_location_diff(pg_current_xlog_location(), write_location) AS write_lsn,
+               pg_xlog_location_diff(pg_current_xlog_location(), flush_location) AS flush_lsn,
+               pg_xlog_location_diff(pg_current_xlog_location(), replay_location) AS replay_lsn,
+               NULL AS write_lag,
+               NULL AS flush_lag,
+               NULL AS replay_lag,
+               sync_state
+        FROM pg_stat_replication
+        WHERE host(client_addr) || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = %(repid)s
+      100000: >
+        SELECT pid, usename,
+               EXTRACT(EPOCH FROM backend_start) AS backend_start,
+               state,
+               pg_wal_lsn_diff(pg_current_wal_lsn(), sent_lsn) AS sent_lsn,
+               pg_wal_lsn_diff(pg_current_wal_lsn(), write_lsn) AS write_lsn,
+               pg_wal_lsn_diff(pg_current_wal_lsn(), flush_lsn) AS flush_lsn,
+               pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) AS replay_lsn,
+               COALESCE(EXTRACT(EPOCH FROM write_lag), 0) AS write_lag,
+               COALESCE(EXTRACT(EPOCH FROM flush_lag), 0) AS flush_lag,
+               COALESCE(EXTRACT(EPOCH FROM replay_lag), 0) AS replay_lag,
+               sync_state
+        FROM pg_stat_replication
+        WHERE host(client_addr) || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = %(repid)s
+    test_args:
+      repid: 127.0.0.1_test_rep
+
+
+  ##
+  # Per-slot metrics
+  ##
+  slot_stats:
+    type: row
+    query:
+      90400: >
+        SELECT NULL as active_pid,
+               xmin,
+               pg_xlog_location_diff(pg_current_xlog_location(), restart_lsn) AS restart_bytes,
+               NULL AS confirmed_flush_bytes
+        FROM pg_replication_slots WHERE slot_name = %(slot)s
+      90600: >
+        SELECT active_pid,
+               xmin,
+               pg_xlog_location_diff(pg_current_xlog_location(), restart_lsn) AS restart_bytes,
+               pg_xlog_location_diff(pg_current_xlog_location(), confirmed_flush_lsn) AS confirmed_flush_bytes
+        FROM pg_replication_slots WHERE slot_name = %(slot)s
+      100000: >
+        SELECT active_pid,
+               xmin,
+               pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) AS restart_bytes,
+               pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn) AS confirmed_flush_bytes
+        FROM pg_replication_slots WHERE slot_name = %(slot)s
+    test_args:
+      slot: test_slot
+
+
+  ##
   # Debugging
+  ##
   ntables:
     type: value
     query:
       0: SELECT count(*) AS ntables FROM pg_stat_user_tables
-
-  # Per-replication metrics
-  rep_stats:
-    type: row
-    query:
-      0: SELECT * FROM pg_stat_database WHERE client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = '{repid}'
-
-  # Debugging
-  sleep:
-    type: value
-    query:
-      0: SELECT now(), pg_sleep(5);
-
-  # Per-slot metrics
-  slot_stats:
-    type: row
-    query:
-      0: SELECT active_pid, xmin, pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) AS restart_bytes, pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn) AS confirmed_flush_bytes FROM pg_replication_slots WHERE slot_name = '{slot}'

sample-config/pgmon.yml

@@ -1,3 +1,6 @@
+# The address the agent binds to
+#address: 127.0.0.1
+
 # The port the agent listens on for requests
 #port: 5400
 
@@ -26,6 +29,9 @@
 # Default database to connect to when none is specified for a metric
 #dbname: 'postgres'
 
+# SSL connection mode
+#ssl_mode: require
+
 # Timeout for getting a connection slot from a pool
 #pool_slot_timeout: 5
 

src/test_pgmon.py

@@ -1,22 +1,127 @@
+"""
+Unit tests for pgmon
+"""
+
+# pylint: disable=too-many-lines
+
 import unittest
 
+import os
 from datetime import datetime, timedelta
 import tempfile
 
 import logging
 
+from decimal import Decimal
+import json
+
 import pgmon
 
 # Silence most logging output
 logging.disable(logging.CRITICAL)
 
+VERSIONS_RSS = """
+<?xml version="1.0" encoding="utf-8"?>
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>PostgreSQL latest versions</title><link>https://www.postgresql.org/</link><description>PostgreSQL latest versions</description><atom:link href="https://www.postgresql.org/versions.rss" rel="self"/><language>en-us</language><lastBuildDate>Thu, 08 May 2025 00:00:00 +0000</lastBuildDate><item><title>17.5
+</title><link>https://www.postgresql.org/docs/17/release-17-5.html</link><description>17.5 is the latest release in the 17 series.
+
+</description><pubDate>Thu, 08 May 2025 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/17/release-17-5.html</guid></item><item><title>16.9
+</title><link>https://www.postgresql.org/docs/16/release-16-9.html</link><description>16.9 is the latest release in the 16 series.
+
+</description><pubDate>Thu, 08 May 2025 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/16/release-16-9.html</guid></item><item><title>15.13
+</title><link>https://www.postgresql.org/docs/15/release-15-13.html</link><description>15.13 is the latest release in the 15 series.
+
+</description><pubDate>Thu, 08 May 2025 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/15/release-15-13.html</guid></item><item><title>14.18
+</title><link>https://www.postgresql.org/docs/14/release-14-18.html</link><description>14.18 is the latest release in the 14 series.
+
+</description><pubDate>Thu, 08 May 2025 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/14/release-14-18.html</guid></item><item><title>13.21
+</title><link>https://www.postgresql.org/docs/13/release-13-21.html</link><description>13.21 is the latest release in the 13 series.
+
+</description><pubDate>Thu, 08 May 2025 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/13/release-13-21.html</guid></item><item><title>12.22
+</title><link>https://www.postgresql.org/docs/12/release-12-22.html</link><description>12.22 is the latest release in the 12 series.
+This version is unsupported!
+</description><pubDate>Thu, 21 Nov 2024 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/12/release-12-22.html</guid></item><item><title>11.22
+</title><link>https://www.postgresql.org/docs/11/release-11-22.html</link><description>11.22 is the latest release in the 11 series.
+This version is unsupported!
+</description><pubDate>Thu, 09 Nov 2023 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/11/release-11-22.html</guid></item><item><title>10.23
+</title><link>https://www.postgresql.org/docs/10/release-10-23.html</link><description>10.23 is the latest release in the 10 series.
+This version is unsupported!
+</description><pubDate>Thu, 10 Nov 2022 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/10/release-10-23.html</guid></item><item><title>9.6.24
+</title><link>https://www.postgresql.org/docs/9.6/release-9-6-24.html</link><description>9.6.24 is the latest release in the 9.6 series.
+This version is unsupported!
+</description><pubDate>Thu, 11 Nov 2021 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.6/release-9-6-24.html</guid></item><item><title>9.5.25
+</title><link>https://www.postgresql.org/docs/9.5/release-9-5-25.html</link><description>9.5.25 is the latest release in the 9.5 series.
+This version is unsupported!
+</description><pubDate>Thu, 11 Feb 2021 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.5/release-9-5-25.html</guid></item><item><title>9.4.26
+</title><link>https://www.postgresql.org/docs/9.4/release-9-4-26.html</link><description>9.4.26 is the latest release in the 9.4 series.
+This version is unsupported!
+</description><pubDate>Thu, 13 Feb 2020 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.4/release-9-4-26.html</guid></item><item><title>9.3.25
+</title><link>https://www.postgresql.org/docs/9.3/release-9-3-25.html</link><description>9.3.25 is the latest release in the 9.3 series.
+This version is unsupported!
+</description><pubDate>Thu, 08 Nov 2018 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.3/release-9-3-25.html</guid></item><item><title>9.2.24
+</title><link>https://www.postgresql.org/docs/9.2/release-9-2-24.html</link><description>9.2.24 is the latest release in the 9.2 series.
+This version is unsupported!
+</description><pubDate>Thu, 09 Nov 2017 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.2/release-9-2-24.html</guid></item><item><title>9.1.24
+</title><link>https://www.postgresql.org/docs/9.1/release-9-1-24.html</link><description>9.1.24 is the latest release in the 9.1 series.
+This version is unsupported!
+</description><pubDate>Thu, 27 Oct 2016 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.1/release-9-1-24.html</guid></item><item><title>9.0.23
+</title><link>https://www.postgresql.org/docs/9.0/release-9-0-23.html</link><description>9.0.23 is the latest release in the 9.0 series.
+This version is unsupported!
+</description><pubDate>Thu, 08 Oct 2015 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.0/release-9-0-23.html</guid></item><item><title>8.4.22
+</title><link>https://www.postgresql.org/docs/8.4/release-8-4-22.html</link><description>8.4.22 is the latest release in the 8.4 series.
+This version is unsupported!
+</description><pubDate>Thu, 24 Jul 2014 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/8.4/release-8-4-22.html</guid></item><item><title>8.3.23
+</title><link>https://www.postgresql.org/docs/8.3/release-8-3-23.html</link><description>8.3.23 is the latest release in the 8.3 series.
+This version is unsupported!
+</description><pubDate>Thu, 07 Feb 2013 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/8.3/release-8-3-23.html</guid></item><item><title>8.2.23
+</title><link>https://www.postgresql.org/docs/8.2/release-8-2-23.html</link><description>8.2.23 is the latest release in the 8.2 series.
+This version is unsupported!
+</description><pubDate>Mon, 05 Dec 2011 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/8.2/release-8-2-23.html</guid></item><item><title>8.1.23
+</title><link>https://www.postgresql.org/docs/8.1/release.html</link><description>8.1.23 is the latest release in the 8.1 series.
+This version is unsupported!
+</description><pubDate>Thu, 16 Dec 2010 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/8.1/release.html</guid></item><item><title>8.0.26
+</title><link>https://www.postgresql.org/docs/8.0/release.html</link><description>8.0.26 is the latest release in the 8.0 series.
+This version is unsupported!
+</description><pubDate>Mon, 04 Oct 2010 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/8.0/release.html</guid></item><item><title>7.4.30
+</title><link>https://www.postgresql.org/docs/7.4/release.html</link><description>7.4.30 is the latest release in the 7.4 series.
+This version is unsupported!
+</description><pubDate>Mon, 04 Oct 2010 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/7.4/release.html</guid></item><item><title>7.3.21
+</title><link>https://www.postgresql.org/docs/7.3/release.html</link><description>7.3.21 is the latest release in the 7.3 series.
+This version is unsupported!
+</description><pubDate>Mon, 07 Jan 2008 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/7.3/release.html</guid></item><item><title>7.2.8
+</title><link>https://www.postgresql.org/docs/7.2/release.html</link><description>7.2.8 is the latest release in the 7.2 series.
+This version is unsupported!
+</description><pubDate>Mon, 09 May 2005 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/7.2/release.html</guid></item><item><title>7.1.3
+</title><link>https://www.postgresql.org/docs/7.1/release.html</link><description>7.1.3 is the latest release in the 7.1 series.
+This version is unsupported!
+</description><pubDate>Fri, 17 Aug 2001 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/7.1/release.html</guid></item><item><title>7.0.3
+</title><link>https://www.postgresql.org/docs/7.0/release.htm</link><description>7.0.3 is the latest release in the 7.0 series.
+This version is unsupported!
+</description><pubDate>Sun, 12 Nov 2000 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/7.0/release.htm</guid></item><item><title>6.5.3
+</title><link>https://www.postgresql.org/docs/6.5/release.htm</link><description>6.5.3 is the latest release in the 6.5 series.
+This version is unsupported!
+</description><pubDate>Thu, 04 Nov 1999 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/6.5/release.htm</guid></item><item><title>6.4.2
+</title><link>https://www.postgresql.org/docs/6.4/release.htm</link><description>6.4.2 is the latest release in the 6.4 series.
+This version is unsupported!
+</description><pubDate>Sun, 03 Jan 1999 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/6.4/release.htm</guid></item><item><title>6.3.2
+</title><link>https://www.postgresql.org/docs/6.3/c2701.htm</link><description>6.3.2 is the latest release in the 6.3 series.
+This version is unsupported!
+</description><pubDate>Mon, 23 Feb 1998 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/6.3/c2701.htm</guid></item></channel></rss>
+"""
+
+
+class TestPgmonMethods(unittest.TestCase):  # pylint: disable=too-many-public-methods
+    """
+    Unit test class for pgmon
+    """
 
-class TestPgmonMethods(unittest.TestCase):
     ##
     # update_deep
     ##
     def test_update_deep__empty_cases(self):
-        # Test empty dict cases
+        """
+        Test various empty dictionary permutations
+        """
         d1 = {}
         d2 = {}
         pgmon.update_deep(d1, d2)
@@ -36,7 +141,9 @@ class TestPgmonMethods(unittest.TestCase):
         self.assertEqual(d2, d1)
 
     def test_update_deep__scalars(self):
-        # Test adding/updating scalar values
+        """
+        Test adding/updating scalar values
+        """
         d1 = {"foo": 1, "bar": "text", "hello": "world"}
         d2 = {"foo": 2, "baz": "blah"}
         pgmon.update_deep(d1, d2)
@@ -44,7 +151,9 @@ class TestPgmonMethods(unittest.TestCase):
         self.assertEqual(d2, {"foo": 2, "baz": "blah"})
 
     def test_update_deep__lists(self):
-        # Test adding to lists
+        """
+        Test adding to lists
+        """
         d1 = {"lst1": []}
         d2 = {"lst1": [1, 2]}
         pgmon.update_deep(d1, d2)
@@ -80,7 +189,9 @@ class TestPgmonMethods(unittest.TestCase):
         self.assertEqual(d2, {"obj1": {"l1": [3, 4]}})
 
     def test_update_deep__dicts(self):
-        # Test adding to lists
+        """
+        Test adding to dictionaries
+        """
         d1 = {"obj1": {}}
         d2 = {"obj1": {"a": 1, "b": 2}}
         pgmon.update_deep(d1, d2)
@@ -107,7 +218,9 @@ class TestPgmonMethods(unittest.TestCase):
         self.assertEqual(d2, {"obj1": {"d1": {"a": 5, "c": 12}}})
 
     def test_update_deep__types(self):
-        # Test mismatched types
+        """
+        Test mismatched types
+        """
         d1 = {"foo": 5}
         d2 = None
         self.assertRaises(TypeError, pgmon.update_deep, d1, d2)
@@ -126,15 +239,19 @@ class TestPgmonMethods(unittest.TestCase):
     ##
 
     def test_get_pool__simple(self):
-        # Just get a pool in a normal case
-        pgmon.config.update(pgmon.default_config)
+        """
+        Test getting a pool in a normal case
+        """
+        pgmon.Context.config.update(pgmon.DEFAULT_CONFIG)
         pool = pgmon.get_pool("postgres")
         self.assertIsNotNone(pool)
 
     def test_get_pool__unhappy(self):
-        # Test getting an unhappy database pool
-        pgmon.config.update(pgmon.default_config)
-        pgmon.unhappy_cooldown["postgres"] = datetime.now() + timedelta(60)
+        """
+        Test getting an unhappy database pool
+        """
+        pgmon.Context.config.update(pgmon.DEFAULT_CONFIG)
+        pgmon.Context.unhappy_cooldown["postgres"] = datetime.now() + timedelta(60)
         self.assertRaises(pgmon.UnhappyDBError, pgmon.get_pool, "postgres")
 
         # Test getting a different database when there's an unhappy one
@@ -146,39 +263,49 @@ class TestPgmonMethods(unittest.TestCase):
     ##
 
     def test_handle_connect_failure__simple(self):
-        # Test adding to an empty unhappy list
-        pgmon.config.update(pgmon.default_config)
-        pgmon.unhappy_cooldown = {}
+        """
+        Test adding to an empty unhappy list
+        """
+        pgmon.Context.config.update(pgmon.DEFAULT_CONFIG)
+        pgmon.Context.unhappy_cooldown = {}
         pool = pgmon.get_pool("postgres")
         pgmon.handle_connect_failure(pool)
-        self.assertGreater(pgmon.unhappy_cooldown["postgres"], datetime.now())
+        self.assertGreater(pgmon.Context.unhappy_cooldown["postgres"], datetime.now())
 
         # Test adding another database
         pool = pgmon.get_pool("template0")
         pgmon.handle_connect_failure(pool)
-        self.assertGreater(pgmon.unhappy_cooldown["postgres"], datetime.now())
-        self.assertGreater(pgmon.unhappy_cooldown["template0"], datetime.now())
-        self.assertEqual(len(pgmon.unhappy_cooldown), 2)
+        self.assertGreater(pgmon.Context.unhappy_cooldown["postgres"], datetime.now())
+        self.assertGreater(pgmon.Context.unhappy_cooldown["template0"], datetime.now())
+        self.assertEqual(len(pgmon.Context.unhappy_cooldown), 2)
 
     ##
     # get_query
     ##
 
     def test_get_query__basic(self):
-        # Test getting a query with one version
+        """
+        Test getting a query with just a default version.
+        """
         metric = {"type": "value", "query": {0: "DEFAULT"}}
         self.assertEqual(pgmon.get_query(metric, 100000), "DEFAULT")
 
     def test_get_query__versions(self):
+        """
+        Test getting queries when multiple versions are present.
+        """
         metric = {"type": "value", "query": {0: "DEFAULT", 110000: "NEW"}}
 
-        # Test getting the default version of a query with no lower bound and a newer version
+        # Test getting the default version of a query with no lower bound and a newer
+        # version
         self.assertEqual(pgmon.get_query(metric, 100000), "DEFAULT")
 
-        # Test getting the newer version of a query with no lower bound and a newer version for the newer version
+        # Test getting the newer version of a query with no lower bound and a newer
+        # version for the newer version
         self.assertEqual(pgmon.get_query(metric, 110000), "NEW")
 
-        # Test getting the newer version of a query with no lower bound and a newer version for an even newer version
+        # Test getting the newer version of a query with no lower bound and a newer
+        # version for an even newer version
         self.assertEqual(pgmon.get_query(metric, 160000), "NEW")
 
         # Test getting a version in bwtween two other versions
@@ -186,6 +313,9 @@ class TestPgmonMethods(unittest.TestCase):
         self.assertEqual(pgmon.get_query(metric, 100000), "OLD")
 
     def test_get_query__missing_version(self):
+        """
+        Test trying to get a query that is not defined for the requested version.
+        """
         metric = {"type": "value", "query": {96000: "OLD", 110000: "NEW", 150000: ""}}
 
         # Test getting a metric that only exists for newer versions
@@ -199,11 +329,16 @@ class TestPgmonMethods(unittest.TestCase):
     ##
 
     def test_read_config__simple(self):
-        pgmon.config = {}
+        """
+        Test reading a simple config.
+        """
+        pgmon.Context.config = {}
 
         # Test reading just a metric and using the defaults for everything else
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 # This is a comment!
@@ -215,18 +350,20 @@ metrics:
 """
                 )
 
-            pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(os.path.join(tmpdirname, "config.yml"))
 
         self.assertEqual(
-            pgmon.config["max_pool_size"], pgmon.default_config["max_pool_size"]
+            pgmon.Context.config["max_pool_size"], pgmon.DEFAULT_CONFIG["max_pool_size"]
         )
-        self.assertEqual(pgmon.config["dbuser"], pgmon.default_config["dbuser"])
+        self.assertEqual(pgmon.Context.config["dbuser"], pgmon.DEFAULT_CONFIG["dbuser"])
 
-        pgmon.config = {}
+        pgmon.Context.config = {}
 
         # Test reading a basic config
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 # This is a comment!
@@ -262,21 +399,27 @@ metrics:
 """
                 )
 
-            pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(os.path.join(tmpdirname, "config.yml"))
 
-        self.assertEqual(pgmon.config["dbuser"], "someone")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["type"], "value")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
-        self.assertEqual(pgmon.config["metrics"]["test2"]["query"][0], "TEST2")
+        self.assertEqual(pgmon.Context.config["dbuser"], "someone")
+        self.assertEqual(pgmon.Context.config["metrics"]["test1"]["type"], "value")
+        self.assertEqual(pgmon.Context.config["metrics"]["test1"]["query"][0], "TEST1")
+        self.assertEqual(pgmon.Context.config["metrics"]["test2"]["query"][0], "TEST2")
 
     def test_read_config__include(self):
-        pgmon.config = {}
+        """
+        Test including one config from another.
+        """
+        pgmon.Context.config = {}
 
-        # Test reading a config that includes other files (absolute and relative paths, multiple levels)
+        # Test reading a config that includes other files (absolute and relative paths,
+        # multiple levels)
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
-                    f"""---
+                    """---
 # This is a comment!
 min_pool_size: 1
 max_pool_size: 2
@@ -288,13 +431,17 @@ reconnect_cooldown: 15
 version_check_period: 3600
 include:
   - dbsettings.yml
-  - {tmpdirname}/metrics.yml
-"""
+  - {}/metrics.yml
+""".format(
+                        tmpdirname
+                    )
                 )
 
-            with open(f"{tmpdirname}/dbsettings.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "dbsettings.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
-                    f"""---
+                    """---
 dbuser: someone
 dbhost: localhost
 dbport: 5555
@@ -302,9 +449,11 @@ dbname: template0
 """
                 )
 
-            with open(f"{tmpdirname}/metrics.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "metrics.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
-                    f"""---
+                    """---
 metrics:
   test1:
     type: value
@@ -319,9 +468,11 @@ include:
 """
                 )
 
-            with open(f"{tmpdirname}/more_metrics.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "more_metrics.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
-                    f"""---
+                    """---
 metrics:
   test3:
     type: value
@@ -329,20 +480,25 @@ metrics:
       0: TEST3
 """
                 )
-            pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(os.path.join(tmpdirname, "config.yml"))
 
-        self.assertEqual(pgmon.config["max_idle_time"], 10)
-        self.assertEqual(pgmon.config["dbuser"], "someone")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
-        self.assertEqual(pgmon.config["metrics"]["test2"]["query"][0], "TEST2")
-        self.assertEqual(pgmon.config["metrics"]["test3"]["query"][0], "TEST3")
+        self.assertEqual(pgmon.Context.config["max_idle_time"], 10)
+        self.assertEqual(pgmon.Context.config["dbuser"], "someone")
+        self.assertEqual(pgmon.Context.config["metrics"]["test1"]["query"][0], "TEST1")
+        self.assertEqual(pgmon.Context.config["metrics"]["test2"]["query"][0], "TEST2")
+        self.assertEqual(pgmon.Context.config["metrics"]["test3"]["query"][0], "TEST3")
 
     def test_read_config__reload(self):
-        pgmon.config = {}
+        """
+        Test reloading a config.
+        """
+        pgmon.Context.config = {}
 
         # Test rereading a config to update an existing config
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 # This is a comment!
@@ -370,12 +526,14 @@ metrics:
 """
                 )
 
-            pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(os.path.join(tmpdirname, "config.yml"))
 
             # Just make sure the first config was read
-            self.assertEqual(len(pgmon.config["metrics"]), 2)
+            self.assertEqual(len(pgmon.Context.config["metrics"]), 2)
 
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 # This is a comment!
@@ -388,18 +546,23 @@ metrics:
 """
                 )
 
-            pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(os.path.join(tmpdirname, "config.yml"))
 
-        self.assertEqual(pgmon.config["min_pool_size"], 7)
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "NEW1")
-        self.assertEqual(len(pgmon.config["metrics"]), 1)
+        self.assertEqual(pgmon.Context.config["min_pool_size"], 7)
+        self.assertEqual(pgmon.Context.config["metrics"]["test1"]["query"][0], "NEW1")
+        self.assertEqual(len(pgmon.Context.config["metrics"]), 1)
 
     def test_read_config__query_file(self):
-        pgmon.config = {}
+        """
+        Test reading a query definition from a separate file
+        """
+        pgmon.Context.config = {}
 
         # Read a config file that reads a query from a file
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 metrics:
@@ -410,22 +573,30 @@ metrics:
 """
                 )
 
-            with open(f"{tmpdirname}/some_query.sql", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "some_query.sql"), "w", encoding="utf-8"
+            ) as f:
                 f.write("This is a query")
 
-            pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(os.path.join(tmpdirname, "config.yml"))
 
         self.assertEqual(
-            pgmon.config["metrics"]["test1"]["query"][0], "This is a query"
+            pgmon.Context.config["metrics"]["test1"]["query"][0], "This is a query"
         )
 
-    def test_read_config__invalid(self):
-        pgmon.config = {}
+    def init_invalid_config_test(self):
+        """
+        Initialize an invalid config read test.  Basically just set up a simple valid config in
+        order to confirm that an invalid read does not modify the live config.
+        """
+        pgmon.Context.config = {}
 
         # For all of these tests, we start with a valid config and also ensure that
         #  it is not modified when a new config read fails
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 metrics:
@@ -436,20 +607,48 @@ metrics:
 """
                 )
 
-            pgmon.read_config(f"{tmpdirname}/config.yml")
+            pgmon.read_config(os.path.join(tmpdirname, "config.yml"))
 
         # Just make sure the config was read
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+        self.assertEqual(pgmon.Context.config["metrics"]["test1"]["query"][0], "TEST1")
+
+    def verify_invalid_config_test(self):
+        """
+        Verify that an invalid read did not modify the live config.
+        """
+        self.assertEqual(pgmon.Context.config["dbuser"], "postgres")
+        self.assertEqual(pgmon.Context.config["metrics"]["test1"]["query"][0], "TEST1")
+
+    def test_read_config__missing(self):
+        """
+        Test reading a nonexistant config file.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
 
         # Test reading a nonexistant config file
         with tempfile.TemporaryDirectory() as tmpdirname:
             self.assertRaises(
-                FileNotFoundError, pgmon.read_config, f"{tmpdirname}/missing.yml"
+                FileNotFoundError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "missing.yml"),
             )
 
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    def test_read_config__invalid(self):
+        """
+        Test reading an invalid config file.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
+
         # Test reading an invalid config file
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """[default]
 This looks a lot like an ini file to me
@@ -458,12 +657,26 @@ Or maybe a TOML?
 """
                 )
             self.assertRaises(
-                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+                pgmon.ConfigError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "config.yml"),
             )
 
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    def test_read_config__invalid_include(self):
+        """
+        Test reading an invalid config file.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
+
         # Test reading a config that includes an invalid file
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 dbuser: evil
@@ -477,14 +690,26 @@ include:
 """
                 )
             self.assertRaises(
-                FileNotFoundError, pgmon.read_config, f"{tmpdirname}/config.yml"
+                FileNotFoundError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "config.yml"),
             )
-        self.assertEqual(pgmon.config["dbuser"], "postgres")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    def test_read_config__invalid_log_level(self):
+        """
+        Test reading an invalid log level from a config file.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
 
         # Test invalid log level
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 log_level: noisy
@@ -497,14 +722,26 @@ metrics:
 """
                 )
             self.assertRaises(
-                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+                pgmon.ConfigError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "config.yml"),
             )
-        self.assertEqual(pgmon.config["dbuser"], "postgres")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    def test_read_config__invalid_type(self):
+        """
+        Test reading an invalid query result type form a config file.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
 
         # Test invalid query return type
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 dbuser: evil
@@ -516,32 +753,57 @@ metrics:
 """
                 )
             self.assertRaises(
-                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+                pgmon.ConfigError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "config.yml"),
             )
-        self.assertEqual(pgmon.config["dbuser"], "postgres")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    def test_read_config__invalid_query_dict(self):
+        """
+        Test reading an invalid query definition structure type form a config file.  In other words
+        what's supposed to be a dictionary of the form version => query, we give it something else.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
 
         # Test invalid query dict type
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 dbuser: evil
 metrics:
   test1:
-    type: lots_of_data
+    type: row
     query: EVIL1
 """
                 )
             self.assertRaises(
-                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+                pgmon.ConfigError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "config.yml"),
             )
-        self.assertEqual(pgmon.config["dbuser"], "postgres")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    def test_read_config__missing_type(self):
+        """
+        Test reading a metric with a missing result type from a config file.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
 
         # Test incomplete metric: missing type
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 dbuser: evil
@@ -552,14 +814,26 @@ metrics:
 """
                 )
             self.assertRaises(
-                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+                pgmon.ConfigError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "config.yml"),
             )
-        self.assertEqual(pgmon.config["dbuser"], "postgres")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    def test_read_config__missing_queries(self):
+        """
+        Test reading a metric with no queries from a config file.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
 
         # Test incomplete metric: missing queries
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 dbuser: evil
@@ -569,14 +843,26 @@ metrics:
 """
                 )
             self.assertRaises(
-                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+                pgmon.ConfigError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "config.yml"),
             )
-        self.assertEqual(pgmon.config["dbuser"], "postgres")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    def test_read_config__empty_query_dict(self):
+        """
+        Test reading a fetric with an empty query dict from a config file.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
 
         # Test incomplete metric: empty queries
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 dbuser: evil
@@ -587,14 +873,26 @@ metrics:
 """
                 )
             self.assertRaises(
-                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+                pgmon.ConfigError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "config.yml"),
             )
-        self.assertEqual(pgmon.config["dbuser"], "postgres")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    def test_read_config__none_query_dict(self):
+        """
+        Test reading a metric where the query dict is None from a config file.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
 
         # Test incomplete metric: query dict is None
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 dbuser: evil
@@ -605,28 +903,53 @@ metrics:
 """
                 )
             self.assertRaises(
-                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+                pgmon.ConfigError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "config.yml"),
             )
-        self.assertEqual(pgmon.config["dbuser"], "postgres")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    def test_read_config__missing_metrics(self):
+        """
+        Test reading a config file with no metrics.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
 
         # Test reading a config with no metrics
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 dbuser: evil
 """
                 )
             self.assertRaises(
-                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+                pgmon.ConfigError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "config.yml"),
             )
-        self.assertEqual(pgmon.config["dbuser"], "postgres")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    def test_read_config__missing_query_file(self):
+        """
+        Test reading a metric from a config file where the query definition cones from a missing
+        file.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
 
         # Test reading a query defined in a file but the file is missing
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 dbuser: evil
@@ -638,14 +961,26 @@ metrics:
 """
                 )
             self.assertRaises(
-                FileNotFoundError, pgmon.read_config, f"{tmpdirname}/config.yml"
+                FileNotFoundError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "config.yml"),
             )
-        self.assertEqual(pgmon.config["dbuser"], "postgres")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    def test_read_config__invalid_version(self):
+        """
+        Test reading a metric with an invalid PostgreSQL version from a config file.
+        """
+        # Set up the test
+        self.init_invalid_config_test()
 
         # Test invalid query versions
         with tempfile.TemporaryDirectory() as tmpdirname:
-            with open(f"{tmpdirname}/config.yml", "w") as f:
+            with open(
+                os.path.join(tmpdirname, "config.yml"), "w", encoding="utf-8"
+            ) as f:
                 f.write(
                     """---
 dbuser: evil
@@ -657,7 +992,96 @@ metrics:
 """
                 )
             self.assertRaises(
-                pgmon.ConfigError, pgmon.read_config, f"{tmpdirname}/config.yml"
+                pgmon.ConfigError,
+                pgmon.read_config,
+                os.path.join(tmpdirname, "config.yml"),
             )
-        self.assertEqual(pgmon.config["dbuser"], "postgres")
-        self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+
+        # Confirm nothing changed
+        self.verify_invalid_config_test()
+
+    ##
+    # version_num
+    ##
+
+    def test_version_num_to_release__valid(self):
+        """
+        Test converting PostgreSQL versions before and after 10 when the numbering scheme changed.
+        """
+        self.assertEqual(pgmon.version_num_to_release(90602), 9.6)
+        self.assertEqual(pgmon.version_num_to_release(130002), 13)
+
+    ##
+    # parse_version_rss
+    ##
+
+    def test_parse_version_rss__supported(self):
+        """
+        Test parsing a supported version from the RSS feed
+        """
+        pgmon.parse_version_rss(VERSIONS_RSS, 13)
+        self.assertEqual(pgmon.Context.latest_version, 130021)
+        self.assertTrue(pgmon.Context.release_supported)
+
+    def test_parse_version_rss__unsupported(self):
+        """
+        Test parsing an unsupported version from the RSS feed
+        """
+        pgmon.parse_version_rss(VERSIONS_RSS, 9.6)
+        self.assertEqual(pgmon.Context.latest_version, 90624)
+        self.assertFalse(pgmon.Context.release_supported)
+
+    def test_parse_version_rss__missing(self):
+        """
+        Test asking about versions that don't exist in the RSS feed
+        """
+        self.assertRaises(
+            pgmon.LatestVersionCheckError, pgmon.parse_version_rss, VERSIONS_RSS, 9.7
+        )
+
+        self.assertRaises(
+            pgmon.LatestVersionCheckError, pgmon.parse_version_rss, VERSIONS_RSS, 99
+        )
+
+    ##
+    # get_latest_version
+    ##
+
+    def test_get_latest_version(self):
+        """
+        Test getting the latest version from the actual RSS feed
+        """
+        # Define a cluster version here so the test doesn't need a database
+        pgmon.Context.cluster_version_next_check = datetime.now() + timedelta(hours=1)
+        pgmon.Context.cluster_version = 90623
+
+        # Set up a default config
+        pgmon.update_deep(pgmon.Context.config, pgmon.DEFAULT_CONFIG)
+
+        # Make sure we can pull the RSS file (we assume the 9.6 series won't be getting
+        # any more updates)
+        self.assertEqual(pgmon.get_latest_version(), 90624)
+
+    ##
+    # json_encode_special
+    ##
+
+    def test_json_encode_special(self):
+        """
+        Test encoding Decimal types as JSON
+        """
+        # Confirm that we're getting the right type
+        self.assertFalse(isinstance(Decimal("0.5"), float))
+        self.assertTrue(isinstance(pgmon.json_encode_special(Decimal("0.5")), float))
+
+        # Make sure we get sane values
+        self.assertEqual(pgmon.json_encode_special(Decimal("0.5")), 0.5)
+        self.assertEqual(pgmon.json_encode_special(Decimal("12")), 12.0)
+
+        # Make sure we can still fail for other types
+        self.assertRaises(TypeError, pgmon.json_encode_special, object)
+
+        # Make sure we can actually serialize a Decimal
+        self.assertEqual(
+            json.dumps(Decimal("2.5"), default=pgmon.json_encode_special), "2.5"
+        )

systemd/pgmon@.service

@@ -7,7 +7,7 @@ After=network.target
 [Service]
 EnvironmentFile=/etc/pgmon/%i-service.conf
 User=${SERVICE_USER:-postgres}
-ExecStart=/usr/local/bin/pgmon /etc/pgmon/%i.yml
+ExecStart=/usr/bin/pgmon -c /etc/pgmon/%i.yml
 ExecReload=kill -HUP $MAINPID
 Restart=on-failure
 Type=exec

tests/Dockerfile

@@ -0,0 +1,23 @@
+FROM alpine:3.21
+
+RUN apk update && \
+    apk add py3-psycopg2 \
+            py3-requests \
+            py3-yaml \
+            tini
+
+WORKDIR /app
+
+COPY src/pgmon.py /app/
+
+COPY sample-config/pgmon-metrics.yml /app/
+
+COPY tests/test-config.yml /app/
+
+COPY --chmod=0600 --chown=postgres:postgres tests/pgpass /root/.pgpass
+
+ENTRYPOINT ["tini", "--"]
+
+EXPOSE 5400
+
+CMD ["/app/pgmon.py", "-c", "/app/test-config.yml", "--test"]

tests/docker-compose.yml

@@ -0,0 +1,32 @@
+---
+
+services:
+  agent:
+    image: pgmon
+    build:
+      context: ..
+      dockerfile: tests/Dockerfile
+    ports:
+      - :5400
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    image: "postgres:${PGTAG:-17-bookworm}"
+    ports:
+      - :5432
+    environment:
+      POSTGRES_PASSWORD: secret
+    healthcheck:
+      #test: [ "CMD", "pg_isready", "-U", "postgres" ]
+      test: [ "CMD-SHELL", "pg_controldata /var/lib/postgresql/data/ | grep -q 'in production'" ]
+      interval: 5s
+      timeout: 2s
+      retries: 40
+    command: >
+      postgres -c ssl=on
+               -c ssl_cert_file='/etc/ssl/certs/ssl-cert-snakeoil.pem'
+               -c ssl_key_file='/etc/ssl/private/ssl-cert-snakeoil.key'
+               -c listen_addresses='*'
+

tests/pgpass

@@ -0,0 +1 @@
+db:5432:*:postgres:secret

tests/run-tests.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Versions to test
+versions=( $@ )
+
+# If we weren't given any versions, test them all
+if [ ${#versions[@]} -eq 0 ]
+then
+    versions=( 9.2 9.4 9.6 10 11 12 13 14 15 16 17 )
+fi
+
+# Image tags to use
+declare -A images=()
+images["9.2"]='9.2'
+images["9.3"]='9.3'
+images["9.4"]='9.4'
+images["9.5"]='9.5'
+images["9.6"]='9.6-bullseye'
+images["10"]='10-bullseye'
+images["11"]='11-bookworm'
+images["12"]='12-bookworm'
+images["13"]='13-bookworm'
+images["14"]='14-bookworm'
+images["15"]='15-bookworm'
+images["16"]='16-bookworm'
+images["17"]='17-bookworm'
+
+declare -A results=()
+
+# Make sure everything's down to start with
+docker compose down
+
+# Make sure our agent container is up to date
+docker compose build agent
+
+for version in "${versions[@]}"
+do
+    echo
+    echo "Testing: PostgreSQL ${version}"
+
+    # Specify the version we're testing against
+    export PGTAG="${images["$version"]}"
+
+    # Start the containers
+    docker compose up --exit-code-from=agent agent
+    rc=$?
+
+    results["$version"]=$rc
+
+    # Destroy the containers
+    docker compose down
+done
+
+echo
+echo
+for v in "${versions[@]}"
+do
+    case "${results["$v"]}" in
+        0) msg="OK" ;;
+        1) msg="Query failure detected" ;;
+        18) msg="Docker image error: 18" ;;
+        *) msg="Unexpected error: ${results["$v"]}" ;;
+    esac
+    echo "$v -> $msg"
+done

tests/test-config.yml

@@ -0,0 +1,17 @@
+---
+
+# Bind to all interfaces so we can submit requests from outside the test container
+address: 0.0.0.0
+
+# We always just connect to the db container
+dbhost: db
+dbport: 5432
+dbuser: postgres
+
+# The SSL cipher parameters are too old in the 9.2 container, so we allow the tests
+# to be run without encryption
+ssl_mode: prefer
+
+# Pull in the standard metrics
+include:
+  - pgmon-metrics.yml