Add initial query for sequence usage

* Add a query for pg_stat_io metrics grouped by backend type

* Switch some unsupported values from 0 to NULL to better reflect their
  nonexistence

DEBIAN/control

@@ -3,7 +3,7 @@ Version: 1.0
 Section: utils
 Priority: optional
 Architecture: all
-Depends: logrotate, python3 (>= 3.6), python3-psycopg2, python3-yaml, systemd
+Depends: logrotate, python3 (>= 3.6), python3-psycopg2, python3-requests, python3-yaml, systemd
 Maintainer: James Campbell <james@commandprompt.com>
 Homepage: https://www.commandprompt.com
 Description: A bridge to sit between monitoring tools and PostgreSQL

GENTOO/pgmon-1.0.2.ebuild

@@ -0,0 +1,73 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+PYTHON_COMPAT=( python3_{6..13} )
+
+inherit python-r1
+
+DESCRIPTION="PostgreSQL monitoring bridge"
+HOMEPAGE="None"
+
+LICENSE="BSD"
+SLOT="0"
+KEYWORDS="amd64"
+
+SRC_URI="https://code2.shh-dot-com.org/james/${PN}/archive/v${PV}.tar.bz2 -> ${P}.tar.bz2"
+
+IUSE="-systemd"
+
+DEPEND="
+	${PYTHON_DEPS}
+	dev-python/psycopg:2
+	dev-python/pyyaml
+	app-admin/logrotate
+	"
+RDEPEND="${DEPEND}"
+BDEPEND=""
+
+RESTRICT="fetch"
+
+#S="${WORKDIR}/${PN}"
+
+pkg_nofetch() {
+	einfo "Please download"
+	einfo "  - ${P}.tar.bz2"
+	einfo "from ${HOMEPAGE} and place it in your DISTDIR directory."
+	einfo "The file should be owned by portage:portage."
+}
+
+src_compile() {
+	true
+}
+
+src_install() {
+	# Install init script
+	if ! use systemd ; then
+		newinitd "openrc/pgmon.initd" pgmon
+		newconfd "openrc/pgmon.confd" pgmon
+	fi
+
+	# Install systemd unit
+	if use systemd ; then
+		systemd_dounit "systemd/pgmon.service"
+	fi
+
+	# Install script
+	exeinto /usr/bin
+	newexe "src/pgmon.py" pgmon
+
+	# Install default config
+	diropts -o root -g root -m 0755
+	insinto /etc/pgmon
+	doins "sample-config/pgmon.yml"
+	doins "sample-config/pgmon-metrics.yml"
+
+	# Install logrotate config
+	insinto /etc/logrotate.d
+	newins "logrotate/pgmon.logrotate" pgmon
+
+	# Install man page
+	doman manpages/pgmon.1
+}

GENTOO/pgmon-1.0.3.ebuild

@@ -0,0 +1,74 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+PYTHON_COMPAT=( python3_{6..13} )
+
+inherit python-r1 systemd
+
+DESCRIPTION="PostgreSQL monitoring bridge"
+HOMEPAGE="None"
+
+LICENSE="BSD"
+SLOT="0"
+KEYWORDS="amd64"
+
+SRC_URI="https://code2.shh-dot-com.org/james/${PN}/releases/download/v${PV}/${P}.tar.bz2"
+
+IUSE="-systemd"
+
+DEPEND="
+	${PYTHON_DEPS}
+	dev-python/psycopg:2
+	dev-python/pyyaml
+	dev-python/requests
+	app-admin/logrotate
+	"
+RDEPEND="${DEPEND}"
+BDEPEND=""
+
+#RESTRICT="fetch"
+
+#S="${WORKDIR}/${PN}"
+
+#pkg_nofetch() {
+#	einfo "Please download"
+#	einfo "  - ${P}.tar.bz2"
+#	einfo "from ${HOMEPAGE} and place it in your DISTDIR directory."
+#	einfo "The file should be owned by portage:portage."
+#}
+
+src_compile() {
+	true
+}
+
+src_install() {
+	# Install init script
+	if ! use systemd ; then
+		newinitd "openrc/pgmon.initd" pgmon
+		newconfd "openrc/pgmon.confd" pgmon
+	fi
+
+	# Install systemd unit
+	if use systemd ; then
+		systemd_dounit "systemd/pgmon.service"
+	fi
+
+	# Install script
+	exeinto /usr/bin
+	newexe "src/pgmon.py" pgmon
+
+	# Install default config
+	diropts -o root -g root -m 0755
+	insinto /etc/pgmon
+	doins "sample-config/pgmon.yml"
+	doins "sample-config/pgmon-metrics.yml"
+
+	# Install logrotate config
+	insinto /etc/logrotate.d
+	newins "logrotate/pgmon.logrotate" pgmon
+
+	# Install man page
+	doman manpages/pgmon.1
+}

Makefile

@@ -1,9 +1,10 @@
 # Package details
 PACKAGE_NAME := pgmon
-VERSION := 1.0.1
 
 SCRIPT := src/$(PACKAGE_NAME).py
 
+VERSION := $(shell grep -m 1 '^VERSION = ' "$(SCRIPT)" | sed -ne 's/.*"\(.*\)".*/\1/p')
+
 # Where packages are built
 BUILD_DIR := build
 
@@ -22,7 +23,7 @@ SUPPORTED := ubuntu-20.04 \
 # These targets are the main ones to use for most things.
 ##
 
-.PHONY: all clean tgz test install
+.PHONY: all clean tgz test query-tests install-common install-openrc install-systemd
 
 
 # Build all packages
@@ -33,7 +34,7 @@ all: $(foreach distro_release, $(SUPPORTED), package-$(distro_release))
 .PHONY: package-gentoo
 package-gentoo:
 	mkdir -p $(BUILD_DIR)/gentoo
-	tar --transform "s,^\.,$(PACKAGE_NAME)-$(VERSION)," -acjf $(BUILD_DIR)/gentoo/$(PACKAGE_NAME)-$(VERSION).tar.bz2 --exclude $(BUILD_DIR) .
+	tar --transform "s,^,$(PACKAGE_NAME)-$(VERSION)/," -acjf $(BUILD_DIR)/gentoo/$(PACKAGE_NAME)-$(VERSION).tar.bz2 --exclude .gitignore $(shell git ls-tree --full-tree --name-only -r HEAD)
 
 
 # Create a deb package
@@ -53,7 +54,7 @@ package-%:
 tgz:
 	rm -rf $(BUILD_DIR)/tgz/root
 	mkdir -p $(BUILD_DIR)/tgz/root
-	$(MAKE) install DESTDIR=$(BUILD_DIR)/tgz/root
+	$(MAKE) install-openrc DESTDIR=$(BUILD_DIR)/tgz/root
 	tar -cz -f $(BUILD_DIR)/tgz/$(PACKAGE_NAME)-$(VERSION).tgz -C $(BUILD_DIR)/tgz/root .
 
 # Clean up the build directory
@@ -64,18 +65,21 @@ clean:
 test:
 	cd src ; python3 -m unittest
 
-# Install the script at the specified base directory
-install:
+# Run query tests
+query-tests:
+	cd tests ; ./run-tests.sh
+
+# Install the script at the specified base directory (common components)
+install-common:
 	# Set up directories
 	mkdir -p $(DESTDIR)/etc/$(PACKAGE_NAME)
 	mkdir -p ${DESTDIR}/etc/logrotate.d
-	mkdir -p $(DESTDIR)/lib/systemd/system
-	mkdir -p $(DESTDIR)/usr/local/bin
+	mkdir -p $(DESTDIR)/usr/bin
 	mkdir -p $(DESTDIR)/usr/share/man/man1
 	
 	# Install script
-	cp $(SCRIPT) $(DESTDIR)/usr/local/bin/$(PACKAGE_NAME)
-	chmod 755 $(DESTDIR)/usr/local/bin/$(PACKAGE_NAME)
+	cp $(SCRIPT) $(DESTDIR)/usr/bin/$(PACKAGE_NAME)
+	chmod 755 $(DESTDIR)/usr/bin/$(PACKAGE_NAME)
 	
 	# Install manpage
 	cp manpages/* $(DESTDIR)/usr/share/man/man1/
@@ -84,15 +88,39 @@ install:
 	# Install sample config
 	cp sample-config/* $(DESTDIR)/etc/$(PACKAGE_NAME)/
 	
-	# Install systemd unit files
-	cp systemd/* $(DESTDIR)/lib/systemd/system/
-	
 	# Install logrotate config
 	cp logrotate/${PACKAGE_NAME}.logrotate ${DESTDIR}/etc/logrotate.d/${PACKAGE_NAME}
 
+# Install for systemd
+install-systemd:
+	# Install the common stuff
+	$(MAKE) install-common
+	
+	# Set up directories
+	mkdir -p $(DESTDIR)/lib/systemd/system
+	
+	# Install systemd unit files
+	cp systemd/* $(DESTDIR)/lib/systemd/system/
+
+# Install for open-rc
+install-openrc:
+	# Install the common stuff
+	$(MAKE) install-common
+	
+	# Set up directories
+	mkdir -p $(DESTDIR)/etc/init.d
+	mkdir -p $(DESTDIR)/etc/conf.d
+	
+	# Install init script
+	cp openrc/pgmon.initd $(DESTDIR)/etc/init.d/pgmon
+	chmod 755 $(DESTDIR)/etc/init.d/pgmon
+	
+	# Install init script config file
+	cp openrc/pgmon.confd $(DESTDIR)/etc/conf.d/pgmon
+
 
 # Run all of the install tests
-.PHONY: install-tests debian-%-install-test rockylinux-%-install-test ubuntu-%-install-test
+.PHONY: install-tests debian-%-install-test rockylinux-%-install-test ubuntu-%-install-test gentoo-install-test
 install-tests: $(foreach distro_release, $(SUPPORTED), $(distro_release)-install-test)
 
 
@@ -124,6 +152,11 @@ oraclelinux-%-install-test:
 	     oraclelinux:7 \
 	     bash -c 'yum makecache && yum install -y /output/$(PACKAGE_NAME)-$(VERSION)-1.el7.noarch.rpm'
 
+# Run a Gentoo install test
+gentoo-install-test:
+	# May impliment this in the future, but would require additional headaches to set up a repo
+	true
+
 ##
 # Container targets
 #
@@ -157,7 +190,7 @@ package-image-%:
 
 # Debian package creation
 actually-package-debian-%:
-	$(MAKE) install DESTDIR=/output/debian-$*
+	$(MAKE) install-systemd DESTDIR=/output/debian-$*
 	cp -r --preserve=mode DEBIAN /output/debian-$*/
 	dpkg-deb -Zgzip --build /output/debian-$* "/output/$(PACKAGE_NAME)-$(VERSION)-debian-$*.deb"
 
@@ -172,7 +205,7 @@ actually-package-rockylinux-%:
 
 # Ubuntu package creation
 actually-package-ubuntu-%:
-	$(MAKE) install DESTDIR=/output/ubuntu-$*
+	$(MAKE) install-systemd DESTDIR=/output/ubuntu-$*
 	cp -r --preserve=mode DEBIAN /output/ubuntu-$*/
 	dpkg-deb -Zgzip --build /output/ubuntu-$* "/output/$(PACKAGE_NAME)-$(VERSION)-ubuntu-$*.deb"
 

RPM/pgmon-el7.spec

@@ -7,7 +7,7 @@ License: MIT
 URL: https://www.commandprompt.com
 
 BuildArch: noarch
-Requires: logrotate, python, python-psycopg2, PyYAML, systemd
+Requires: logrotate, python, python-psycopg2, PyYAML, python-requests, systemd
 
 %description
 A bridge to sit between monitoring tools and PostgreSQL
@@ -19,7 +19,7 @@ A bridge to sit between monitoring tools and PostgreSQL
 # Do nothing since we have nothing to build
 
 %install
-make -C /src install DESTDIR=%{buildroot}
+make -C /src install-systemd DESTDIR=%{buildroot}
 
 %files
 /etc/logrotate.d/pgmon
@@ -28,7 +28,7 @@ make -C /src install DESTDIR=%{buildroot}
 /etc/pgmon/pgmon-service.conf
 /lib/systemd/system/pgmon.service
 /lib/systemd/system/pgmon@.service
-/usr/local/bin/pgmon
+/usr/bin/pgmon
 /usr/share/man/man1/pgmon.1.gz
 
 %post

RPM/pgmon.spec

@@ -7,7 +7,7 @@ License: MIT
 URL: https://www.commandprompt.com
 
 BuildArch: noarch
-Requires: logrotate, python3, python3-psycopg2, python3-pyyaml, systemd
+Requires: logrotate, python3, python3-psycopg2, python3-pyyaml, python3-requests, systemd
 
 %description
 A bridge to sit between monitoring tools and PostgreSQL
@@ -19,7 +19,7 @@ A bridge to sit between monitoring tools and PostgreSQL
 # Do nothing since we have nothing to build
 
 %install
-make -C /src install DESTDIR=%{buildroot}
+make -C /src install-systemd DESTDIR=%{buildroot}
 
 %files
 /etc/logrotate.d/pgmon
@@ -28,7 +28,7 @@ make -C /src install DESTDIR=%{buildroot}
 /etc/pgmon/pgmon-service.conf
 /lib/systemd/system/pgmon.service
 /lib/systemd/system/pgmon@.service
-/usr/local/bin/pgmon
+/usr/bin/pgmon
 /usr/share/man/man1/pgmon.1.gz
 
 %post

openrc/pgmon.initd

@@ -19,6 +19,6 @@ start_pre() {
 }
 
 command="/usr/bin/pgmon"
-command_args="'$CONFIG_FILE'"
+command_args="-c '$CONFIG_FILE'"
 command_background="true"
 command_user="${PGMON_USER}:${PGMON_GROUP}"

requirements-dev.yml

@@ -0,0 +1,4 @@
+-r requirements.txt
+testcontainers[postgresql]
+pytest
+black

sample-config/pgmon-metrics.yml

@@ -1,55 +1,291 @@
 metrics:
+  ##
   # Discovery metrics
+  ##
   discover_dbs:
     type: set
     query:
-      0: SELECT datname AS dbname FROM pg_database
+      0: >
+        SELECT datname AS dbname
+        FROM pg_database
   discover_rep:
     type: set
     query:
-      0: SELECT client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') AS repid, client_addr, state FROM pg_stat_replication
+      0: >
+        SELECT client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') AS repid,
+               client_addr,
+               state
+        FROM pg_stat_replication
   discover_slots:
     type: set
     query:
-      0: SELECT slot_name, plugin, slot_type, database, temporary, active FROM pg_replication_slots
+      90400: >
+        SELECT slot_name,
+               plugin,
+               slot_type,
+               database,
+               false as temporary,
+               active
+        FROM pg_replication_slots
+      100000: >
+        SELECT slot_name,
+               plugin,
+               slot_type,
+               database,
+               temporary,
+               active
+        FROM pg_replication_slots
 
+  ##
   # cluster-wide metrics
+  ##
   version:
     type: value
     query:
       0: SHOW server_version_num
-  max_frozen_age:
-    type: value
-    query:
-      0: SELECT max(age(datfrozenxid)) FROM pg_database
 
+  max_frozen_age:
+    type: row
+    query:
+      0: >
+        SELECT max(age(datfrozenxid)) AS xid_age,
+               NULL AS mxid_age
+        FROM pg_database
+      90600: >
+        SELECT max(age(datfrozenxid)) AS xid_age,
+               max(mxid_age(datminmxid)) AS mxid_age
+        FROM pg_database
+
+  bgwriter:
+    type: row
+    query:
+      0: >
+        SELECT checkpoints_timed,
+               checkpoints_req,
+               checkpoint_write_time,
+               checkpoint_sync_time,
+               buffers_checkpoint,
+               buffers_clean,
+               maxwritten_clean,
+               buffers_backend,
+               buffers_backend_fsync,
+               buffers_alloc
+        FROM pg_stat_bgwriter
+      170000: >
+        SELECT cp.num_timed AS checkpoints_timed,
+               cp.num_requested AS checkpoints_req,
+               cp.write_time AS checkpoint_write_time,
+               cp.sync_time AS checkpoint_sync_time,
+               cp.buffers_written AS buffers_checkpoint,
+               bg.buffers_clean AS buffers_clean,
+               bg.maxwritten_clean AS maxwritten_clean,
+               NULL AS buffers_backend,
+               NULL AS buffers_backend_fsync,
+               bg.buffers_alloc AS buffers_alloc
+        FROM pg_stat_bgwriter bg
+        CROSS JOIN pg_stat_checkpointer cp
+
+
+  io_per_backend:
+    type: set
+    query:
+      160000: >
+        SELECT backend_type,
+               COALESCE(SUM(reads * op_bytes), 0)::bigint AS reads,
+               COALESCE(SUM(read_time), 0)::bigint AS read_time,
+               COALESCE(SUM(writes * op_bytes), 0)::bigint AS writes,
+               COALESCE(SUM(write_time), 0)::bigint AS write_time,
+               COALESCE(SUM(writebacks * op_bytes), 0)::bigint AS writebacks,
+               COALESCE(SUM(writeback_time), 0)::bigint AS writeback_time,
+               COALESCE(SUM(extends * op_bytes), 0)::bigint AS extends,
+               COALESCE(SUM(extend_time), 0)::bigint AS extend_time,
+               COALESCE(SUM(op_bytes), 0)::bigint AS op_bytes,
+               COALESCE(SUM(hits), 0)::bigint AS hits,
+               COALESCE(SUM(evictions), 0)::bigint AS evictions,
+               COALESCE(SUM(reuses), 0)::bigint AS reuses,
+               COALESCE(SUM(fsyncs), 0)::bigint AS fsyncs,
+               COALESCE(SUM(fsync_time), 0)::bigint AS fsync_time
+        FROM pg_stat_io
+        GROUP BY backend_type
+
+  ##
   # Per-database metrics
+  ##
   db_stats:
     type: row
     query:
-      0: SELECT numbackends, xact_commit, xact_rollback, blks_read, blks_hit, tup_returned, tup_fetched, tup_inserted, tup_updated, tup_deleted, conflicts, temp_files, temp_bytes, deadlocks, blk_read_time, blk_write_time, extract('epoch' from stats_reset)::float FROM pg_stat_database WHERE datname = %(dbname)s
-      140000: SELECT numbackends, xact_commit, xact_rollback, blks_read, blks_hit, tup_returned, tup_fetched, tup_inserted, tup_updated, tup_deleted, conflicts, temp_files, temp_bytes, deadlocks, COALESCE(checksum_failures, 0) AS checksum_failures, blk_read_time, blk_write_time, session_time, active_time, idle_in_transaction_time, sessions, sessions_abandoned, sessions_fatal, sessions_killed, extract('epoch' from stats_reset)::float FROM pg_stat_database WHERE datname = %(dbname)s
+      0: >
+        SELECT numbackends,
+               xact_commit,
+               xact_rollback,
+               blks_read,
+               blks_hit,
+               tup_returned,
+               tup_fetched,
+               tup_inserted,
+               tup_updated,
+               tup_deleted,
+               conflicts,
+               temp_files,
+               temp_bytes,
+               deadlocks,
+               NULL AS checksum_failures,
+               blk_read_time,
+               blk_write_time,
+               NULL AS session_time,
+               NULL AS active_time,
+               NULL AS idle_in_transaction_time,
+               NULL AS sessions,
+               NULL AS sessions_abandoned,
+               NULL AS sessions_fatal,
+               NULL AS sessions_killed,
+               extract('epoch' from stats_reset)::float AS stats_reset
+        FROM pg_stat_database WHERE datname = %(dbname)s
+      140000: >
+        SELECT numbackends,
+               xact_commit,
+               xact_rollback,
+               blks_read,
+               blks_hit,
+               tup_returned,
+               tup_fetched,
+               tup_inserted,
+               tup_updated,
+               tup_deleted,
+               conflicts,
+               temp_files,
+               temp_bytes,
+               deadlocks,
+               COALESCE(checksum_failures, 0) AS checksum_failures,
+               blk_read_time,
+               blk_write_time,
+               session_time,
+               active_time,
+               idle_in_transaction_time,
+               sessions,
+               sessions_abandoned,
+               sessions_fatal,
+               sessions_killed,
+               extract('epoch' from stats_reset)::float AS stats_reset
+        FROM pg_stat_database WHERE datname = %(dbname)s
+    test_args:
+      dbname: postgres
 
+  hit_ratios:
+    type: row
+    query:
+      0: >
+        SELECT sum(heap_blks_read)::float / NULLIF(sum(heap_blks_read + heap_blks_hit), 0) AS avg_heap_hit_ratio,
+               sum(idx_blks_hit)::float / NULLIF(sum(idx_blks_read + idx_blks_hit), 0) AS avg_idx_hit_ratio,
+               sum(toast_blks_hit)::float / NULLIF(sum(toast_blks_read + toast_blks_hit), 0) AS avg_toast_hit_ratio,
+               sum(tidx_blks_hit)::float / NULLIF(sum(tidx_blks_read + tidx_blks_hit), 0) AS avg_tidx_hit_ratio
+        FROM pg_statio_all_tables
+    test_args:
+      dbname: postgres
+
+  activity:
+    type: set
+    query:
+      0: >
+        SELECT state,
+               count(*) AS backend_count,
+               COALESCE(EXTRACT(EPOCH FROM max(now() - state_change))::float, 0) AS max_state_time
+        FROM pg_stat_activity
+        WHERE datname = %(dbname)s
+        GROUP BY state
+    test_args:
+      dbname: postgres
+
+  sequence_usage:
+    type: value
+    query:
+# 9.2 lacks lateral joins, the pg_sequence_last_value function, and the pg_sequences view
+#      0: >
+#        SELECT COALESCE(MAX(pg_sequence_last_value(c.oid)::float / (pg_sequence_parameters(oid)).maximum_value), 0) AS max_usage
+#        FROM pg_class c
+#        WHERE c.relkind = 'S'
+# 9.3 - 9.6 lacks the pg_sequence_last_value function, and pg_sequences view
+#      90300: >
+#        SELECT COALESCE(MAX(pg_sequence_last_value(c.oid)::float / s.maximum_value), 0) AS max_usage
+#        FROM pg_class c
+#        CROSS JOIN LATERAL pg_sequence_parameters(c.oid) AS s
+#        WHERE c.relkind = 'S'
+      100000: >
+        SELECT COALESCE(MAX(pg_sequence_last_value(c.oid)::float / sp.maximum_value), 0) AS max_usage
+        FROM pg_class c
+        CROSS JOIN LATERAL pg_sequence_parameters(c.oid) AS sp
+        WHERE c.relkind = 'S'
+    test_args:
+      dbname: postgres
+
+  ##
+  # Per-replication metrics
+  ##
+  rep_stats:
+    type: row
+    query:
+      90400: >
+        SELECT pid, usename,
+               EXTRACT(EPOCH FROM backend_start)::integer AS backend_start,
+               state,
+               pg_xlog_location_diff(pg_current_xlog_location(), sent_location) AS sent_lsn,
+               pg_xlog_location_diff(pg_current_xlog_location(), write_location) AS write_lsn,
+               pg_xlog_location_diff(pg_current_xlog_location(), flush_location) AS flush_lsn,
+               pg_xlog_location_diff(pg_current_xlog_location(), replay_location) AS replay_lsn,
+               NULL AS write_lag,
+               NULL AS flush_lag,
+               NULL AS replay_lag,
+               sync_state
+        FROM pg_stat_replication
+        WHERE client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = %(repid)s
+      100000: >
+        SELECT pid, usename,
+               EXTRACT(EPOCH FROM backend_start)::integer AS backend_start,
+               state,
+               pg_wal_lsn_diff(pg_current_wal_lsn(), sent_lsn) AS sent_lsn,
+               pg_wal_lsn_diff(pg_current_wal_lsn(), write_lsn) AS write_lsn,
+               pg_wal_lsn_diff(pg_current_wal_lsn(), flush_lsn) AS flush_lsn,
+               pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) AS replay_lsn,
+               COALESCE(EXTRACT(EPOCH FROM write_lag), 0)::integer AS write_lag,
+               COALESCE(EXTRACT(EPOCH FROM flush_lag), 0)::integer AS flush_lag,
+               COALESCE(EXTRACT(EPOCH FROM replay_lag), 0)::integer AS replay_lag,
+               sync_state
+        FROM pg_stat_replication WHERE client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = %(repid)s
+    test_args:
+      repid: 127.0.0.1_test_rep
+
+
+  ##
+  # Per-slot metrics
+  ##
+  slot_stats:
+    type: row
+    query:
+      90400: >
+        SELECT NULL as active_pid,
+               xmin,
+               pg_xlog_location_diff(pg_current_xlog_location(), restart_lsn) AS restart_bytes,
+               NULL AS confirmed_flush_bytes
+        FROM pg_replication_slots WHERE slot_name = %(slot)s
+      90600: >
+        SELECT active_pid,
+               xmin,
+               pg_xlog_location_diff(pg_current_xlog_location(), restart_lsn) AS restart_bytes,
+               pg_xlog_location_diff(pg_current_xlog_location(), confirmed_flush_lsn) AS confirmed_flush_bytes
+        FROM pg_replication_slots WHERE slot_name = %(slot)s
+      100000: >
+        SELECT active_pid,
+               xmin,
+               pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) AS restart_bytes,
+               pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn) AS confirmed_flush_bytes
+        FROM pg_replication_slots WHERE slot_name = %(slot)s
+    test_args:
+      slot: test_slot
+
+  ##
   # Debugging
+  ##
   ntables:
     type: value
     query:
       0: SELECT count(*) AS ntables FROM pg_stat_user_tables
-
-  # Per-replication metrics
-  rep_stats:
-    type: row
-    query:
-      0: SELECT * FROM pg_stat_database WHERE client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = '{repid}'
-
-  # Debugging
-  sleep:
-    type: value
-    query:
-      0: SELECT now(), pg_sleep(5);
-
-  # Per-slot metrics
-  slot_stats:
-    type: row
-    query:
-      0: SELECT active_pid, xmin, pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) AS restart_bytes, pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn) AS confirmed_flush_bytes FROM pg_replication_slots WHERE slot_name = '{slot}'

sample-config/pgmon.yml

@@ -1,3 +1,6 @@
+# The address the agent binds to
+#address: 127.0.0.1
+
 # The port the agent listens on for requests
 #port: 5400
 
@@ -26,6 +29,9 @@
 # Default database to connect to when none is specified for a metric
 #dbname: 'postgres'
 
+# SSL connection mode
+#ssl_mode: require
+
 # Timeout for getting a connection slot from a pool
 #pool_slot_timeout: 5
 

src/pgmon.py

@@ -4,6 +4,7 @@ import yaml
 import json
 import time
 import os
+import sys
 
 import argparse
 import logging
@@ -23,7 +24,10 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
 from http.server import ThreadingHTTPServer
 from urllib.parse import urlparse, parse_qs
 
-VERSION = "1.0.1"
+import requests
+import re
+
+VERSION = "1.0.3"
 
 # Configuration
 config = {}
@@ -42,6 +46,12 @@ cluster_version = None
 cluster_version_next_check = None
 cluster_version_lock = Lock()
 
+# PostgreSQL latest version information
+latest_version = None
+latest_version_next_check = None
+latest_version_lock = Lock()
+release_supported = None
+
 # Running state (used to gracefully shut down)
 running = True
 
@@ -74,12 +84,22 @@ class UnhappyDBError(Exception):
     pass
 
 
+class UnknownMetricError(Exception):
+    pass
+
+
 class MetricVersionError(Exception):
     pass
 
 
+class LatestVersionCheckError(Exception):
+    pass
+
+
 # Default config settings
 default_config = {
+    # The address the agent binds to
+    "address": "127.0.0.1",
     # The port the agent listens on for requests
     "port": 5400,
     # Min PostgreSQL connection pool size (per database)
@@ -98,6 +118,8 @@ default_config = {
     "dbport": 5432,
     # Default database to connect to when none is specified for a metric
     "dbname": "postgres",
+    # SSL connection mode
+    "ssl_mode": "require",
     # Timeout for getting a connection slot from a pool
     "pool_slot_timeout": 5,
     # PostgreSQL connection timeout (seconds)
@@ -107,6 +129,8 @@ default_config = {
     "reconnect_cooldown": 30,
     # How often to check the version of PostgreSQL (seconds)
     "version_check_period": 300,
+    # How often to check the latest supported version of PostgreSQL (seconds)
+    "latest_version_check_period": 86400,
     # Metrics
     "metrics": {},
 }
@@ -320,6 +344,7 @@ def get_pool(dbname):
             # lock
             if dbname not in connections:
                 log.info("Creating connection pool for: {}".format(dbname))
+                # Actually create the connection pool
                 connections[dbname] = ConnectionPool(
                     dbname,
                     int(config["min_pool_size"]),
@@ -329,7 +354,7 @@ def get_pool(dbname):
                     port=config["dbport"],
                     user=config["dbuser"],
                     connect_timeout=int(config["connect_timeout"]),
-                    sslmode="require",
+                    sslmode=config["ssl_mode"],
                 )
             # Clear the unhappy indicator if present
             unhappy_cooldown.pop(dbname, None)
@@ -377,10 +402,16 @@ def run_query_no_retry(pool, return_type, query, args):
                 res = curs.fetchall()
 
                 if return_type == "value":
+                    if len(res) == 0:
+                        return ""
                     return str(list(res[0].values())[0])
                 elif return_type == "row":
+                    if len(res) == 0:
+                        return "[]"
                     return json.dumps(res[0])
                 elif return_type == "column":
+                    if len(res) == 0:
+                        return "[]"
                     return json.dumps([list(r.values())[0] for r in res])
                 elif return_type == "set":
                     return json.dumps(res)
@@ -388,7 +419,7 @@ def run_query_no_retry(pool, return_type, query, args):
             dbname = pool.name
             if dbname in unhappy_cooldown:
                 raise UnhappyDBError()
-            elif conn.broken:
+            elif conn.closed != 0:
                 raise DisconnectedError()
             else:
                 raise
@@ -466,6 +497,191 @@ def get_cluster_version():
     return cluster_version
 
 
+def version_num_to_release(version_num):
+    """
+    Extract the revease from a version_num.
+
+    In other words, this converts things like:
+        90603  => 9.6
+        130010 => 13
+    """
+    if version_num // 10000 < 10:
+        return version_num // 10000 + (version_num % 10000 // 100 / 10)
+    else:
+        return version_num // 10000
+
+
+def parse_version_rss(raw_rss, release):
+    """
+    Parse the raw RSS from the versions.rss feed to extract the latest version of
+    PostgreSQL that's availabe for the cluster being monitored.
+
+    This sets these global variables:
+        latest_version
+        release_supported
+
+    It is expected that the caller already holds the latest_version_lock lock.
+
+    params:
+        raw_rss: The raw rss text from versions.rss
+        release: The PostgreSQL release we care about (ex: 9.2, 14)
+    """
+    global latest_version
+    global release_supported
+
+    # Regular expressions for parsing the RSS document
+    version_line = re.compile(
+        r".*?([0-9][0-9.]+) is the latest release in the {} series.*".format(release)
+    )
+    unsupported_line = re.compile(r"^This version is unsupported")
+
+    # Loop through the RSS until we find the current release
+    release_found = False
+    for line in raw_rss.splitlines():
+        m = version_line.match(line)
+        if m:
+            # Note that we found the version we were looking for
+            release_found = True
+
+            # Convert the version to version_num format
+            version = m.group(1)
+            parts = list(map(int, version.split(".")))
+            if parts[0] < 10:
+                latest_version = int(
+                    "{}{:02}{:02}".format(parts[0], parts[1], parts[2])
+                )
+            else:
+                latest_version = int("{}00{:02}".format(parts[0], parts[1]))
+        elif release_found:
+            # The next line after the version tells if the version is supported
+            if unsupported_line.match(line):
+                release_supported = False
+            else:
+                release_supported = True
+            break
+
+    # Make sure we actually found it
+    if not release_found:
+        raise LatestVersionCheckError("Current release ({}) not found".format(release))
+
+    log.info(
+        "Got latest PostgreSQL version: {} supported={}".format(
+            latest_version, release_supported
+        )
+    )
+    log.debug(
+        "Next latest PostgreSQL version check will be after: {}".format(
+            latest_version_next_check
+        )
+    )
+
+
+def get_latest_version():
+    """
+    Get the latest supported version of the major PostgreSQL release running on the server being monitored.
+    """
+
+    global latest_version_next_check
+
+    # If we don't know the latest version or it's past the recheck time, get the
+    # version from the PostgreSQL RSS feed.  Only one thread needs to do this, so
+    # they all try to grab the lock, and then make sure nobody else beat them to it.
+    if (
+        latest_version is None
+        or latest_version_next_check is None
+        or latest_version_next_check < datetime.now()
+    ):
+        # Note: we get the cluster version here before grabbing the latest_version_lock
+        #       lock so it's not held while trying to talk with the DB.
+        release = version_num_to_release(get_cluster_version())
+
+        with latest_version_lock:
+            # Only check if nobody already got the version before us
+            if (
+                latest_version is None
+                or latest_version_next_check is None
+                or latest_version_next_check < datetime.now()
+            ):
+                log.info("Checking latest PostgreSQL version")
+                latest_version_next_check = datetime.now() + timedelta(
+                    seconds=int(config["latest_version_check_period"])
+                )
+
+                # Grab the RSS feed
+                raw_rss = requests.get("https://www.postgresql.org/versions.rss")
+                if raw_rss.status_code != 200:
+                    raise LatestVersionCheckError("code={}".format(r.status_code))
+
+                # Parse the RSS body and set global variables
+                parse_version_rss(raw_rss.text, release)
+
+    return latest_version
+
+
+def sample_metric(dbname, metric_name, args, retry=True):
+    """
+    Run the appropriate query for the named metric against the specified database
+    """
+    # Get the metric definition
+    try:
+        metric = config["metrics"][metric_name]
+    except KeyError:
+        raise UnknownMetricError("Unknown metric: {}".format(metric_name))
+
+    # Get the connection pool for the database, or create one if it doesn't
+    # already exist.
+    pool = get_pool(dbname)
+
+    # Identify the PostgreSQL version
+    version = get_cluster_version()
+
+    # Get the query version
+    query = get_query(metric, version)
+
+    # Execute the quert
+    if retry:
+        return run_query(pool, metric["type"], query, args)
+    else:
+        return run_query_no_retry(pool, metric["type"], query, args)
+
+
+def test_queries():
+    """
+    Run all of the metric queries against a database and check the results
+    """
+    # We just use the default db for tests
+    dbname = config["dbname"]
+    # Loop through all defined metrics.
+    for name, metric in config["metrics"].items():
+        # If the metric has arguments to use while testing, grab those
+        args = metric.get("test_args", {})
+        print("Testing {} [{}]".format(name, ", ".join(["{}={}".format(key, value) for key, value in args.items()])))
+        # When testing against a docker container, we may end up connecting
+        # before the service is truly up (it restarts during the initialization
+        # phase).  To cope with this, we'll allow a few connection failures.
+        tries = 5
+        while True:
+            # Run the query without the ability to retry
+            try:
+                res = sample_metric(dbname, name, args, retry=False)
+                break
+            except MetricVersionError:
+                res = "Unsupported for this version"
+                break
+            except psycopg2.OperationalError as e:
+                print("Error encountered, {} tries left: {}".format(tries, e))
+                if tries <= 0:
+                    raise
+                time.sleep(1)
+            tries -= 1
+        # Compare the result to the provided sample results
+        # TODO
+        print("{} -> {}".format(name, res))
+    # Return the number of errors
+    # TODO
+    return 0
+
+
 class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
     """
     This is our request handling server.  It is responsible for listening for
@@ -494,71 +710,58 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
         """
         # Parse the URL
         parsed_path = urlparse(self.path)
-        name = parsed_path.path.strip("/")
+        metric_name = parsed_path.path.strip("/")
         parsed_query = parse_qs(parsed_path.query)
 
-        if name == "agent_version":
+        if metric_name == "agent_version":
             self._reply(200, VERSION)
             return
+        elif metric_name == "latest_version_info":
+            try:
+                get_latest_version()
+                self._reply(
+                    200,
+                    json.dumps(
+                        {
+                            "latest": latest_version,
+                            "supported": 1 if release_supported else 0,
+                        }
+                    ),
+                )
+            except LatestVersionCheckError as e:
+                log.error("Failed to retrieve latest version information: {}".format(e))
+                self._reply(503, "Failed to retrieve latest version info")
+            return
 
         # Note: parse_qs returns the values as a list.  Since we always expect
         #       single values, just grab the first from each.
         args = {key: values[0] for key, values in parsed_query.items()}
 
-        # Get the metric definition
-        try:
-            metric = config["metrics"][name]
-        except KeyError:
-            log.error("Unknown metric: {}".format(name))
-            self._reply(404, "Unknown metric")
-            return
-
         # Get the dbname.  If none was provided, use the default from the
         # config.
         dbname = args.get("dbname", config["dbname"])
 
-        # Get the connection pool for the database, or create one if it doesn't
-        # already exist.
+        # Sample the metric
         try:
-            pool = get_pool(dbname)
-        except UnhappyDBError:
+            self._reply(200, sample_metric(dbname, metric_name, args))
+            return
+        except UnknownMetricError as e:
+            log.error("Unknown metric: {}".format(metric_name))
+            self._reply(404, "Unknown metric")
+            return
+        except MetricVersionError as e:
+            log.error(
+                "Failed to find a version of {} for {}".format(metric_name, version)
+            )
+            self._reply(404, "Unsupported version")
+            return
+        except UnhappyDBError as e:
             log.info("Database {} is unhappy, please be patient".format(dbname))
             self._reply(503, "Database unavailable")
             return
-
-        # Identify the PostgreSQL version
-        try:
-            version = get_cluster_version()
-        except UnhappyDBError:
-            return
         except Exception as e:
-            if dbname in unhappy_cooldown:
-                log.info("Database {} is unhappy, please be patient".format(dbname))
-                self._reply(503, "Database unavailable")
-            else:
-                log.error("Failed to get PostgreSQL version: {}".format(e))
-                self._reply(500, "Error getting DB version")
-            return
-
-        # Get the query version
-        try:
-            query = get_query(metric, version)
-        except KeyError:
-            log.error("Failed to find a version of {} for {}".format(name, version))
-            self._reply(404, "Unsupported version")
-            return
-
-        # Execute the quert
-        try:
-            self._reply(200, run_query(pool, metric["type"], query, args))
-            return
-        except Exception as e:
-            if dbname in unhappy_cooldown:
-                log.info("Database {} is unhappy, please be patient".format(dbname))
-                self._reply(503, "Database unavailable")
-            else:
-                log.error("Error running query: {}".format(e))
-                self._reply(500, "Error running query")
+            log.error("Error running query: {}".format(e))
+            self._reply(500, "Unexpected error: {}".format(e))
             return
 
     def _reply(self, code, content):
@@ -579,12 +782,17 @@ if __name__ == "__main__":
     )
 
     parser.add_argument(
-        "config_file",
+        "-c",
+        "--config_file",
         default="pgmon.yml",
         nargs="?",
         help="The config file to read (default: %(default)s)",
     )
 
+    parser.add_argument(
+        "-t", "--test", action="store_true", help="Run query tests and exit"
+    )
+
     args = parser.parse_args()
 
     # Set the config file path
@@ -593,8 +801,16 @@ if __name__ == "__main__":
     # Read the config file
     read_config(config_file)
 
+    # Run query tests and exit if test mode is enabled
+    if args.test:
+        errors = test_queries()
+        if errors > 0:
+            sys.exit(1)
+        else:
+            sys.exit(0)
+
     # Set up the http server to receive requests
-    server_address = ("127.0.0.1", config["port"])
+    server_address = (config["address"], config["port"])
     httpd = ThreadingHTTPServer(server_address, SimpleHTTPRequestHandler)
 
     # Set up the signal handler

src/test_pgmon.py

@@ -10,6 +10,95 @@ import pgmon
 # Silence most logging output
 logging.disable(logging.CRITICAL)
 
+versions_rss = """
+<?xml version="1.0" encoding="utf-8"?>
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>PostgreSQL latest versions</title><link>https://www.postgresql.org/</link><description>PostgreSQL latest versions</description><atom:link href="https://www.postgresql.org/versions.rss" rel="self"/><language>en-us</language><lastBuildDate>Thu, 08 May 2025 00:00:00 +0000</lastBuildDate><item><title>17.5
+</title><link>https://www.postgresql.org/docs/17/release-17-5.html</link><description>17.5 is the latest release in the 17 series.
+
+</description><pubDate>Thu, 08 May 2025 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/17/release-17-5.html</guid></item><item><title>16.9
+</title><link>https://www.postgresql.org/docs/16/release-16-9.html</link><description>16.9 is the latest release in the 16 series.
+
+</description><pubDate>Thu, 08 May 2025 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/16/release-16-9.html</guid></item><item><title>15.13
+</title><link>https://www.postgresql.org/docs/15/release-15-13.html</link><description>15.13 is the latest release in the 15 series.
+
+</description><pubDate>Thu, 08 May 2025 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/15/release-15-13.html</guid></item><item><title>14.18
+</title><link>https://www.postgresql.org/docs/14/release-14-18.html</link><description>14.18 is the latest release in the 14 series.
+
+</description><pubDate>Thu, 08 May 2025 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/14/release-14-18.html</guid></item><item><title>13.21
+</title><link>https://www.postgresql.org/docs/13/release-13-21.html</link><description>13.21 is the latest release in the 13 series.
+
+</description><pubDate>Thu, 08 May 2025 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/13/release-13-21.html</guid></item><item><title>12.22
+</title><link>https://www.postgresql.org/docs/12/release-12-22.html</link><description>12.22 is the latest release in the 12 series.
+This version is unsupported!
+</description><pubDate>Thu, 21 Nov 2024 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/12/release-12-22.html</guid></item><item><title>11.22
+</title><link>https://www.postgresql.org/docs/11/release-11-22.html</link><description>11.22 is the latest release in the 11 series.
+This version is unsupported!
+</description><pubDate>Thu, 09 Nov 2023 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/11/release-11-22.html</guid></item><item><title>10.23
+</title><link>https://www.postgresql.org/docs/10/release-10-23.html</link><description>10.23 is the latest release in the 10 series.
+This version is unsupported!
+</description><pubDate>Thu, 10 Nov 2022 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/10/release-10-23.html</guid></item><item><title>9.6.24
+</title><link>https://www.postgresql.org/docs/9.6/release-9-6-24.html</link><description>9.6.24 is the latest release in the 9.6 series.
+This version is unsupported!
+</description><pubDate>Thu, 11 Nov 2021 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.6/release-9-6-24.html</guid></item><item><title>9.5.25
+</title><link>https://www.postgresql.org/docs/9.5/release-9-5-25.html</link><description>9.5.25 is the latest release in the 9.5 series.
+This version is unsupported!
+</description><pubDate>Thu, 11 Feb 2021 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.5/release-9-5-25.html</guid></item><item><title>9.4.26
+</title><link>https://www.postgresql.org/docs/9.4/release-9-4-26.html</link><description>9.4.26 is the latest release in the 9.4 series.
+This version is unsupported!
+</description><pubDate>Thu, 13 Feb 2020 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.4/release-9-4-26.html</guid></item><item><title>9.3.25
+</title><link>https://www.postgresql.org/docs/9.3/release-9-3-25.html</link><description>9.3.25 is the latest release in the 9.3 series.
+This version is unsupported!
+</description><pubDate>Thu, 08 Nov 2018 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.3/release-9-3-25.html</guid></item><item><title>9.2.24
+</title><link>https://www.postgresql.org/docs/9.2/release-9-2-24.html</link><description>9.2.24 is the latest release in the 9.2 series.
+This version is unsupported!
+</description><pubDate>Thu, 09 Nov 2017 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.2/release-9-2-24.html</guid></item><item><title>9.1.24
+</title><link>https://www.postgresql.org/docs/9.1/release-9-1-24.html</link><description>9.1.24 is the latest release in the 9.1 series.
+This version is unsupported!
+</description><pubDate>Thu, 27 Oct 2016 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.1/release-9-1-24.html</guid></item><item><title>9.0.23
+</title><link>https://www.postgresql.org/docs/9.0/release-9-0-23.html</link><description>9.0.23 is the latest release in the 9.0 series.
+This version is unsupported!
+</description><pubDate>Thu, 08 Oct 2015 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/9.0/release-9-0-23.html</guid></item><item><title>8.4.22
+</title><link>https://www.postgresql.org/docs/8.4/release-8-4-22.html</link><description>8.4.22 is the latest release in the 8.4 series.
+This version is unsupported!
+</description><pubDate>Thu, 24 Jul 2014 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/8.4/release-8-4-22.html</guid></item><item><title>8.3.23
+</title><link>https://www.postgresql.org/docs/8.3/release-8-3-23.html</link><description>8.3.23 is the latest release in the 8.3 series.
+This version is unsupported!
+</description><pubDate>Thu, 07 Feb 2013 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/8.3/release-8-3-23.html</guid></item><item><title>8.2.23
+</title><link>https://www.postgresql.org/docs/8.2/release-8-2-23.html</link><description>8.2.23 is the latest release in the 8.2 series.
+This version is unsupported!
+</description><pubDate>Mon, 05 Dec 2011 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/8.2/release-8-2-23.html</guid></item><item><title>8.1.23
+</title><link>https://www.postgresql.org/docs/8.1/release.html</link><description>8.1.23 is the latest release in the 8.1 series.
+This version is unsupported!
+</description><pubDate>Thu, 16 Dec 2010 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/8.1/release.html</guid></item><item><title>8.0.26
+</title><link>https://www.postgresql.org/docs/8.0/release.html</link><description>8.0.26 is the latest release in the 8.0 series.
+This version is unsupported!
+</description><pubDate>Mon, 04 Oct 2010 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/8.0/release.html</guid></item><item><title>7.4.30
+</title><link>https://www.postgresql.org/docs/7.4/release.html</link><description>7.4.30 is the latest release in the 7.4 series.
+This version is unsupported!
+</description><pubDate>Mon, 04 Oct 2010 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/7.4/release.html</guid></item><item><title>7.3.21
+</title><link>https://www.postgresql.org/docs/7.3/release.html</link><description>7.3.21 is the latest release in the 7.3 series.
+This version is unsupported!
+</description><pubDate>Mon, 07 Jan 2008 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/7.3/release.html</guid></item><item><title>7.2.8
+</title><link>https://www.postgresql.org/docs/7.2/release.html</link><description>7.2.8 is the latest release in the 7.2 series.
+This version is unsupported!
+</description><pubDate>Mon, 09 May 2005 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/7.2/release.html</guid></item><item><title>7.1.3
+</title><link>https://www.postgresql.org/docs/7.1/release.html</link><description>7.1.3 is the latest release in the 7.1 series.
+This version is unsupported!
+</description><pubDate>Fri, 17 Aug 2001 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/7.1/release.html</guid></item><item><title>7.0.3
+</title><link>https://www.postgresql.org/docs/7.0/release.htm</link><description>7.0.3 is the latest release in the 7.0 series.
+This version is unsupported!
+</description><pubDate>Sun, 12 Nov 2000 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/7.0/release.htm</guid></item><item><title>6.5.3
+</title><link>https://www.postgresql.org/docs/6.5/release.htm</link><description>6.5.3 is the latest release in the 6.5 series.
+This version is unsupported!
+</description><pubDate>Thu, 04 Nov 1999 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/6.5/release.htm</guid></item><item><title>6.4.2
+</title><link>https://www.postgresql.org/docs/6.4/release.htm</link><description>6.4.2 is the latest release in the 6.4 series.
+This version is unsupported!
+</description><pubDate>Sun, 03 Jan 1999 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/6.4/release.htm</guid></item><item><title>6.3.2
+</title><link>https://www.postgresql.org/docs/6.3/c2701.htm</link><description>6.3.2 is the latest release in the 6.3 series.
+This version is unsupported!
+</description><pubDate>Mon, 23 Feb 1998 00:00:00 +0000</pubDate><guid>https://www.postgresql.org/docs/6.3/c2701.htm</guid></item></channel></rss>
+"""
+
 
 class TestPgmonMethods(unittest.TestCase):
     ##
@@ -172,13 +261,16 @@ class TestPgmonMethods(unittest.TestCase):
     def test_get_query__versions(self):
         metric = {"type": "value", "query": {0: "DEFAULT", 110000: "NEW"}}
 
-        # Test getting the default version of a query with no lower bound and a newer version
+        # Test getting the default version of a query with no lower bound and a newer
+        # version
         self.assertEqual(pgmon.get_query(metric, 100000), "DEFAULT")
 
-        # Test getting the newer version of a query with no lower bound and a newer version for the newer version
+        # Test getting the newer version of a query with no lower bound and a newer
+        # version for the newer version
         self.assertEqual(pgmon.get_query(metric, 110000), "NEW")
 
-        # Test getting the newer version of a query with no lower bound and a newer version for an even newer version
+        # Test getting the newer version of a query with no lower bound and a newer
+        # version for an even newer version
         self.assertEqual(pgmon.get_query(metric, 160000), "NEW")
 
         # Test getting a version in bwtween two other versions
@@ -272,7 +364,8 @@ metrics:
     def test_read_config__include(self):
         pgmon.config = {}
 
-        # Test reading a config that includes other files (absolute and relative paths, multiple levels)
+        # Test reading a config that includes other files (absolute and relative paths,
+        # multiple levels)
         with tempfile.TemporaryDirectory() as tmpdirname:
             with open(f"{tmpdirname}/config.yml", "w") as f:
                 f.write(
@@ -661,3 +754,38 @@ metrics:
             )
         self.assertEqual(pgmon.config["dbuser"], "postgres")
         self.assertEqual(pgmon.config["metrics"]["test1"]["query"][0], "TEST1")
+
+    def test_version_num_to_release__valid(self):
+        self.assertEqual(pgmon.version_num_to_release(90602), 9.6)
+        self.assertEqual(pgmon.version_num_to_release(130002), 13)
+
+    def test_parse_version_rss__simple(self):
+        pgmon.parse_version_rss(versions_rss, 13)
+        self.assertEqual(pgmon.latest_version, 130021)
+        self.assertTrue(pgmon.release_supported)
+
+        pgmon.parse_version_rss(versions_rss, 9.6)
+        self.assertEqual(pgmon.latest_version, 90624)
+        self.assertFalse(pgmon.release_supported)
+
+    def test_parse_version_rss__missing(self):
+        # Test asking about versions that don't exist
+        self.assertRaises(
+            pgmon.LatestVersionCheckError, pgmon.parse_version_rss, versions_rss, 9.7
+        )
+
+        self.assertRaises(
+            pgmon.LatestVersionCheckError, pgmon.parse_version_rss, versions_rss, 99
+        )
+
+    def test_get_latest_version(self):
+        # Define a cluster version here so the test doesn't need a database
+        pgmon.cluster_version_next_check = datetime.now() + timedelta(hours=1)
+        pgmon.cluster_version = 90623
+
+        # Set up a default config
+        pgmon.update_deep(pgmon.config, pgmon.default_config)
+
+        # Make sure we can pull the RSS file (we assume the 9.6 series won't be getting
+        # any more updates)
+        self.assertEqual(pgmon.get_latest_version(), 90624)

systemd/pgmon@.service

@@ -7,7 +7,7 @@ After=network.target
 [Service]
 EnvironmentFile=/etc/pgmon/%i-service.conf
 User=${SERVICE_USER:-postgres}
-ExecStart=/usr/local/bin/pgmon /etc/pgmon/%i.yml
+ExecStart=/usr/bin/pgmon -c /etc/pgmon/%i.yml
 ExecReload=kill -HUP $MAINPID
 Restart=on-failure
 Type=exec

tests/Dockerfile

@@ -0,0 +1,23 @@
+FROM alpine:3.21
+
+RUN apk update && \
+    apk add py3-psycopg2 \
+            py3-requests \
+            py3-yaml \
+            tini
+
+WORKDIR /app
+
+COPY src/pgmon.py /app/
+
+COPY sample-config/pgmon-metrics.yml /app/
+
+COPY tests/test-config.yml /app/
+
+COPY --chmod=0600 --chown=postgres:postgres tests/pgpass /root/.pgpass
+
+ENTRYPOINT ["tini", "--"]
+
+EXPOSE 5400
+
+CMD ["/app/pgmon.py", "-c", "/app/test-config.yml", "--test"]

tests/docker-compose.yml

@@ -0,0 +1,32 @@
+---
+
+services:
+  agent:
+    image: pgmon
+    build:
+      context: ..
+      dockerfile: tests/Dockerfile
+    ports:
+      - :5400
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    image: "postgres:${PGTAG:-17-bookworm}"
+    ports:
+      - :5432
+    environment:
+      POSTGRES_PASSWORD: secret
+    healthcheck:
+      #test: [ "CMD", "pg_isready", "-U", "postgres" ]
+      test: [ "CMD-SHELL", "pg_controldata /var/lib/postgresql/data/ | grep -q 'in production'" ]
+      interval: 5s
+      timeout: 2s
+      retries: 40
+    command: >
+      postgres -c ssl=on
+               -c ssl_cert_file='/etc/ssl/certs/ssl-cert-snakeoil.pem'
+               -c ssl_key_file='/etc/ssl/private/ssl-cert-snakeoil.key'
+               -c listen_addresses='*'
+

tests/pgpass

@@ -0,0 +1 @@
+db:5432:*:postgres:secret

tests/run-tests.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Versions to test
+versions=( $@ )
+
+# If we weren't given any versions, test them all
+if [ ${#versions[@]} -eq 0 ]
+then
+    versions=( 9.2 9.4 9.6 10 11 12 13 14 15 16 17 )
+fi
+
+# Image tags to use
+declare -A images=()
+images["9.2"]='9.2'
+images["9.3"]='9.3'
+images["9.4"]='9.4'
+images["9.5"]='9.5'
+images["9.6"]='9.6-bullseye'
+images["10"]='10-bullseye'
+images["11"]='11-bookworm'
+images["12"]='12-bookworm'
+images["13"]='13-bookworm'
+images["14"]='14-bookworm'
+images["15"]='15-bookworm'
+images["16"]='16-bookworm'
+images["17"]='17-bookworm'
+
+declare -A results=()
+
+# Make sure everything's down to start with
+docker compose down
+
+# Make sure our agent container is up to date
+docker compose build agent
+
+for version in "${versions[@]}"
+do
+    echo
+    echo "Testing: PostgreSQL ${version}"
+
+    # Specify the version we're testing against
+    export PGTAG="${images["$version"]}"
+
+    # Start the containers
+    docker compose up --exit-code-from=agent agent
+    rc=$?
+
+    results["$version"]=$rc
+
+    # Destroy the containers
+    docker compose down
+done
+
+echo
+echo
+for v in "${versions[@]}"
+do
+    case "${results["$v"]}" in
+        0) msg="OK" ;;
+        1) msg="Query failure detected" ;;
+        18) msg="Docker image error: 18" ;;
+        *) msg="Unexpected error: ${results["$v"]}" ;;
+    esac
+    echo "$v -> $msg"
+done

tests/test-config.yml

@@ -0,0 +1,17 @@
+---
+
+# Bind to all interfaces so we can submit requests from outside the test container
+address: 0.0.0.0
+
+# We always just connect to the db container
+dbhost: db
+dbport: 5432
+dbuser: postgres
+
+# The SSL cipher parameters are too old in the 9.2 container, so we allow the tests
+# to be run without encryption
+ssl_mode: prefer
+
+# Pull in the standard metrics
+include:
+  - pgmon-metrics.yml

zabbix_templates/pgmon_templates.yaml

@@ -11,6 +11,77 @@ zabbix_export:
       groups:
         - name: 'My Templates'
       items:
+        - uuid: cc4a50ca681b4c4fb8c1c98f4bc81c96
+          name: 'Max Frozen MXID Age'
+          type: DEPENDENT
+          key: pgmon.age.mxid
+          delay: '0'
+          history: 90d
+          description: 'Maximum age of any frozen MXID in any database'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - $.mxid_age
+            - type: MATCHES_REGEX
+              parameters:
+                - '^[0-9]+$'
+          master_item:
+            key: 'pgmon[max_frozen_age]'
+          tags:
+            - tag: Application
+              value: PostgreSQL
+        - uuid: 2fa86d8c4a3a481e856a9414c5c70216
+          name: 'Max Frozen XID Age'
+          type: DEPENDENT
+          key: pgmon.age.xid
+          delay: '0'
+          history: 90d
+          description: 'Maximum age of any frozen XID in any database'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - $.xid_age
+            - type: MATCHES_REGEX
+              parameters:
+                - '^[0-9]+$'
+          master_item:
+            key: 'pgmon[max_frozen_age]'
+          tags:
+            - tag: Application
+              value: PostgreSQL
+        - uuid: de1fa757395440118026f4c7a7c4ebbe
+          name: 'PostgreSQL latest supported version'
+          type: DEPENDENT
+          key: pgmon.release.latest
+          delay: '0'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - $.latest
+          master_item:
+            key: 'pgmon[latest_version_info]'
+          tags:
+            - tag: Application
+              value: PostgreSQL
+        - uuid: e1a003317bc043908ee0803704a6e1d0
+          name: 'PostgreSQL release support status'
+          type: DEPENDENT
+          key: pgmon.release.supported
+          delay: '0'
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - $.supported
+          master_item:
+            key: 'pgmon[latest_version_info]'
+          tags:
+            - tag: Application
+              value: PostgreSQL
+          triggers:
+            - uuid: 88eac74402d64e7cb853e8cb505b1939
+              expression: 'last(/PostgreSQL by pgmon/pgmon.release.supported)<>1'
+              name: 'PostgreSQL major release is lo longer supported'
+              priority: INFO
         - uuid: 763920af8da84db8a9a2667d9653cb21
           name: 'PostgreSQL Agent Version'
           type: HTTP_AGENT
@@ -24,20 +95,50 @@ zabbix_export:
           tags:
             - tag: Application
               value: PostgreSQL
+        - uuid: 06b1d082ed1e4796bc31cc25f7db6326
+          name: 'PostgreSQL Backend IO Info'
+          type: HTTP_AGENT
+          key: 'pgmon[io_per_backend]'
+          history: '0'
+          value_type: TEXT
+          trends: '0'
+          description: 'Aggregated statistics about I/O activity for different backend types'
+          url: 'http://localhost:{$AGENT_PORT}/io_per_backend'
+          tags:
+            - tag: Application
+              value: PostgreSQL
+            - tag: Type
+              value: Raw
+        - uuid: d890e395fbbc4f2bacbd50e7321fcb9f
+          name: 'PostgreSQL latest version info'
+          type: HTTP_AGENT
+          key: 'pgmon[latest_version_info]'
+          delay: 12h
+          history: '0'
+          value_type: TEXT
+          trends: '0'
+          description: 'Information about the latest PostgreSQL version in the current series'
+          url: 'http://localhost:{$AGENT_PORT}/latest_version_info'
+          tags:
+            - tag: Application
+              value: PostgreSQL
+            - tag: Type
+              value: Raw
         - uuid: 8706eccb7edc4fa394f552fc31f401a9
-          name: 'Max Frozen XID Age'
+          name: 'PostgreSQL ID Age Info'
           type: HTTP_AGENT
           key: 'pgmon[max_frozen_age]'
-          history: 90d
-          description: 'Maximum age of any frozen XID in any database'
-          preprocessing:
-            - type: MATCHES_REGEX
-              parameters:
-                - '^[0-9]+$'
+          delay: 5m
+          history: '0'
+          value_type: TEXT
+          trends: '0'
+          description: 'Maximum age of any frozen XID and MXID in any database'
           url: 'http://localhost:{$AGENT_PORT}/max_frozen_age'
           tags:
             - tag: Application
               value: PostgreSQL
+            - tag: Type
+              value: Raw
         - uuid: ee88f5f4d2384f97946d049af5af4502
           name: 'PostgreSQL version'
           type: HTTP_AGENT
@@ -590,6 +691,100 @@ zabbix_export:
                   value: '{#DBNAME}'
                 - tag: Type
                   value: Calculated
+            - uuid: bb0690e123e94e35a9bbd0f05ee322b4
+              name: 'Database Hit Ratios for {#DBNAME}'
+              type: HTTP_AGENT
+              key: 'pgmon_db_hit_ratios[{#DBNAME}]'
+              history: '0'
+              value_type: TEXT
+              trends: '0'
+              url: 'http://localhost:{$AGENT_PORT}/hit_ratios'
+              query_fields:
+                - name: dbname
+                  value: '{#DBNAME}'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Database
+                  value: '{#DBNAME}'
+                - tag: Type
+                  value: Raw
+            - uuid: e0edbaf6f98249a6b357ade2ab41a76a
+              name: 'Average heap hit ratio on {#DBNAME}'
+              type: DEPENDENT
+              key: 'pgmon_db_hit_ratio[heap,{#DBNAME}]'
+              delay: '0'
+              history: 90d
+              value_type: FLOAT
+              description: 'The cache (shared_buffers) hit ratio for heap entries for the given database'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - $.avg_heap_hit_ratio
+              master_item:
+                key: 'pgmon_db_hit_ratios[{#DBNAME}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Database
+                  value: '{#DBNAME}'
+            - uuid: e4956d433a47455ca0c00d67a0f59355
+              name: 'Average index hit ratio on {#DBNAME}'
+              type: DEPENDENT
+              key: 'pgmon_db_hit_ratio[index,{#DBNAME}]'
+              delay: '0'
+              history: 90d
+              value_type: FLOAT
+              description: 'The cache (shared_buffers) hit ratio for index entries for the given database'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - $.avg_idx_hit_ratio
+              master_item:
+                key: 'pgmon_db_hit_ratios[{#DBNAME}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Database
+                  value: '{#DBNAME}'
+            - uuid: 0335267c14a8410a80a16b38167efcb3
+              name: 'Average toast hit ratio on {#DBNAME}'
+              type: DEPENDENT
+              key: 'pgmon_db_hit_ratio[toast,{#DBNAME}]'
+              delay: '0'
+              history: 90d
+              value_type: FLOAT
+              description: 'The cache (shared_buffers) hit ratio for heap entries for the given database'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - $.avg_toast_hit_ratio
+              master_item:
+                key: 'pgmon_db_hit_ratios[{#DBNAME}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Database
+                  value: '{#DBNAME}'
+            - uuid: 3f7da45e351443528c6c36145d38f9b8
+              name: 'Average toast index hit ratio on {#DBNAME}'
+              type: DEPENDENT
+              key: 'pgmon_db_hit_ratio[toast_index,{#DBNAME}]'
+              delay: '0'
+              history: 90d
+              value_type: FLOAT
+              description: 'The cache (shared_buffers) hit ratio for toast index entries for the given database'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - $.avg_tidx_hit_ratio
+              master_item:
+                key: 'pgmon_db_hit_ratios[{#DBNAME}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Database
+                  value: '{#DBNAME}'
             - uuid: 492b3cac15f348c2b85f97b69c114d1b
               name: 'Database Stats for {#DBNAME}'
               type: HTTP_AGENT
@@ -606,6 +801,8 @@ zabbix_export:
                   value: PostgreSQL
                 - tag: Database
                   value: '{#DBNAME}'
+                - tag: Type
+                  value: Raw
           graph_prototypes:
             - uuid: 1f7de43b77714f819e61c31273712b70
               name: 'DML Totals for {#DBNAME}'
@@ -698,6 +895,596 @@ zabbix_export:
           lld_macro_paths:
             - lld_macro: '{#DBNAME}'
               path: $.dbname
+        - uuid: 6afbe12fb9f54fa2ad29d647429eb16e
+          name: 'Discover I/O Backend Types'
+          type: DEPENDENT
+          key: pgmon_discover_io_backend_types
+          delay: '0'
+          item_prototypes:
+            - uuid: b1ac2e56b30f4812bf33ce973ef16b10
+              name: 'I/O Evictions by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[evictions,{#BACKEND_TYPE}]'
+              delay: '0'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].evictions.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: bde8e6c28e714be58fa963a395e14619
+              name: 'I/O Extends by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[extends,{#BACKEND_TYPE}]'
+              delay: '0'
+              units: B
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].extends.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 5ce7a815c19241a6a398e17539cca1d4
+              name: 'I/O fsyncs by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[fsyncs,{#BACKEND_TYPE}]'
+              delay: '0'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].fsyncs.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 3be8cb08b6bc42f7888d8d7877ab759f
+              name: 'I/O Hits by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[hits,{#BACKEND_TYPE}]'
+              delay: '0'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].hits.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: e28566682196415fb3bdad9d9a294ae9
+              name: 'I/O Reads by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[reads,{#BACKEND_TYPE}]'
+              delay: '0'
+              units: B
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].reads.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: e5e9b21c7f7649b284a8ca5b4502cb96
+              name: 'I/O Reuses by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[reuses,{#BACKEND_TYPE}]'
+              delay: '0'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].reuses.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 6de5bc1a98dc4d8da23ac0a90983c8aa
+              name: 'I/O Writebacks by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[writebacks,{#BACKEND_TYPE}]'
+              delay: '0'
+              units: B
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].writebacks.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 87d09d3891c145a898050d390ea7f16e
+              name: 'I/O Writes by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[writes,{#BACKEND_TYPE}]'
+              delay: '0'
+              units: B
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].writes.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 1587592caf694d91ba9457cafaca7761
+              name: 'I/O Evictions by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[evictions,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              params: 'last(//pgmon_io_backend[evictions,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[evictions,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 637b0601458147b5b72bc40ab22b2117
+              name: 'I/O Evictions Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[evictions,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[evictions,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: c7ed8e1bcb0945ffa2a36b06ea5fe0c5
+              name: 'I/O Extends by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[extends,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              units: B/h
+              params: 'last(//pgmon_io_backend[extends,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[extends,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 855cbbc83d884b00a6fea457f8e09806
+              name: 'I/O Extends Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[extends,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              units: Bps
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[extends,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 20c6135bb6c8491b9ca68fa506232b04
+              name: 'I/O fsyncs by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[fsyncs,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              params: 'last(//pgmon_io_backend[fsyncs,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[fsyncs,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 7391c372d1704f21b314379c2fe5bb64
+              name: 'I/O fsyncs Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[fsyncs,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[fsyncs,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: ab8d0766bbf64db3bb09f9442d268a83
+              name: 'I/O Hits by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[hits,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              params: 'last(//pgmon_io_backend[hits,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[hits,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: be9aa056b9364c12a8809d5dd3af8387
+              name: 'I/O Hits Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[hits,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[hits,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 0c42a40a263948dda4464856747ac6ef
+              name: 'I/O Reads by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[reads,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              units: B/h
+              params: 'last(//pgmon_io_backend[reads,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[reads,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 01bc131aaec448e6aafb7ef13d1ea5ed
+              name: 'I/O Reads Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[reads,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              units: Bps
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[reads,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 740d5c84be6e4b99b4ca3513b6c6c5ea
+              name: 'I/O Reuses by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[reuses,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              params: 'last(//pgmon_io_backend[reuses,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[reuses,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 8cbc16fc472a44d5945498285e8407be
+              name: 'I/O Reuses Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[reuses,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[reuses,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 2e796f4c75a94967a0b11a37a2d64726
+              name: 'I/O Writebacks by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[writebacks,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              units: B/h
+              params: 'last(//pgmon_io_backend[writebacks,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[writebacks,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 52d2536db79047948893ccb764b2a220
+              name: 'I/O Writebacks Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[writebacks,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              units: Bps
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[writebacks,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 6a741d3b857b48eea0e1a4d1deea7183
+              name: 'I/O Writes by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[writes,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              units: B/h
+              params: 'last(//pgmon_io_backend[writes,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[writes,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 6d10339b08b84a91a931a64050a471cc
+              name: 'I/O Writes Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[writes,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              units: Bps
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[writes,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+          graph_prototypes:
+            - uuid: faf1e3f41dd84adba42c414f6efac152
+              name: 'Backend I/O for {#BACKEND_TYPE}'
+              graph_items:
+                - color: 199C0D
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[evictions,{#BACKEND_TYPE}]'
+                - sortorder: '1'
+                  color: F63100
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[extends,{#BACKEND_TYPE}]'
+                - sortorder: '2'
+                  color: 2774A4
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[fsyncs,{#BACKEND_TYPE}]'
+                - sortorder: '3'
+                  color: F7941D
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[hits,{#BACKEND_TYPE}]'
+                - sortorder: '4'
+                  color: FC6EA3
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[reads,{#BACKEND_TYPE}]'
+                - sortorder: '5'
+                  color: 6C59DC
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[reuses,{#BACKEND_TYPE}]'
+                - sortorder: '6'
+                  color: C7A72D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[writebacks,{#BACKEND_TYPE}]'
+                - sortorder: '7'
+                  color: BA2A5D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[writes,{#BACKEND_TYPE}]'
+            - uuid: 24e2e3ec67cd43058c8b6a02d686fc6a
+              name: 'Backend I/O for {#BACKEND_TYPE} - 1h delta'
+              graph_items:
+                - color: 199C0D
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[evictions,1h,{#BACKEND_TYPE}]'
+                - sortorder: '1'
+                  color: F63100
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[extends,1h,{#BACKEND_TYPE}]'
+                - sortorder: '2'
+                  color: 2774A4
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[fsyncs,1h,{#BACKEND_TYPE}]'
+                - sortorder: '3'
+                  color: F7941D
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[hits,1h,{#BACKEND_TYPE}]'
+                - sortorder: '4'
+                  color: FC6EA3
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[reads,1h,{#BACKEND_TYPE}]'
+                - sortorder: '5'
+                  color: 6C59DC
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[reuses,1h,{#BACKEND_TYPE}]'
+                - sortorder: '6'
+                  color: C7A72D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[writebacks,1h,{#BACKEND_TYPE}]'
+                - sortorder: '7'
+                  color: BA2A5D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[writes,1h,{#BACKEND_TYPE}]'
+            - uuid: 815f1a00c19d45a18b10846644710668
+              name: 'Backend I/O Rates for {#BACKEND_TYPE}'
+              graph_items:
+                - color: 199C0D
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[evictions,{#BACKEND_TYPE}]'
+                - sortorder: '1'
+                  color: F63100
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[extends,{#BACKEND_TYPE}]'
+                - sortorder: '2'
+                  color: 2774A4
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[fsyncs,{#BACKEND_TYPE}]'
+                - sortorder: '3'
+                  color: F7941D
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[hits,{#BACKEND_TYPE}]'
+                - sortorder: '4'
+                  color: FC6EA3
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[reads,{#BACKEND_TYPE}]'
+                - sortorder: '5'
+                  color: 6C59DC
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[reuses,{#BACKEND_TYPE}]'
+                - sortorder: '6'
+                  color: C7A72D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[writebacks,{#BACKEND_TYPE}]'
+                - sortorder: '7'
+                  color: BA2A5D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[writes,{#BACKEND_TYPE}]'
+          master_item:
+            key: 'pgmon[io_per_backend]'
+          lld_macro_paths:
+            - lld_macro: '{#BACKEND_TYPE}'
+              path: $.backend_type
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$[:]'
         - uuid: 8ec029d577ae4872858e2e5cfd1cc40e
           name: 'Discover Replication'
           type: HTTP_AGENT
@@ -903,6 +1690,8 @@ zabbix_export:
                   value: Replication
                 - tag: Database
                   value: '{#DBNAME}'
+                - tag: Type
+                  value: Raw
           url: 'http://localhost:{$AGENT_PORT}/discover_rep'
           lld_macro_paths:
             - lld_macro: '{#CLIENT_ADDR}'
@@ -911,6 +1700,76 @@ zabbix_export:
               path: $.repid
             - lld_macro: '{#STATE}'
               path: $.state
+        - uuid: e22315b0295d4b43aa6b6611ede13e84
+          name: 'Discover Replication Slots'
+          type: HTTP_AGENT
+          key: pgmon_discover_slots
+          delay: 10m
+          item_prototypes:
+            - uuid: 536c5f82e3074ddfbfd842b3a2e8d46c
+              name: 'Slot {#SLOT_NAME} - Confirmed Flushed Bytes Lag'
+              type: DEPENDENT
+              key: 'pgmon_slot[confirmed_flush_bytes,{#SLOT_NAME}]'
+              delay: '0'
+              description: 'The lag in bytes for the slot.  This metric is how far behind the consumer has reported to be with regard to flushing writes to disk (or equivalent).'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - $.confirmed_flush_bytes
+              master_item:
+                key: 'pgmon_slot_stats[{#SLOT_NAME}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Slot
+                  value: '{#SLOT_NAME}'
+            - uuid: f239449d8e9e4ce79965528400f1669a
+              name: 'Slot {#SLOT_NAME} - Restart Bytes Lag'
+              type: DEPENDENT
+              key: 'pgmon_slot[restart_bytes,{#SLOT_NAME}]'
+              delay: '0'
+              description: 'The lag in bytes for the slot.  This metric is how far behind the restart location is, as reported by the consumer.'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - $.restart_bytes
+              master_item:
+                key: 'pgmon_slot_stats[{#SLOT_NAME}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Slot
+                  value: '{#SLOT_NAME}'
+            - uuid: a9f53efdce0a4bdc970925f8588e01c5
+              name: 'Replication Slot Stats for {#SLOT_NAME}'
+              type: HTTP_AGENT
+              key: 'pgmon_slot_stats[{#SLOT_NAME}]'
+              history: '0'
+              value_type: TEXT
+              trends: '0'
+              url: 'http://localhost:{$AGENT_PORT}/slot_stats'
+              query_fields:
+                - name: slot
+                  value: '{#SLOT_NAME}'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Slot
+                  value: '{#SLOT_NAME}'
+                - tag: Type
+                  value: Raw
+          url: 'http://localhost:{$AGENT_PORT}/discover_slots'
+          lld_macro_paths:
+            - lld_macro: '{#ACTIVE}'
+              path: $.active
+            - lld_macro: '{#DATABASE}'
+              path: $.database
+            - lld_macro: '{#PLUGIN}'
+              path: $.plugin
+            - lld_macro: '{#SLOT_NAME}'
+              path: $.slot_name
+            - lld_macro: '{#TEMPORARY}'
+              path: $.temporary
       macros:
         - macro: '{$AGENT_PORT}'
           value: '5400'
@@ -988,3 +1847,22 @@ zabbix_export:
                     - type: STRING
                       name: reference
                       value: XSCMZ
+  triggers:
+    - uuid: 94ce93c828aa46b4bd4f3732fc7090c6
+      expression: 'last(/PostgreSQL by pgmon/pgmon.release.latest) - last(/PostgreSQL by pgmon/pgmon[version]) >= 1'
+      name: 'PostgreSQL is at least 1 version behind'
+      priority: INFO
+      dependencies:
+        - name: 'PostgreSQL is at least 2 version behind'
+          expression: 'last(/PostgreSQL by pgmon/pgmon.release.latest) - last(/PostgreSQL by pgmon/pgmon[version]) >= 2'
+    - uuid: 2701bcd1223c4ee787f3eca30288e6c6
+      expression: 'last(/PostgreSQL by pgmon/pgmon.release.latest) - last(/PostgreSQL by pgmon/pgmon[version]) >= 2'
+      name: 'PostgreSQL is at least 2 version behind'
+      priority: INFO
+      dependencies:
+        - name: 'PostgreSQL is at least 3 version behind'
+          expression: 'last(/PostgreSQL by pgmon/pgmon.release.latest) - last(/PostgreSQL by pgmon/pgmon[version]) >= 3'
+    - uuid: 3c9adbe8b6f84b3788c6a3e513075df5
+      expression: 'last(/PostgreSQL by pgmon/pgmon.release.latest) - last(/PostgreSQL by pgmon/pgmon[version]) >= 3'
+      name: 'PostgreSQL is at least 3 version behind'
+      priority: INFO