Add script to shot tempalte coverage

* Add a script to compare the defined metrics with those used in the
  Zabbix template.

* Add a Makefile target to run the above script.

.gitignore

@@ -8,3 +8,4 @@ missing
 __pycache__
 venv
 build
+.coverage

GENTOO/pgmon-1.0.4.ebuild

@@ -0,0 +1,74 @@
+# Copyright 2024 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+PYTHON_COMPAT=( python3_{6..13} )
+
+inherit python-r1 systemd
+
+DESCRIPTION="PostgreSQL monitoring bridge"
+HOMEPAGE="None"
+
+LICENSE="BSD"
+SLOT="0"
+KEYWORDS="amd64"
+
+SRC_URI="https://code2.shh-dot-com.org/james/${PN}/releases/download/v${PV}/${P}.tar.bz2"
+
+IUSE="-systemd"
+
+DEPEND="
+	${PYTHON_DEPS}
+	dev-python/psycopg:2
+	dev-python/pyyaml
+	dev-python/requests
+	app-admin/logrotate
+	"
+RDEPEND="${DEPEND}"
+BDEPEND=""
+
+#RESTRICT="fetch"
+
+#S="${WORKDIR}/${PN}"
+
+#pkg_nofetch() {
+#	einfo "Please download"
+#	einfo "  - ${P}.tar.bz2"
+#	einfo "from ${HOMEPAGE} and place it in your DISTDIR directory."
+#	einfo "The file should be owned by portage:portage."
+#}
+
+src_compile() {
+	true
+}
+
+src_install() {
+	# Install init script
+	if ! use systemd ; then
+		newinitd "openrc/pgmon.initd" pgmon
+		newconfd "openrc/pgmon.confd" pgmon
+	fi
+
+	# Install systemd unit
+	if use systemd ; then
+		systemd_dounit "systemd/pgmon.service"
+	fi
+
+	# Install script
+	exeinto /usr/bin
+	newexe "src/pgmon.py" pgmon
+
+	# Install default config
+	diropts -o root -g root -m 0755
+	insinto /etc/pgmon
+	doins "sample-config/pgmon.yml"
+	doins "sample-config/pgmon-metrics.yml"
+
+	# Install logrotate config
+	insinto /etc/logrotate.d
+	newins "logrotate/pgmon.logrotate" pgmon
+
+	# Install man page
+	doman manpages/pgmon.1
+}

Makefile

@@ -3,7 +3,28 @@ PACKAGE_NAME := pgmon
 
 SCRIPT := src/$(PACKAGE_NAME).py
 
-VERSION := $(shell grep -m 1 '^VERSION = ' "$(SCRIPT)" | sed -ne 's/.*"\(.*\)".*/\1/p')
+# Figure out the version components
+# Note: The release is for RPM packages, where prerelease releases are written as 0.<release>
+FULL_VERSION := $(shell grep -m 1 '^VERSION = ' "$(SCRIPT)" | sed -ne 's/.*"\(.*\)".*/\1/p')
+VERSION := $(shell echo $(FULL_VERSION) | sed -n 's/\(.*\)\(-rc.*\|$$\)/\1/p')
+RELEASE := $(shell echo $(FULL_VERSION) | sed -n 's/.*-rc\([0-9]\+\)$$/\1/p')
+
+ifeq ($(RELEASE),)
+RPM_RELEASE := 1
+RPM_VERSION := $(VERSION)-$(RPM_RELEASE)
+DEB_VERSION := $(VERSION)
+else
+RPM_RELEASE := 0.$(RELEASE)
+RPM_VERSION := $(VERSION)-$(RPM_RELEASE)
+DEB_VERSION := $(VERSION)~rc$(RELEASE)
+endif
+
+# Python stuff
+PYTHON ?= python3
+PYLINT ?= pylint
+BLACK ?= black
+
+HAVE_COVERAGE := $(shell $(PYTHON) -c 'import coverage' 2>/dev/null && echo yes)
 
 # Where packages are built
 BUILD_DIR := build
@@ -23,18 +44,22 @@ SUPPORTED := ubuntu-20.04 \
 # These targets are the main ones to use for most things.
 ##
 
-.PHONY: all clean tgz test query-tests install-common install-openrc install-systemd
+.PHONY: all clean tgz lint format test query-tests install-common install-openrc install-systemd
 
+all: package-all
+
+version:
+	@echo "full version=$(FULL_VERSION) version=$(VERSION) rel=$(RELEASE) rpm=$(RPM_VERSION) deb=$(DEB_VERSION)"
 
 # Build all packages
 .PHONY: package-all
-all: $(foreach distro_release, $(SUPPORTED), package-$(distro_release))
+package-all: $(foreach distro_release, $(SUPPORTED), package-$(distro_release))
 
 # Gentoo package (tar.gz) creation
 .PHONY: package-gentoo
 package-gentoo:
 	mkdir -p $(BUILD_DIR)/gentoo
-	tar --transform "s,^,$(PACKAGE_NAME)-$(VERSION)/," -acjf $(BUILD_DIR)/gentoo/$(PACKAGE_NAME)-$(VERSION).tar.bz2 --exclude .gitignore $(shell git ls-tree --full-tree --name-only -r HEAD)
+	tar --transform "s,^,$(PACKAGE_NAME)-$(FULL_VERSION)/," -acjf $(BUILD_DIR)/gentoo/$(PACKAGE_NAME)-$(FULL_VERSION).tar.bz2 --exclude .gitignore $(shell git ls-tree --full-tree --name-only -r HEAD)
 
 
 # Create a deb package
@@ -55,20 +80,41 @@ tgz:
 	rm -rf $(BUILD_DIR)/tgz/root
 	mkdir -p $(BUILD_DIR)/tgz/root
 	$(MAKE) install-openrc DESTDIR=$(BUILD_DIR)/tgz/root
-	tar -cz -f $(BUILD_DIR)/tgz/$(PACKAGE_NAME)-$(VERSION).tgz -C $(BUILD_DIR)/tgz/root .
+	tar -cz -f $(BUILD_DIR)/tgz/$(PACKAGE_NAME)-$(FULL_VERSION).tgz -C $(BUILD_DIR)/tgz/root .
 
 # Clean up the build directory
 clean:
 	rm -rf $(BUILD_DIR)
 
+# Check for lint
+lint:
+	$(PYLINT) src/pgmon.py
+	$(PYLINT) src/test_pgmon.py
+	$(BLACK) --check --diff src/pgmon.py
+	$(BLACK) --check --diff src/test_pgmon.py
+
+# Format the code using black
+format:
+	$(BLACK) src/pgmon.py
+	$(BLACK) src/test_pgmon.py
+
 # Run unit tests for the script
 test:
-	cd src ; python3 -m unittest
+ifeq ($(HAVE_COVERAGE),yes)
+	cd src ; $(PYTHON) -m coverage run -m unittest && python3 -m coverage report -m
+else
+	cd src ; $(PYTHON) -m unittest
+endif
 
 # Run query tests
 query-tests:
 	cd tests ; ./run-tests.sh
 
+# Compare the sample metrics with the Zabbix template
+template-coverage:
+	$(PYTHON) zabbix_templates/coverage.py sample-config/pgmon-metrics.yml zabbix_templates/pgmon_templates.yaml
+
+
 # Install the script at the specified base directory (common components)
 install-common:
 	# Set up directories
@@ -129,28 +175,28 @@ debian-%-install-test:
 	docker run --rm \
 	     -v ./$(BUILD_DIR):/output \
 	     debian:$* \
-	     bash -c 'apt-get update && apt-get install -y /output/$(PACKAGE_NAME)-$(VERSION)-debian-$*.deb'
+	     bash -c 'apt-get update && apt-get install -y /output/$(PACKAGE_NAME)-$(DEB_VERSION)-debian-$*.deb'
 
 # Run a RedHat install test
 rockylinux-%-install-test:
 	docker run --rm \
 	     -v ./$(BUILD_DIR):/output \
 	     rockylinux:$* \
-	     bash -c 'dnf makecache && dnf install -y /output/$(PACKAGE_NAME)-$(VERSION)-1.el$*.noarch.rpm'
+	     bash -c 'dnf makecache && dnf install -y /output/$(PACKAGE_NAME)-$(RPM_VERSION).el$*.noarch.rpm'
 
 # Run an Ubuntu install test
 ubuntu-%-install-test:
 	docker run --rm \
 	     -v ./$(BUILD_DIR):/output \
 	     ubuntu:$* \
-	     bash -c 'apt-get update && apt-get install -y /output/$(PACKAGE_NAME)-$(VERSION)-ubuntu-$*.deb'
+	     bash -c 'apt-get update && apt-get install -y /output/$(PACKAGE_NAME)-$(DEB_VERSION)-ubuntu-$*.deb'
 
 # Run an OracleLinux install test (this is for EL7 since CentOS7 images no longer exist)
 oraclelinux-%-install-test:
 	docker run --rm \
 	     -v ./$(BUILD_DIR):/output \
 	     oraclelinux:7 \
-	     bash -c 'yum makecache && yum install -y /output/$(PACKAGE_NAME)-$(VERSION)-1.el7.noarch.rpm'
+	     bash -c 'yum makecache && yum install -y /output/$(PACKAGE_NAME)-$(RPM_VERSION).el7.noarch.rpm'
 
 # Run a Gentoo install test
 gentoo-install-test:
@@ -192,28 +238,28 @@ package-image-%:
 actually-package-debian-%:
 	$(MAKE) install-systemd DESTDIR=/output/debian-$*
 	cp -r --preserve=mode DEBIAN /output/debian-$*/
-	dpkg-deb -Zgzip --build /output/debian-$* "/output/$(PACKAGE_NAME)-$(VERSION)-debian-$*.deb"
+	dpkg-deb -Zgzip --build /output/debian-$* "/output/$(PACKAGE_NAME)-$(DEB_VERSION)-debian-$*.deb"
 
 # RedHat package creation
 actually-package-rockylinux-%:
 	mkdir -p /output/rockylinux-$*/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
-	sed -e "s/@@VERSION@@/$(VERSION)/g" RPM/$(PACKAGE_NAME).spec > /output/rockylinux-$*/SPECS/$(PACKAGE_NAME).spec
+	sed -e "s/@@VERSION@@/$(VERSION)/g" -e "s/@@RELEASE@@/$(RPM_RELEASE)/g" RPM/$(PACKAGE_NAME).spec > /output/rockylinux-$*/SPECS/$(PACKAGE_NAME).spec
 	rpmbuild --define '_topdir /output/rockylinux-$*' \
-	         --define 'version $(VERSION)' \
+	         --define 'version $(RPM_VERSION)' \
 	         -bb /output/rockylinux-$*/SPECS/$(PACKAGE_NAME).spec
-	cp /output/rockylinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(VERSION)-1.el$*.noarch.rpm /output/
+	cp /output/rockylinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(RPM_VERSION).el$*.noarch.rpm /output/
 
 # Ubuntu package creation
 actually-package-ubuntu-%:
 	$(MAKE) install-systemd DESTDIR=/output/ubuntu-$*
 	cp -r --preserve=mode DEBIAN /output/ubuntu-$*/
-	dpkg-deb -Zgzip --build /output/ubuntu-$* "/output/$(PACKAGE_NAME)-$(VERSION)-ubuntu-$*.deb"
+	dpkg-deb -Zgzip --build /output/ubuntu-$* "/output/$(PACKAGE_NAME)-$(DEB_VERSION)-ubuntu-$*.deb"
 
 # OracleLinux package creation
 actually-package-oraclelinux-%:
 	mkdir -p /output/oraclelinux-$*/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
-	sed -e "s/@@VERSION@@/$(VERSION)/g" RPM/$(PACKAGE_NAME)-el7.spec > /output/oraclelinux-$*/SPECS/$(PACKAGE_NAME).spec
+	sed -e "s/@@VERSION@@/$(VERSION)/g" -e "s/@@RELEASE@@/$(RPM_RELEASE)/g" RPM/$(PACKAGE_NAME)-el7.spec > /output/oraclelinux-$*/SPECS/$(PACKAGE_NAME).spec
 	rpmbuild --define '_topdir /output/oraclelinux-$*' \
-	         --define 'version $(VERSION)' \
+	         --define 'version $(RPM_VERSION)' \
 	         -bb /output/oraclelinux-$*/SPECS/$(PACKAGE_NAME).spec
-	cp /output/oraclelinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(VERSION)-1.el$*.noarch.rpm /output/
+	cp /output/oraclelinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(RPM_VERSION).el$*.noarch.rpm /output/

RPM/pgmon-el7.spec

@@ -1,6 +1,6 @@
 Name: pgmon
 Version: @@VERSION@@
-Release: 1%{?dist}
+Release: @@RELEASE@@%{?dist}
 Summary: A bridge to sit between monitoring tools and PostgreSQL
 
 License: MIT

RPM/pgmon.spec

@@ -1,6 +1,6 @@
 Name: pgmon
 Version: @@VERSION@@
-Release: 1%{?dist}
+Release: @@RELEASE@@%{?dist}
 Summary: A bridge to sit between monitoring tools and PostgreSQL
 
 License: MIT

pylintrc

@@ -0,0 +1,4 @@
+[MASTER]
+py-version=3.5
+
+disable=fixme

sample-config/pgmon-metrics.yml

@@ -8,14 +8,22 @@ metrics:
       0: >
         SELECT datname AS dbname
         FROM pg_database
+
+  # Note: If the user lacks sufficient privileges, these fields will be NULL.
+  #       The WHERE clause is intended to prevent Zabbix from discovering a
+  #       connection it cannot monitor.  Ideally this would generate an error
+  #       instead.
   discover_rep:
     type: set
     query:
       0: >
-        SELECT client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') AS repid,
+        SELECT host(client_addr) || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') AS repid,
+               application_name,
                client_addr,
                state
         FROM pg_stat_replication
+        WHERE state IS NOT NULL
+
   discover_slots:
     type: set
     query:
@@ -36,6 +44,7 @@ metrics:
                active
         FROM pg_replication_slots
 
+
   ##
   # cluster-wide metrics
   ##
@@ -49,7 +58,7 @@ metrics:
     query:
       0: >
         SELECT max(age(datfrozenxid)) AS xid_age,
-               0 AS mxid_age
+               NULL AS mxid_age
         FROM pg_database
       90600: >
         SELECT max(age(datfrozenxid)) AS xid_age,
@@ -85,6 +94,28 @@ metrics:
         FROM pg_stat_bgwriter bg
         CROSS JOIN pg_stat_checkpointer cp
 
+  io_per_backend:
+    type: set
+    query:
+      160000: >
+        SELECT backend_type,
+               COALESCE(SUM(reads * op_bytes), 0) AS reads,
+               COALESCE(SUM(read_time), 0) AS read_time,
+               COALESCE(SUM(writes * op_bytes), 0) AS writes,
+               COALESCE(SUM(write_time), 0) AS write_time,
+               COALESCE(SUM(writebacks * op_bytes), 0) AS writebacks,
+               COALESCE(SUM(writeback_time), 0) AS writeback_time,
+               COALESCE(SUM(extends * op_bytes), 0) AS extends,
+               COALESCE(SUM(extend_time), 0) AS extend_time,
+               COALESCE(SUM(op_bytes), 0) AS op_bytes,
+               COALESCE(SUM(hits), 0) AS hits,
+               COALESCE(SUM(evictions), 0) AS evictions,
+               COALESCE(SUM(reuses), 0) AS reuses,
+               COALESCE(SUM(fsyncs), 0) AS fsyncs,
+               COALESCE(SUM(fsync_time), 0) AS fsync_time
+        FROM pg_stat_io
+        GROUP BY backend_type
+
 
   ##
   # Per-database metrics
@@ -107,17 +138,17 @@ metrics:
                temp_files,
                temp_bytes,
                deadlocks,
-               0 AS checksum_failures,
+               NULL AS checksum_failures,
                blk_read_time,
                blk_write_time,
-               0 AS session_time,
-               0 AS active_time,
-               0 AS idle_in_transaction_time,
-               0 AS sessions,
-               0 AS sessions_abandoned,
-               0 AS sessions_fatal,
-               0 AS sessions_killed,
-               extract('epoch' from stats_reset)::float AS stats_reset
+               NULL AS session_time,
+               NULL AS active_time,
+               NULL AS idle_in_transaction_time,
+               NULL AS sessions,
+               NULL AS sessions_abandoned,
+               NULL AS sessions_fatal,
+               NULL AS sessions_killed,
+               extract('epoch' from stats_reset) AS stats_reset
         FROM pg_stat_database WHERE datname = %(dbname)s
       140000: >
         SELECT numbackends,
@@ -134,8 +165,7 @@ metrics:
                temp_files,
                temp_bytes,
                deadlocks,
-               COALESCE(checksum_failures,
-               0) AS checksum_failures,
+               COALESCE(checksum_failures, 0) AS checksum_failures,
                blk_read_time,
                blk_write_time,
                session_time,
@@ -145,7 +175,7 @@ metrics:
                sessions_abandoned,
                sessions_fatal,
                sessions_killed,
-               extract('epoch' from stats_reset)::float AS stats_reset
+               extract('epoch' from stats_reset) AS stats_reset
         FROM pg_stat_database WHERE datname = %(dbname)s
     test_args:
       dbname: postgres
@@ -168,13 +198,52 @@ metrics:
       0: >
         SELECT state,
                count(*) AS backend_count,
-               COALESCE(EXTRACT(EPOCH FROM max(now() - state_change))::float, 0) AS max_state_time
+               COALESCE(EXTRACT(EPOCH FROM max(now() - state_change)), 0) AS max_state_time
         FROM pg_stat_activity
         WHERE datname = %(dbname)s
         GROUP BY state
     test_args:
       dbname: postgres
 
+  sequence_usage:
+    type: value
+    query:
+# 9.2 lacks lateral joins, the pg_sequence_last_value function, and the pg_sequences view
+#      0: >
+#        SELECT COALESCE(MAX(pg_sequence_last_value(c.oid)::float / (pg_sequence_parameters(oid)).maximum_value), 0) AS max_usage
+#        FROM pg_class c
+#        WHERE c.relkind = 'S'
+# 9.3 - 9.6 lacks the pg_sequence_last_value function, and pg_sequences view
+#      90300: >
+#        SELECT COALESCE(MAX(pg_sequence_last_value(c.oid)::float / s.maximum_value), 0) AS max_usage
+#        FROM pg_class c
+#        CROSS JOIN LATERAL pg_sequence_parameters(c.oid) AS s
+#        WHERE c.relkind = 'S'
+      100000: SELECT COALESCE(MAX(last_value::float / max_value), 0) AS max_usage FROM pg_sequences;
+    test_args:
+      dbname: postgres
+
+  sequence_visibility:
+    type: row
+    query:
+      100000: >
+        SELECT COUNT(*) FILTER (WHERE has_sequence_privilege(c.oid, 'SELECT,USAGE')) AS visible_sequences,
+               COUNT(*) AS total_sequences
+        FROM pg_class AS c
+        WHERE relkind = 'S'
+
+  locks:
+    type: row
+    query:
+      0:
+        SELECT COUNT(*) AS total,
+               SUM(CASE WHEN granted THEN 1 ELSE 0 END) AS granted
+        FROM pg_locks
+      90400: >
+        SELECT COUNT(*) AS total,
+               COUNT(*) FILTER (WHERE granted) AS granted
+        FROM pg_locks
+
 
   ##
   # Per-replication metrics
@@ -184,7 +253,7 @@ metrics:
     query:
       90400: >
         SELECT pid, usename,
-               EXTRACT(EPOCH FROM backend_start)::integer AS backend_start,
+               EXTRACT(EPOCH FROM backend_start) AS backend_start,
                state,
                pg_xlog_location_diff(pg_current_xlog_location(), sent_location) AS sent_lsn,
                pg_xlog_location_diff(pg_current_xlog_location(), write_location) AS write_lsn,
@@ -195,20 +264,21 @@ metrics:
                NULL AS replay_lag,
                sync_state
         FROM pg_stat_replication
-        WHERE client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = %(repid)s
+        WHERE host(client_addr) || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = %(repid)s
       100000: >
         SELECT pid, usename,
-               EXTRACT(EPOCH FROM backend_start)::integer AS backend_start,
+               EXTRACT(EPOCH FROM backend_start) AS backend_start,
                state,
                pg_wal_lsn_diff(pg_current_wal_lsn(), sent_lsn) AS sent_lsn,
                pg_wal_lsn_diff(pg_current_wal_lsn(), write_lsn) AS write_lsn,
                pg_wal_lsn_diff(pg_current_wal_lsn(), flush_lsn) AS flush_lsn,
                pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) AS replay_lsn,
-               COALESCE(EXTRACT(EPOCH FROM write_lag), 0)::integer AS write_lag,
-               COALESCE(EXTRACT(EPOCH FROM flush_lag), 0)::integer AS flush_lag,
-               COALESCE(EXTRACT(EPOCH FROM replay_lag), 0)::integer AS replay_lag,
+               COALESCE(EXTRACT(EPOCH FROM write_lag), 0) AS write_lag,
+               COALESCE(EXTRACT(EPOCH FROM flush_lag), 0) AS flush_lag,
+               COALESCE(EXTRACT(EPOCH FROM replay_lag), 0) AS replay_lag,
                sync_state
-        FROM pg_stat_replication WHERE client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = %(repid)s
+        FROM pg_stat_replication
+        WHERE host(client_addr) || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = %(repid)s
     test_args:
       repid: 127.0.0.1_test_rep
 
@@ -240,6 +310,7 @@ metrics:
     test_args:
       slot: test_slot
 
+
   ##
   # Debugging
   ##

src/pgmon.py

@@ -1,103 +1,147 @@
 #!/usr/bin/env python3
+"""
+pgmon is a monitoring intermediary that sits between a PostgreSQL cluster and a monitoring systen
+that is capable of parsing JSON responses over an HTTP connection.
+"""
+
+# pylint: disable=too-few-public-methods
 
-import yaml
 import json
 import time
 import os
 import sys
-
+import signal
 import argparse
 import logging
+import re
+
+from decimal import Decimal
+
+from urllib.parse import urlparse, parse_qs
+
+from contextlib import contextmanager
 
 from datetime import datetime, timedelta
 
+from http.server import BaseHTTPRequestHandler
+from http.server import ThreadingHTTPServer
+
+from threading import Lock
+
+import yaml
+
 import psycopg2
 from psycopg2.extras import RealDictCursor
 from psycopg2.pool import ThreadedConnectionPool
 
-from contextlib import contextmanager
-
-import signal
-from threading import Thread, Lock, Semaphore
-
-from http.server import BaseHTTPRequestHandler, HTTPServer
-from http.server import ThreadingHTTPServer
-from urllib.parse import urlparse, parse_qs
-
 import requests
-import re
 
-VERSION = "1.0.3"
 
-# Configuration
-config = {}
+VERSION = "1.0.4"
 
-# Dictionary of current PostgreSQL connection pools
-connections_lock = Lock()
-connections = {}
 
-# Dictionary of unhappy databases.  Keys are database names, value is the time
-# the database was determined to be unhappy plus the cooldown setting.  So,
-# basically it's the time when we should try to connect to the database again.
-unhappy_cooldown = {}
+class Context:
+    """
+    The global context for connections, config, version, nad IPC
+    """
 
-# Version information
-cluster_version = None
-cluster_version_next_check = None
-cluster_version_lock = Lock()
+    # Configuration
+    config = {}
 
-# PostgreSQL latest version information
-latest_version = None
-latest_version_next_check = None
-latest_version_lock = Lock()
-release_supported = None
+    # Dictionary of current PostgreSQL connection pools
+    connections_lock = Lock()
+    connections = {}
 
-# Running state (used to gracefully shut down)
-running = True
+    # Dictionary of unhappy databases.  Keys are database names, value is the time
+    # the database was determined to be unhappy plus the cooldown setting.  So,
+    # basically it's the time when we should try to connect to the database again.
+    unhappy_cooldown = {}
 
-# The http server object
-httpd = None
+    # Version information
+    cluster_version = None
+    cluster_version_next_check = None
+    cluster_version_lock = Lock()
 
-# Where the config file lives
-config_file = None
+    # PostgreSQL latest version information
+    latest_version = None
+    latest_version_next_check = None
+    latest_version_lock = Lock()
+    release_supported = None
 
-# Configure logging
-log = logging.getLogger(__name__)
-formatter = logging.Formatter(
-    "%(asctime)s - %(levelname)s - %(filename)s: %(funcName)s() line %(lineno)d: %(message)s"
-)
-console_log_handler = logging.StreamHandler()
-console_log_handler.setFormatter(formatter)
-log.addHandler(console_log_handler)
+    # Running state (used to gracefully shut down)
+    running = True
+
+    # The http server object
+    httpd = None
+
+    # Where the config file lives
+    config_file = None
+
+    # Configure logging
+    log = logging.getLogger(__name__)
+
+    @classmethod
+    def init_logging(cls):
+        """
+        Actually initialize the logging framework.  Since we don't ever instantiate the Context
+        class, this provides a way to make a few modifications to the log handler.
+        """
+
+        formatter = logging.Formatter(
+            "%(asctime)s - %(levelname)s - %(filename)s: "
+            "%(funcName)s() line %(lineno)d: %(message)s"
+        )
+        console_log_handler = logging.StreamHandler()
+        console_log_handler.setFormatter(formatter)
+        cls.log.addHandler(console_log_handler)
 
 
 # Error types
 class ConfigError(Exception):
-    pass
+    """
+    Error type for all config related errors.
+    """
 
 
 class DisconnectedError(Exception):
-    pass
+    """
+    Error indicating a previously active connection to the database has been disconnected.
+    """
 
 
 class UnhappyDBError(Exception):
-    pass
+    """
+    Error indicating that a database the code has been asked to connect to is on the unhappy list.
+    """
 
 
 class UnknownMetricError(Exception):
-    pass
+    """
+    Error indicating that an undefined metric was requested.
+    """
 
 
 class MetricVersionError(Exception):
-    pass
+    """
+    Error indicating that there is no suitable query for a metric that was requested for the
+    version of PostgreSQL being monitored.
+    """
 
 
 class LatestVersionCheckError(Exception):
-    pass
+    """
+    Error indicating that there was a problem retrieving or parsing the latest version information.
+    """
+
+
+class InvalidDataError(Exception):
+    """
+    Error indicating query results were somehow invalid
+    """
 
 
 # Default config settings
-default_config = {
+DEFAULT_CONFIG = {
     # The address the agent binds to
     "address": "127.0.0.1",
     # The port the agent listens on for requests
@@ -175,6 +219,60 @@ def update_deep(d1, d2):
     return d1
 
 
+def validate_metric(path, name, metric):
+    """
+    Validate a metric definition from a given file.  If any query definitions come from external
+    files, the metric dict will be updated with the actual query.
+
+    Params:
+        path: path to the file which contains this definition
+        name: name of the metric
+        metric: the dictionary containing the metric definition
+    """
+    # Validate return types
+    try:
+        if metric["type"] not in ["value", "row", "column", "set"]:
+            raise ConfigError(
+                "Invalid return type: {} for metric {} in {}".format(
+                    metric["type"], name, path
+                )
+            )
+    except KeyError as e:
+        raise ConfigError(
+            "No type specified for metric {} in {}".format(name, path)
+        ) from e
+
+    # Ensure queries exist
+    query_dict = metric.get("query", {})
+    if not isinstance(query_dict, dict):
+        raise ConfigError(
+            "Query definition should be a dictionary, got: {} for metric {} in {}".format(
+                query_dict, name, path
+            )
+        )
+
+    if len(query_dict) == 0:
+        raise ConfigError("Missing queries for metric {} in {}".format(name, path))
+
+    # Read external sql files and validate version keys
+    config_base = os.path.dirname(path)
+    for vers, query in metric["query"].items():
+        try:
+            int(vers)
+        except Exception as e:
+            raise ConfigError(
+                "Invalid version: {} for metric {} in {}".format(vers, name, path)
+            ) from e
+
+        # Read in the external query and update the definition in the metricdictionary
+        if query.startswith("file:"):
+            query_path = query[5:]
+            if not query_path.startswith("/"):
+                query_path = os.path.join(config_base, query_path)
+            with open(query_path, "r", encoding="utf-8") as f:
+                metric["query"][vers] = f.read()
+
+
 def read_config(path, included=False):
     """
     Read a config file.
@@ -183,61 +281,21 @@ def read_config(path, included=False):
         path: path to the file to read
         included: is this file included by another file?
     """
+
     # Read config file
-    log.info("Reading log file: {}".format(path))
-    with open(path, "r") as f:
+    Context.log.info("Reading log file: %s", path)
+    with open(path, "r", encoding="utf-8") as f:
         try:
             cfg = yaml.safe_load(f)
         except yaml.parser.ParserError as e:
-            raise ConfigError("Inavlid config file: {}: {}".format(path, e))
-
-    # Since we use it a few places, get the base directory from the config
-    config_base = os.path.dirname(path)
+            raise ConfigError("Inavlid config file: {}: {}".format(path, e)) from e
 
     # Read any external queries and validate metric definitions
     for name, metric in cfg.get("metrics", {}).items():
-        # Validate return types
-        try:
-            if metric["type"] not in ["value", "row", "column", "set"]:
-                raise ConfigError(
-                    "Invalid return type: {} for metric {} in {}".format(
-                        metric["type"], name, path
-                    )
-                )
-        except KeyError:
-            raise ConfigError(
-                "No type specified for metric {} in {}".format(name, path)
-            )
-
-        # Ensure queries exist
-        query_dict = metric.get("query", {})
-        if type(query_dict) is not dict:
-            raise ConfigError(
-                "Query definition should be a dictionary, got: {} for metric {} in {}".format(
-                    query_dict, name, path
-                )
-            )
-
-        if len(query_dict) == 0:
-            raise ConfigError("Missing queries for metric {} in {}".format(name, path))
-
-        # Read external sql files and validate version keys
-        for vers, query in metric["query"].items():
-            try:
-                int(vers)
-            except:
-                raise ConfigError(
-                    "Invalid version: {} for metric {} in {}".format(vers, name, path)
-                )
-
-            if query.startswith("file:"):
-                query_path = query[5:]
-                if not query_path.startswith("/"):
-                    query_path = os.path.join(config_base, query_path)
-                with open(query_path, "r") as f:
-                    metric["query"][vers] = f.read()
+        validate_metric(path, name, metric)
 
     # Read any included config files
+    config_base = os.path.dirname(path)
     for inc in cfg.get("include", []):
         # Prefix relative paths with the directory from the current config
         if not inc.startswith("/"):
@@ -248,34 +306,37 @@ def read_config(path, included=False):
     # config
     if included:
         return cfg
-    else:
-        new_config = {}
-        update_deep(new_config, default_config)
-        update_deep(new_config, cfg)
 
-        # Minor sanity checks
-        if len(new_config["metrics"]) == 0:
-            log.error("No metrics are defined")
-            raise ConfigError("No metrics defined")
+    new_config = {}
+    update_deep(new_config, DEFAULT_CONFIG)
+    update_deep(new_config, cfg)
 
-        # Validate the new log level before changing the config
-        if new_config["log_level"].upper() not in [
-            "DEBUG",
-            "INFO",
-            "WARNING",
-            "ERROR",
-            "CRITICAL",
-        ]:
-            raise ConfigError("Invalid log level: {}".format(new_config["log_level"]))
+    # Minor sanity checks
+    if len(new_config["metrics"]) == 0:
+        Context.log.error("No metrics are defined")
+        raise ConfigError("No metrics defined")
 
-        global config
-        config = new_config
+    # Validate the new log level before changing the config
+    if new_config["log_level"].upper() not in [
+        "DEBUG",
+        "INFO",
+        "WARNING",
+        "ERROR",
+        "CRITICAL",
+    ]:
+        raise ConfigError("Invalid log level: {}".format(new_config["log_level"]))
 
-        # Apply changes to log level
-        log.setLevel(logging.getLevelName(config["log_level"].upper()))
+    Context.config = new_config
+
+    # Apply changes to log level
+    Context.log.setLevel(logging.getLevelName(Context.config["log_level"].upper()))
+
+    # Return the config (mostly to make pylint happy, but also in case I opt to remove the side
+    # effect and make this more functional.
+    return Context.config
 
 
-def signal_handler(sig, frame):
+def signal_handler(sig, frame):  # pylint: disable=unused-argument
     """
     Function for handling signals
 
@@ -286,19 +347,22 @@ def signal_handler(sig, frame):
 
     # Signal everything to shut down
     if sig in [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]:
-        log.info("Shutting down ...")
-        global running
-        running = False
-        if httpd is not None:
-            httpd.socket.close()
+        Context.log.info("Shutting down ...")
+        Context.running = False
+        if Context.httpd is not None:
+            Context.httpd.socket.close()
 
     # Signal a reload
     if sig == signal.SIGHUP:
-        log.warning("Received config reload signal")
-        read_config(config_file)
+        Context.log.warning("Received config reload signal")
+        read_config(Context.config_file)
 
 
 class ConnectionPool(ThreadedConnectionPool):
+    """
+    Threaded connection pool that has a context manager.
+    """
+
     def __init__(self, dbname, minconn, maxconn, *args, **kwargs):
         # Make sure dbname isn't different in the kwargs
         kwargs["dbname"] = dbname
@@ -307,7 +371,14 @@ class ConnectionPool(ThreadedConnectionPool):
         self.name = dbname
 
     @contextmanager
-    def connection(self, timeout=None):
+    def connection(self, timeout):
+        """
+        Connection context manager for our connection pool.  This will attempt to retrieve a
+        connection until the timeout is reached.
+
+        Params:
+            timeout: how long to keep trying to get a connection bedore giving up
+        """
         conn = None
         timeout_time = datetime.now() + timedelta(timeout)
         # We will continue to try to get a connection slot until we time out
@@ -331,34 +402,37 @@ class ConnectionPool(ThreadedConnectionPool):
 def get_pool(dbname):
     """
     Get a database connection pool.
+
+    Params:
+        dbname: the name of the database for which a connection pool should be returned.
     """
     # Check if the db is unhappy and wants to be left alone
-    if dbname in unhappy_cooldown:
-        if unhappy_cooldown[dbname] > datetime.now():
+    if dbname in Context.unhappy_cooldown:
+        if Context.unhappy_cooldown[dbname] > datetime.now():
             raise UnhappyDBError()
 
     # Create a connection pool if it doesn't already exist
-    if dbname not in connections:
-        with connections_lock:
+    if dbname not in Context.connections:
+        with Context.connections_lock:
             # Make sure nobody created the pool while we were waiting on the
             # lock
-            if dbname not in connections:
-                log.info("Creating connection pool for: {}".format(dbname))
+            if dbname not in Context.connections:
+                Context.log.info("Creating connection pool for: %s", dbname)
                 # Actually create the connection pool
-                connections[dbname] = ConnectionPool(
+                Context.connections[dbname] = ConnectionPool(
                     dbname,
-                    int(config["min_pool_size"]),
-                    int(config["max_pool_size"]),
+                    int(Context.config["min_pool_size"]),
+                    int(Context.config["max_pool_size"]),
                     application_name="pgmon",
-                    host=config["dbhost"],
-                    port=config["dbport"],
-                    user=config["dbuser"],
-                    connect_timeout=int(config["connect_timeout"]),
-                    sslmode=config["ssl_mode"],
+                    host=Context.config["dbhost"],
+                    port=Context.config["dbport"],
+                    user=Context.config["dbuser"],
+                    connect_timeout=int(Context.config["connect_timeout"]),
+                    sslmode=Context.config["ssl_mode"],
                 )
             # Clear the unhappy indicator if present
-            unhappy_cooldown.pop(dbname, None)
-    return connections[dbname]
+            Context.unhappy_cooldown.pop(dbname, None)
+    return Context.connections[dbname]
 
 
 def handle_connect_failure(pool):
@@ -366,8 +440,8 @@ def handle_connect_failure(pool):
     Mark the database as being unhappy so we can leave it alone for a while
     """
     dbname = pool.name
-    unhappy_cooldown[dbname] = datetime.now() + timedelta(
-        seconds=int(config["reconnect_cooldown"])
+    Context.unhappy_cooldown[dbname] = datetime.now() + timedelta(
+        seconds=int(Context.config["reconnect_cooldown"])
     )
 
 
@@ -391,38 +465,74 @@ def get_query(metric, version):
     raise MetricVersionError("Missing metric query for PostgreSQL {}".format(version))
 
 
+def json_encode_special(obj):
+    """
+    Encoder function to handle types the standard JSON package doesn't know what
+    to do with
+    """
+    if isinstance(obj, Decimal):
+        return float(obj)
+    raise TypeError("Cannot serialize object of {}".format(type(obj)))
+
+
+def json_encode_result(return_type, res):
+    """
+    Return a json string encoding of the results of a query.
+
+    params:
+        return_type: the expected structure to return.  One of:
+                     value, row, column, set
+        res: the query results
+
+    returns: a json string form of the results
+
+    raises:
+        ConfigError: when an invalid return_type is given
+        InvalidDataError: when the query results don't match the return type
+    """
+    try:
+        if return_type == "value":
+            if len(res) == 0:
+                return ""
+            return str(list(res[0].values())[0])
+
+        if return_type == "row":
+            return json.dumps(
+                res[0] if len(res) > 0 else {}, default=json_encode_special
+            )
+
+        if return_type == "column":
+            return json.dumps(
+                [list(r.values())[0] for r in res], default=json_encode_special
+            )
+
+        if return_type == "set":
+            return json.dumps(res, default=json_encode_special)
+    except IndexError as e:
+        raise InvalidDataError(e) from e
+
+    # If we got to this point, the return type is invalid
+    raise ConfigError("Invalid query return type: {}".format(return_type))
+
+
 def run_query_no_retry(pool, return_type, query, args):
     """
     Run the query with no explicit retry code
     """
-    with pool.connection(float(config["connect_timeout"])) as conn:
+    with pool.connection(float(Context.config["connect_timeout"])) as conn:
         try:
             with conn.cursor(cursor_factory=RealDictCursor) as curs:
                 curs.execute(query, args)
                 res = curs.fetchall()
 
-                if return_type == "value":
-                    if len(res) == 0:
-                        return ""
-                    return str(list(res[0].values())[0])
-                elif return_type == "row":
-                    if len(res) == 0:
-                        return "[]"
-                    return json.dumps(res[0])
-                elif return_type == "column":
-                    if len(res) == 0:
-                        return "[]"
-                    return json.dumps([list(r.values())[0] for r in res])
-                elif return_type == "set":
-                    return json.dumps(res)
-        except:
+                return json_encode_result(return_type, res)
+        except Exception as e:
             dbname = pool.name
-            if dbname in unhappy_cooldown:
-                raise UnhappyDBError()
-            elif conn.closed != 0:
-                raise DisconnectedError()
-            else:
-                raise
+            if dbname in Context.unhappy_cooldown:
+                raise UnhappyDBError() from e
+            if conn.closed != 0:
+                raise DisconnectedError() from e
+            raise
 
 
 def run_query(pool, return_type, query, args):
@@ -443,7 +553,7 @@ def run_query(pool, return_type, query, args):
     try:
         return run_query_no_retry(pool, return_type, query, args)
     except DisconnectedError:
-        log.warning("Stale PostgreSQL connection found ... trying again")
+        Context.log.warning("Stale PostgreSQL connection found ... trying again")
         # This sleep is an annoying hack to give the pool workers time to
         # actually mark the connection, otherwise it can be given back in the
         # next connection() call
@@ -451,9 +561,9 @@ def run_query(pool, return_type, query, args):
         time.sleep(1)
         try:
             return run_query_no_retry(pool, return_type, query, args)
-        except:
+        except Exception as e:
             handle_connect_failure(pool)
-            raise UnhappyDBError()
+            raise UnhappyDBError() from e
 
 
 def get_cluster_version():
@@ -461,40 +571,39 @@ def get_cluster_version():
     Get the PostgreSQL version if we don't already know it, or if it's been
     too long sice the last time it was checked.
     """
-    global cluster_version
-    global cluster_version_next_check
 
     # If we don't know the version or it's past the recheck time, get the
     # version from the database.  Only one thread needs to do this, so they all
     # try to grab the lock, and then make sure nobody else beat them to it.
     if (
-        cluster_version is None
-        or cluster_version_next_check is None
-        or cluster_version_next_check < datetime.now()
+        Context.cluster_version is None
+        or Context.cluster_version_next_check is None
+        or Context.cluster_version_next_check < datetime.now()
     ):
-        with cluster_version_lock:
+        with Context.cluster_version_lock:
             # Only check if nobody already got the version before us
             if (
-                cluster_version is None
-                or cluster_version_next_check is None
-                or cluster_version_next_check < datetime.now()
+                Context.cluster_version is None
+                or Context.cluster_version_next_check is None
+                or Context.cluster_version_next_check < datetime.now()
             ):
-                log.info("Checking PostgreSQL cluster version")
-                pool = get_pool(config["dbname"])
-                cluster_version = int(
+                Context.log.info("Checking PostgreSQL cluster version")
+                pool = get_pool(Context.config["dbname"])
+                Context.cluster_version = int(
                     run_query(pool, "value", "SHOW server_version_num", None)
                 )
-                cluster_version_next_check = datetime.now() + timedelta(
-                    seconds=int(config["version_check_period"])
+                Context.cluster_version_next_check = datetime.now() + timedelta(
+                    seconds=int(Context.config["version_check_period"])
                 )
-                log.info("Got PostgreSQL cluster version: {}".format(cluster_version))
-                log.debug(
-                    "Next PostgreSQL cluster version check will be after: {}".format(
-                        cluster_version_next_check
-                    )
+                Context.log.info(
+                    "Got PostgreSQL cluster version: %s", Context.cluster_version
+                )
+                Context.log.debug(
+                    "Next PostgreSQL cluster version check will be after: %s",
+                    Context.cluster_version_next_check,
                 )
 
-    return cluster_version
+    return Context.cluster_version
 
 
 def version_num_to_release(version_num):
@@ -507,8 +616,7 @@ def version_num_to_release(version_num):
     """
     if version_num // 10000 < 10:
         return version_num // 10000 + (version_num % 10000 // 100 / 10)
-    else:
-        return version_num // 10000
+    return version_num // 10000
 
 
 def parse_version_rss(raw_rss, release):
@@ -516,7 +624,7 @@ def parse_version_rss(raw_rss, release):
     Parse the raw RSS from the versions.rss feed to extract the latest version of
     PostgreSQL that's availabe for the cluster being monitored.
 
-    This sets these global variables:
+    This sets these Context variables:
         latest_version
         release_supported
 
@@ -526,8 +634,6 @@ def parse_version_rss(raw_rss, release):
         raw_rss: The raw rss text from versions.rss
         release: The PostgreSQL release we care about (ex: 9.2, 14)
     """
-    global latest_version
-    global release_supported
 
     # Regular expressions for parsing the RSS document
     version_line = re.compile(
@@ -547,75 +653,75 @@ def parse_version_rss(raw_rss, release):
             version = m.group(1)
             parts = list(map(int, version.split(".")))
             if parts[0] < 10:
-                latest_version = int(
+                Context.latest_version = int(
                     "{}{:02}{:02}".format(parts[0], parts[1], parts[2])
                 )
             else:
-                latest_version = int("{}00{:02}".format(parts[0], parts[1]))
+                Context.latest_version = int("{}00{:02}".format(parts[0], parts[1]))
         elif release_found:
             # The next line after the version tells if the version is supported
             if unsupported_line.match(line):
-                release_supported = False
+                Context.release_supported = False
             else:
-                release_supported = True
+                Context.release_supported = True
             break
 
     # Make sure we actually found it
     if not release_found:
         raise LatestVersionCheckError("Current release ({}) not found".format(release))
 
-    log.info(
-        "Got latest PostgreSQL version: {} supported={}".format(
-            latest_version, release_supported
-        )
+    Context.log.info(
+        "Got latest PostgreSQL version: %s supported=%s",
+        Context.latest_version,
+        Context.release_supported,
     )
-    log.debug(
-        "Next latest PostgreSQL version check will be after: {}".format(
-            latest_version_next_check
-        )
+    Context.log.debug(
+        "Next latest PostgreSQL version check will be after: %s",
+        Context.latest_version_next_check,
     )
 
 
 def get_latest_version():
     """
-    Get the latest supported version of the major PostgreSQL release running on the server being monitored.
+    Get the latest supported version of the major PostgreSQL release running on the server being
+    monitored.
     """
 
-    global latest_version_next_check
-
     # If we don't know the latest version or it's past the recheck time, get the
     # version from the PostgreSQL RSS feed.  Only one thread needs to do this, so
     # they all try to grab the lock, and then make sure nobody else beat them to it.
     if (
-        latest_version is None
-        or latest_version_next_check is None
-        or latest_version_next_check < datetime.now()
+        Context.latest_version is None
+        or Context.latest_version_next_check is None
+        or Context.latest_version_next_check < datetime.now()
     ):
         # Note: we get the cluster version here before grabbing the latest_version_lock
         #       lock so it's not held while trying to talk with the DB.
         release = version_num_to_release(get_cluster_version())
 
-        with latest_version_lock:
+        with Context.latest_version_lock:
             # Only check if nobody already got the version before us
             if (
-                latest_version is None
-                or latest_version_next_check is None
-                or latest_version_next_check < datetime.now()
+                Context.latest_version is None
+                or Context.latest_version_next_check is None
+                or Context.latest_version_next_check < datetime.now()
             ):
-                log.info("Checking latest PostgreSQL version")
-                latest_version_next_check = datetime.now() + timedelta(
-                    seconds=int(config["latest_version_check_period"])
+                Context.log.info("Checking latest PostgreSQL version")
+                Context.latest_version_next_check = datetime.now() + timedelta(
+                    seconds=int(Context.config["latest_version_check_period"])
                 )
 
                 # Grab the RSS feed
-                raw_rss = requests.get("https://www.postgresql.org/versions.rss")
+                raw_rss = requests.get(
+                    "https://www.postgresql.org/versions.rss", timeout=30
+                )
                 if raw_rss.status_code != 200:
-                    raise LatestVersionCheckError("code={}".format(r.status_code))
+                    raise LatestVersionCheckError("code={}".format(raw_rss.status_code))
 
-                # Parse the RSS body and set global variables
+                # Parse the RSS body and set Context variables
                 parse_version_rss(raw_rss.text, release)
 
-    return latest_version
+    return Context.latest_version
 
 
 def sample_metric(dbname, metric_name, args, retry=True):
@@ -624,9 +730,9 @@ def sample_metric(dbname, metric_name, args, retry=True):
     """
     # Get the metric definition
     try:
-        metric = config["metrics"][metric_name]
-    except KeyError:
-        raise UnknownMetricError("Unknown metric: {}".format(metric_name))
+        metric = Context.config["metrics"][metric_name]
+    except KeyError as e:
+        raise UnknownMetricError("Unknown metric: {}".format(metric_name)) from e
 
     # Get the connection pool for the database, or create one if it doesn't
     # already exist.
@@ -641,8 +747,7 @@ def sample_metric(dbname, metric_name, args, retry=True):
     # Execute the quert
     if retry:
         return run_query(pool, metric["type"], query, args)
-    else:
-        return run_query_no_retry(pool, metric["type"], query, args)
+    return run_query_no_retry(pool, metric["type"], query, args)
 
 
 def test_queries():
@@ -650,12 +755,17 @@ def test_queries():
     Run all of the metric queries against a database and check the results
     """
     # We just use the default db for tests
-    dbname = config["dbname"]
+    dbname = Context.config["dbname"]
     # Loop through all defined metrics.
-    for name, metric in config["metrics"].items():
+    for name, metric in Context.config["metrics"].items():
         # If the metric has arguments to use while testing, grab those
         args = metric.get("test_args", {})
-        print("Testing {} [{}]".format(name, ", ".join(["{}={}".format(key, value) for key, value in args.items()])))
+        print(
+            "Testing {} [{}]".format(
+                name,
+                ", ".join(["{}={}".format(key, value) for key, value in args.items()]),
+            )
+        )
         # When testing against a docker container, we may end up connecting
         # before the service is truly up (it restarts during the initialization
         # phase).  To cope with this, we'll allow a few connection failures.
@@ -692,9 +802,8 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
         """
         Override to suppress standard request logging
         """
-        pass
 
-    def do_GET(self):
+    def do_GET(self):  # pylint: disable=invalid-name
         """
         Handle a request.  This is just a wrapper around the actual handler
         code to keep things more readable.
@@ -702,7 +811,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
         try:
             self._handle_request()
         except BrokenPipeError:
-            log.error("Client disconnected, exiting handler")
+            Context.log.error("Client disconnected, exiting handler")
 
     def _handle_request(self):
         """
@@ -715,7 +824,6 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
 
         if metric_name == "agent_version":
             self._reply(200, VERSION)
-            return
         elif metric_name == "latest_version_info":
             try:
                 get_latest_version()
@@ -723,46 +831,40 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
                     200,
                     json.dumps(
                         {
-                            "latest": latest_version,
-                            "supported": 1 if release_supported else 0,
+                            "latest": Context.latest_version,
+                            "supported": 1 if Context.release_supported else 0,
                         }
                     ),
                 )
             except LatestVersionCheckError as e:
-                log.error("Failed to retrieve latest version information: {}".format(e))
+                Context.log.error(
+                    "Failed to retrieve latest version information: %s", e
+                )
                 self._reply(503, "Failed to retrieve latest version info")
-            return
+        else:
+            # Note: parse_qs returns the values as a list.  Since we always expect
+            #       single values, just grab the first from each.
+            args = {key: values[0] for key, values in parsed_query.items()}
 
-        # Note: parse_qs returns the values as a list.  Since we always expect
-        #       single values, just grab the first from each.
-        args = {key: values[0] for key, values in parsed_query.items()}
+            # Get the dbname.  If none was provided, use the default from the
+            # config.
+            dbname = args.get("dbname", Context.config["dbname"])
 
-        # Get the dbname.  If none was provided, use the default from the
-        # config.
-        dbname = args.get("dbname", config["dbname"])
-
-        # Sample the metric
-        try:
-            self._reply(200, sample_metric(dbname, metric_name, args))
-            return
-        except UnknownMetricError as e:
-            log.error("Unknown metric: {}".format(metric_name))
-            self._reply(404, "Unknown metric")
-            return
-        except MetricVersionError as e:
-            log.error(
-                "Failed to find a version of {} for {}".format(metric_name, version)
-            )
-            self._reply(404, "Unsupported version")
-            return
-        except UnhappyDBError as e:
-            log.info("Database {} is unhappy, please be patient".format(dbname))
-            self._reply(503, "Database unavailable")
-            return
-        except Exception as e:
-            log.error("Error running query: {}".format(e))
-            self._reply(500, "Unexpected error: {}".format(e))
-            return
+            # Sample the metric
+            try:
+                self._reply(200, sample_metric(dbname, metric_name, args))
+            except UnknownMetricError:
+                Context.log.error("Unknown metric: %s", metric_name)
+                self._reply(404, "Unknown metric")
+            except MetricVersionError:
+                Context.log.error("Failed to find an query version for %s", metric_name)
+                self._reply(404, "Unsupported version")
+            except UnhappyDBError:
+                Context.log.info("Database %s is unhappy, please be patient", dbname)
+                self._reply(503, "Database unavailable")
+            except Exception as e:  # pylint: disable=broad-exception-caught
+                Context.log.error("Error running query: %s", e)
+                self._reply(500, "Unexpected error: {}".format(e))
 
     def _reply(self, code, content):
         """
@@ -775,7 +877,14 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
         self.wfile.write(bytes(content, "utf-8"))
 
 
-if __name__ == "__main__":
+def main():
+    """
+    Main application routine
+    """
+
+    # Initialize the logging framework
+    Context.init_logging()
+
     # Handle cli args
     parser = argparse.ArgumentParser(
         prog="pgmon", description="A PostgreSQL monitoring agent"
@@ -796,33 +905,35 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     # Set the config file path
-    config_file = args.config_file
+    Context.config_file = args.config_file
 
     # Read the config file
-    read_config(config_file)
+    read_config(Context.config_file)
 
     # Run query tests and exit if test mode is enabled
     if args.test:
-        errors = test_queries()
-        if errors > 0:
+        if test_queries() > 0:
             sys.exit(1)
-        else:
-            sys.exit(0)
+        sys.exit(0)
 
     # Set up the http server to receive requests
-    server_address = (config["address"], config["port"])
-    httpd = ThreadingHTTPServer(server_address, SimpleHTTPRequestHandler)
+    server_address = (Context.config["address"], Context.config["port"])
+    Context.httpd = ThreadingHTTPServer(server_address, SimpleHTTPRequestHandler)
 
     # Set up the signal handler
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGHUP, signal_handler)
 
     # Handle requests.
-    log.info("Listening on port {}...".format(config["port"]))
-    while running:
-        httpd.handle_request()
+    Context.log.info("Listening on port %s...", Context.config["port"])
+    while Context.running:
+        Context.httpd.handle_request()
 
     # Clean up PostgreSQL connections
     # TODO: Improve this ... not sure it actually closes all the connections cleanly
-    for pool in connections.values():
+    for pool in Context.connections.values():
         pool.close()
+
+
+if __name__ == "__main__":
+    main()

zabbix_templates/coverage.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+
+# Compare the items defined in a Zabbix template with the metrics defined in a config file.
+
+import sys
+
+import yaml
+
+# Special built in metrics
+SPECIAL = {
+    "agent_version",
+    "latest_version_info"
+}
+
+
+class NonMetricItemError(Exception):
+    """
+    A given item does not directly use a metric
+    """
+
+
+def read_metrics(file):
+    """
+    Read the metrics from a config file and return the list of names
+
+    params:
+        file: the name of the file to read
+
+    returns:
+        list of metric named defined in the file
+
+    raises:
+        yaml.parser.ParserError: invalid yaml file
+    """
+    names = set()
+    config = None
+
+    with open(file, "r", encoding="utf-8") as f:
+        config = yaml.safe_load(f)
+
+    try:
+        for m in config["metrics"].keys():
+            names.add(m)
+    except KeyError:
+        pass
+
+    return names
+
+
+def extract_metric(item):
+    """
+    Extract the metric from an item definition
+
+    params:
+        item: the item/discovery/prototype definition dict
+
+    returns:
+        the name of the metric used in the item
+
+    raises:
+        NonMetricItemError: the item does not directly use a metric
+    """
+    try:
+        if item["type"] == "HTTP_AGENT":
+            url = item["url"]
+            if url.startswith("http://localhost:{$AGENT_PORT}"):
+                return url.split("/")[-1]
+    except KeyError:
+        raise NonMetricItemError()
+
+    raise NonMetricItemError()
+
+
+def read_template(file):
+    """
+    Read the items from a Zabbix template and return the list of metric names
+
+    params:
+        file: the name of the file to read
+
+    returns:
+        list of metric named used in the file
+
+    raises:
+        yaml.parser.ParserError: invalid yaml file
+    """
+    names = set()
+    config = None
+
+    with open(file, "r", encoding="utf-8") as f:
+        config = yaml.safe_load(f)
+
+    try:
+        for template in config["zabbix_export"]["templates"]:
+            for item in template["items"]:
+                try:
+                    names.add(extract_metric(item))
+                except NonMetricItemError:
+                    pass
+
+            for rule in template["discovery_rules"]:
+                try:
+                    names.add(extract_metric(rule))
+                except NonMetricItemError:
+                    pass
+
+                for proto in rule["item_prototypes"]:
+                    try:
+                        names.add(extract_metric(proto))
+                    except NonMetricItemError:
+                        pass
+    except KeyError:
+        pass
+
+    return names
+
+
+if __name__ == '__main__':
+    config_file = sys.argv[1]
+    config_metrics = read_metrics(config_file)
+
+    template_file = sys.argv[2]
+    template_metrics = read_template(template_file) - SPECIAL
+
+    config_only = config_metrics - template_metrics
+    template_only = template_metrics - config_metrics
+
+    print("Config only: {}".format(sorted(list(config_only))))
+    print("Template only: {}".format(sorted(list(template_only))))

zabbix_templates/pgmon_templates.yaml

@@ -95,6 +95,20 @@ zabbix_export:
           tags:
             - tag: Application
               value: PostgreSQL
+        - uuid: 06b1d082ed1e4796bc31cc25f7db6326
+          name: 'PostgreSQL Backend IO Info'
+          type: HTTP_AGENT
+          key: 'pgmon[io_per_backend]'
+          history: '0'
+          value_type: TEXT
+          trends: '0'
+          description: 'Aggregated statistics about I/O activity for different backend types'
+          url: 'http://localhost:{$AGENT_PORT}/io_per_backend'
+          tags:
+            - tag: Application
+              value: PostgreSQL
+            - tag: Type
+              value: Raw
         - uuid: d890e395fbbc4f2bacbd50e7321fcb9f
           name: 'PostgreSQL latest version info'
           type: HTTP_AGENT
@@ -153,7 +167,8 @@ zabbix_export:
                 operator: NOT_MATCHES_REGEX
                 formulaid: A
           lifetime: 30d
-          enabled_lifetime_type: DISABLE_NEVER
+          enabled_lifetime_type: DISABLE_AFTER
+          enabled_lifetime: 1d
           item_prototypes:
             - uuid: a30babe4a6f4440bba2a3ee46eff7ce2
               name: 'Time spent executing statements on {#DBNAME}'
@@ -771,6 +786,77 @@ zabbix_export:
                   value: PostgreSQL
                 - tag: Database
                   value: '{#DBNAME}'
+            - uuid: 5960120dd01c4926b0fc1fbe9c011507
+              name: 'Database max sequence usage in {#DBNAME}'
+              type: HTTP_AGENT
+              key: 'pgmon_db_max_sequence[{#DBNAME}]'
+              delay: 5m
+              value_type: FLOAT
+              units: '%'
+              description: 'The percent of the currently configured value range for the most utilized sequence.'
+              url: 'http://localhost:{$AGENT_PORT}/sequence_usage'
+              query_fields:
+                - name: dbname
+                  value: '{#DBNAME}'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Database
+                  value: '{#DBNAME}'
+            - uuid: 48b9cc80ac4d4aee9e9f3a5d6f7d4a95
+              name: 'Total number of sequences on {#DBNAME}'
+              type: DEPENDENT
+              key: 'pgmon_db_sequences[total,{#DBNAME}]'
+              delay: '0'
+              description: 'Total number of sequences in the database.'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - $.total_sequences
+              master_item:
+                key: 'pgmon_db_sequence_visibility[{#DBNAME}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Database
+                  value: '{#DBNAME}'
+            - uuid: 6521a9bab2ac47bf85429832d289bbac
+              name: 'Visible sequences on {#DBNAME}'
+              type: DEPENDENT
+              key: 'pgmon_db_sequences[visible,{#DBNAME}]'
+              delay: '0'
+              description: 'Number of sequences in the database for which Zabbix can see stats.'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - $.visible_sequences
+              master_item:
+                key: 'pgmon_db_sequence_visibility[{#DBNAME}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Database
+                  value: '{#DBNAME}'
+            - uuid: 00f2da3eb99940839410a6ecd5df153f
+              name: 'Database sequence visibility in {#DBNAME}'
+              type: HTTP_AGENT
+              key: 'pgmon_db_sequence_visibility[{#DBNAME}]'
+              delay: 30m
+              history: '0'
+              value_type: TEXT
+              trends: '0'
+              description: 'Statistics about the number of sequences that exist and the number Zabbix can actually see stats for.'
+              url: 'http://localhost:{$AGENT_PORT}/sequence_visibility'
+              query_fields:
+                - name: dbname
+                  value: '{#DBNAME}'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Database
+                  value: '{#DBNAME}'
+                - tag: Type
+                  value: Raw
             - uuid: 492b3cac15f348c2b85f97b69c114d1b
               name: 'Database Stats for {#DBNAME}'
               type: HTTP_AGENT
@@ -789,6 +875,17 @@ zabbix_export:
                   value: '{#DBNAME}'
                 - tag: Type
                   value: Raw
+          trigger_prototypes:
+            - uuid: d29d0fd9d9d34b5ebd649592b0829ce5
+              expression: 'last(/PostgreSQL by pgmon/pgmon_db_sequences[total,{#DBNAME}]) <> last(/PostgreSQL by pgmon/pgmon_db_sequences[visible,{#DBNAME}])'
+              name: 'Sequences not visible to Zabbix on {#DBNAME}'
+              priority: WARNING
+              description: 'There are sequences for which Zabbix cannot see usage statistics'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: Component
+                  value: Sequence
           graph_prototypes:
             - uuid: 1f7de43b77714f819e61c31273712b70
               name: 'DML Totals for {#DBNAME}'
@@ -881,13 +978,613 @@ zabbix_export:
           lld_macro_paths:
             - lld_macro: '{#DBNAME}'
               path: $.dbname
+        - uuid: 6afbe12fb9f54fa2ad29d647429eb16e
+          name: 'Discover I/O Backend Types'
+          type: DEPENDENT
+          key: pgmon_discover_io_backend_types
+          delay: '0'
+          lifetime: 30d
+          enabled_lifetime_type: DISABLE_AFTER
+          enabled_lifetime: 1h
+          item_prototypes:
+            - uuid: b1ac2e56b30f4812bf33ce973ef16b10
+              name: 'I/O Evictions by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[evictions,{#BACKEND_TYPE}]'
+              delay: '0'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].evictions.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: bde8e6c28e714be58fa963a395e14619
+              name: 'I/O Extends by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[extends,{#BACKEND_TYPE}]'
+              delay: '0'
+              units: B
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].extends.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 5ce7a815c19241a6a398e17539cca1d4
+              name: 'I/O fsyncs by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[fsyncs,{#BACKEND_TYPE}]'
+              delay: '0'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].fsyncs.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 3be8cb08b6bc42f7888d8d7877ab759f
+              name: 'I/O Hits by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[hits,{#BACKEND_TYPE}]'
+              delay: '0'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].hits.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: e28566682196415fb3bdad9d9a294ae9
+              name: 'I/O Reads by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[reads,{#BACKEND_TYPE}]'
+              delay: '0'
+              units: B
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].reads.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: e5e9b21c7f7649b284a8ca5b4502cb96
+              name: 'I/O Reuses by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[reuses,{#BACKEND_TYPE}]'
+              delay: '0'
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].reuses.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 6de5bc1a98dc4d8da23ac0a90983c8aa
+              name: 'I/O Writebacks by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[writebacks,{#BACKEND_TYPE}]'
+              delay: '0'
+              units: B
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].writebacks.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 87d09d3891c145a898050d390ea7f16e
+              name: 'I/O Writes by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend[writes,{#BACKEND_TYPE}]'
+              delay: '0'
+              units: B
+              preprocessing:
+                - type: JSONPATH
+                  parameters:
+                    - '$[?(@.backend_type == "{#BACKEND_TYPE}")].writes.first()'
+              master_item:
+                key: 'pgmon[io_per_backend]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 1587592caf694d91ba9457cafaca7761
+              name: 'I/O Evictions by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[evictions,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              params: 'last(//pgmon_io_backend[evictions,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[evictions,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 637b0601458147b5b72bc40ab22b2117
+              name: 'I/O Evictions Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[evictions,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[evictions,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: c7ed8e1bcb0945ffa2a36b06ea5fe0c5
+              name: 'I/O Extends by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[extends,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              units: B/h
+              params: 'last(//pgmon_io_backend[extends,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[extends,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 855cbbc83d884b00a6fea457f8e09806
+              name: 'I/O Extends Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[extends,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              units: Bps
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[extends,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 20c6135bb6c8491b9ca68fa506232b04
+              name: 'I/O fsyncs by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[fsyncs,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              params: 'last(//pgmon_io_backend[fsyncs,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[fsyncs,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 7391c372d1704f21b314379c2fe5bb64
+              name: 'I/O fsyncs Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[fsyncs,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[fsyncs,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: ab8d0766bbf64db3bb09f9442d268a83
+              name: 'I/O Hits by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[hits,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              params: 'last(//pgmon_io_backend[hits,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[hits,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: be9aa056b9364c12a8809d5dd3af8387
+              name: 'I/O Hits Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[hits,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[hits,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 0c42a40a263948dda4464856747ac6ef
+              name: 'I/O Reads by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[reads,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              units: B/h
+              params: 'last(//pgmon_io_backend[reads,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[reads,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 01bc131aaec448e6aafb7ef13d1ea5ed
+              name: 'I/O Reads Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[reads,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              units: Bps
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[reads,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 740d5c84be6e4b99b4ca3513b6c6c5ea
+              name: 'I/O Reuses by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[reuses,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              params: 'last(//pgmon_io_backend[reuses,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[reuses,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 8cbc16fc472a44d5945498285e8407be
+              name: 'I/O Reuses Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[reuses,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[reuses,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 2e796f4c75a94967a0b11a37a2d64726
+              name: 'I/O Writebacks by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[writebacks,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              units: B/h
+              params: 'last(//pgmon_io_backend[writebacks,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[writebacks,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 52d2536db79047948893ccb764b2a220
+              name: 'I/O Writebacks Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[writebacks,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              units: Bps
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[writebacks,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 6a741d3b857b48eea0e1a4d1deea7183
+              name: 'I/O Writes by {#BACKEND_TYPE} - 1h delta'
+              type: CALCULATED
+              key: 'pgmon_io_backend_delta[writes,1h,{#BACKEND_TYPE}]'
+              delay: 10m
+              units: B/h
+              params: 'last(//pgmon_io_backend[writes,{#BACKEND_TYPE}]) - last(//pgmon_io_backend[writes,{#BACKEND_TYPE}], #1:now-1h)'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+            - uuid: 6d10339b08b84a91a931a64050a471cc
+              name: 'I/O Writes Rate by {#BACKEND_TYPE}'
+              type: DEPENDENT
+              key: 'pgmon_io_backend_delta[writes,{#BACKEND_TYPE}]'
+              delay: '0'
+              value_type: FLOAT
+              units: Bps
+              preprocessing:
+                - type: CHANGE_PER_SECOND
+                  parameters:
+                    - ''
+              master_item:
+                key: 'pgmon_io_backend[writes,{#BACKEND_TYPE}]'
+              tags:
+                - tag: Application
+                  value: PostgreSQL
+                - tag: 'Backend Type'
+                  value: '{#BACKEND_TYPE}'
+                - tag: Component
+                  value: IO
+          graph_prototypes:
+            - uuid: faf1e3f41dd84adba42c414f6efac152
+              name: 'Backend I/O for {#BACKEND_TYPE}'
+              graph_items:
+                - color: 199C0D
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[evictions,{#BACKEND_TYPE}]'
+                - sortorder: '1'
+                  color: F63100
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[extends,{#BACKEND_TYPE}]'
+                - sortorder: '2'
+                  color: 2774A4
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[fsyncs,{#BACKEND_TYPE}]'
+                - sortorder: '3'
+                  color: F7941D
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[hits,{#BACKEND_TYPE}]'
+                - sortorder: '4'
+                  color: FC6EA3
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[reads,{#BACKEND_TYPE}]'
+                - sortorder: '5'
+                  color: 6C59DC
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[reuses,{#BACKEND_TYPE}]'
+                - sortorder: '6'
+                  color: C7A72D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[writebacks,{#BACKEND_TYPE}]'
+                - sortorder: '7'
+                  color: BA2A5D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend[writes,{#BACKEND_TYPE}]'
+            - uuid: 24e2e3ec67cd43058c8b6a02d686fc6a
+              name: 'Backend I/O for {#BACKEND_TYPE} - 1h delta'
+              graph_items:
+                - color: 199C0D
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[evictions,1h,{#BACKEND_TYPE}]'
+                - sortorder: '1'
+                  color: F63100
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[extends,1h,{#BACKEND_TYPE}]'
+                - sortorder: '2'
+                  color: 2774A4
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[fsyncs,1h,{#BACKEND_TYPE}]'
+                - sortorder: '3'
+                  color: F7941D
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[hits,1h,{#BACKEND_TYPE}]'
+                - sortorder: '4'
+                  color: FC6EA3
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[reads,1h,{#BACKEND_TYPE}]'
+                - sortorder: '5'
+                  color: 6C59DC
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[reuses,1h,{#BACKEND_TYPE}]'
+                - sortorder: '6'
+                  color: C7A72D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[writebacks,1h,{#BACKEND_TYPE}]'
+                - sortorder: '7'
+                  color: BA2A5D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[writes,1h,{#BACKEND_TYPE}]'
+            - uuid: 815f1a00c19d45a18b10846644710668
+              name: 'Backend I/O Rates for {#BACKEND_TYPE}'
+              graph_items:
+                - color: 199C0D
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[evictions,{#BACKEND_TYPE}]'
+                - sortorder: '1'
+                  color: F63100
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[extends,{#BACKEND_TYPE}]'
+                - sortorder: '2'
+                  color: 2774A4
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[fsyncs,{#BACKEND_TYPE}]'
+                - sortorder: '3'
+                  color: F7941D
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[hits,{#BACKEND_TYPE}]'
+                - sortorder: '4'
+                  color: FC6EA3
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[reads,{#BACKEND_TYPE}]'
+                - sortorder: '5'
+                  color: 6C59DC
+                  yaxisside: RIGHT
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[reuses,{#BACKEND_TYPE}]'
+                - sortorder: '6'
+                  color: C7A72D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[writebacks,{#BACKEND_TYPE}]'
+                - sortorder: '7'
+                  color: BA2A5D
+                  calc_fnc: ALL
+                  item:
+                    host: 'PostgreSQL by pgmon'
+                    key: 'pgmon_io_backend_delta[writes,{#BACKEND_TYPE}]'
+          master_item:
+            key: 'pgmon[io_per_backend]'
+          lld_macro_paths:
+            - lld_macro: '{#BACKEND_TYPE}'
+              path: $.backend_type
+          preprocessing:
+            - type: JSONPATH
+              parameters:
+                - '$[:]'
         - uuid: 8ec029d577ae4872858e2e5cfd1cc40e
           name: 'Discover Replication'
           type: HTTP_AGENT
           key: pgmon_discover_rep
           delay: 10m
+          filter:
+            conditions:
+              - macro: '{#APPLICATION_NAME}'
+                value: '^pg_[0-9]+_sync_[0-9]+_[0-9]+$'
+                operator: NOT_MATCHES_REGEX
+                formulaid: A
           lifetime: 30d
-          enabled_lifetime_type: DISABLE_NEVER
+          enabled_lifetime_type: DISABLE_AFTER
+          enabled_lifetime: 7d
           item_prototypes:
             - uuid: 3a5a60620e6a4db694e47251148d82f5
               name: 'Flush lag for {#REPID}'
@@ -896,6 +1593,7 @@ zabbix_export:
               delay: '0'
               history: 90d
               value_type: FLOAT
+              units: s
               description: 'Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written and flushed it (but not yet applied it). This can be used to gauge the delay that synchronous_commit level on incurred while committing if this server was configured as a synchronous standby.'
               preprocessing:
                 - type: JSONPATH
@@ -911,13 +1609,12 @@ zabbix_export:
                 - tag: Database
                   value: '{#DBNAME}'
             - uuid: 624f8f085a3642c9a10a03361c17763d
-              name: 'Last flush LSN for {#REPID}'
+              name: 'Last flush LSN lag for {#REPID}'
               type: DEPENDENT
               key: 'pgmon_rep[flush_lsn,repid={#REPID}]'
               delay: '0'
               history: 90d
-              value_type: TEXT
-              trends: '0'
+              units: B
               description: 'Last write-ahead log location flushed to disk by this standby server'
               preprocessing:
                 - type: JSONPATH
@@ -939,6 +1636,7 @@ zabbix_export:
               delay: '0'
               history: 90d
               value_type: FLOAT
+              units: s
               description: 'Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written, flushed and applied it. This can be used to gauge the delay that synchronous_commit level remote_apply incurred while committing if this server was configured as a synchronous standby.'
               preprocessing:
                 - type: JSONPATH
@@ -954,13 +1652,12 @@ zabbix_export:
                 - tag: Database
                   value: '{#DBNAME}'
             - uuid: fe1bed51845d4694bae8f53deed4846d
-              name: 'Last replay LSN for {#REPID}'
+              name: 'Last replay LSN lag for {#REPID}'
               type: DEPENDENT
               key: 'pgmon_rep[replay_lsn,repid={#REPID}]'
               delay: '0'
               history: 90d
-              value_type: TEXT
-              trends: '0'
+              units: B
               description: 'Last write-ahead log location replayed into the database on this standby server'
               preprocessing:
                 - type: JSONPATH
@@ -976,13 +1673,12 @@ zabbix_export:
                 - tag: Database
                   value: '{#DBNAME}'
             - uuid: 68c179d0e33f45f9bf82d2d4125763f0
-              name: 'Last sent LSN for {#REPID}'
+              name: 'Last sent LSN lag for {#REPID}'
               type: DEPENDENT
               key: 'pgmon_rep[sent_lsn,repid={#REPID}]'
               delay: '0'
               history: 90d
-              value_type: TEXT
-              trends: '0'
+              units: B
               description: 'Last write-ahead log location sent on this connection'
               preprocessing:
                 - type: JSONPATH
@@ -1032,6 +1728,7 @@ zabbix_export:
               delay: '0'
               history: 90d
               value_type: FLOAT
+              units: s
               description: 'Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written it (but not yet flushed it or applied it). This can be used to gauge the delay that synchronous_commit level remote_write incurred while committing if this server was configured as a synchronous standby.'
               preprocessing:
                 - type: JSONPATH
@@ -1047,13 +1744,12 @@ zabbix_export:
                 - tag: Database
                   value: '{#DBNAME}'
             - uuid: 57fb03cf63af4b0a91d8e36d6ff64d30
-              name: 'Last write LSN for {#REPID}'
+              name: 'Last write LSN lag for {#REPID}'
               type: DEPENDENT
               key: 'pgmon_rep[write_lsn,repid={#REPID}]'
               delay: '0'
               history: 90d
-              value_type: TEXT
-              trends: '0'
+              units: B
               description: 'Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written it (but not yet flushed it or applied it). This can be used to gauge the delay that synchronous_commit level remote_write incurred while committing if this server was configured as a synchronous standby.'
               preprocessing:
                 - type: JSONPATH
@@ -1090,6 +1786,8 @@ zabbix_export:
                   value: Raw
           url: 'http://localhost:{$AGENT_PORT}/discover_rep'
           lld_macro_paths:
+            - lld_macro: '{#APPLICATION_NAME}'
+              path: $.application_name
             - lld_macro: '{#CLIENT_ADDR}'
               path: $.client_addr
             - lld_macro: '{#REPID}'
@@ -1101,6 +1799,15 @@ zabbix_export:
           type: HTTP_AGENT
           key: pgmon_discover_slots
           delay: 10m
+          filter:
+            conditions:
+              - macro: '{#SLOT_NAME}'
+                value: '^pg_[0-9]+_sync_[0-9]+_[0-9]+$'
+                operator: NOT_MATCHES_REGEX
+                formulaid: A
+          lifetime: 30d
+          enabled_lifetime_type: DISABLE_AFTER
+          enabled_lifetime: 7d
           item_prototypes:
             - uuid: 536c5f82e3074ddfbfd842b3a2e8d46c
               name: 'Slot {#SLOT_NAME} - Confirmed Flushed Bytes Lag'