Compare commits
73 Commits
v1.0.1
...
dev/tmpfil
| Author | SHA1 | Date | |
|---|---|---|---|
|
9b48535a09
|
|||
|
590425fae7
|
|||
|
174cac4ca6
|
|||
|
f740c4012d
|
|||
|
c1dcc6c87b
|
|||
|
4fa3a7b669
|
|||
|
8182ad75b9
|
|||
|
175fd85f9f
|
|||
|
e9c97d65cb
|
|||
|
af5d9cf186
|
|||
|
17c15e7e48
|
|||
|
f55bcdcfae
|
|||
|
e7b97a9e88
|
|||
|
3ade30c04a
|
|||
|
ab039dc412
|
|||
|
75c5d76047
|
|||
|
43cd162313
|
|||
|
29bfd07dad
|
|||
|
60589c2058
|
|||
|
ea3aca3455
|
|||
|
cc71547f5f
|
|||
|
107d5056d6
|
|||
|
e930178e9c
|
|||
|
5afc940df8
|
|||
|
3961aa3448
|
|||
|
8fd57032e7
|
|||
|
5ede7dea07
|
|||
|
5ea007c3f6
|
|||
|
7cb0f7ad40
|
|||
|
83fa12ec54
|
|||
|
98b74d9aed
|
|||
|
45953848e2
|
|||
|
6116f4f885
|
|||
|
24d1214855
|
|||
|
3c39d8aa97
|
|||
|
ebb084aa9d
|
|||
|
86d5e8917b
|
|||
|
be84e2b89a
|
|||
|
fd0a9e1230
|
|||
|
b934a30124
|
|||
|
ea397ef889
|
|||
|
def6bab7f4
|
|||
|
ecb616f6d9
|
|||
|
b7f731c6ac
|
|||
|
4fba81dc2c
|
|||
|
9225e745b0
|
|||
|
b981a9ad36
|
|||
|
c0185d4b86
|
|||
|
d34dfc5bf7
|
|||
|
7c395a80fe
|
|||
|
8fe81e3ab3
|
|||
|
cfe01eb63e
|
|||
|
55b9b16deb
|
|||
|
6a22597f1f
|
|||
|
295d1d6310
|
|||
|
8b804f3912
|
|||
|
bc9e039cb3
|
|||
|
54cf117591
|
|||
|
8e32e01c20
|
|||
|
af3bbf7515
|
|||
|
180fa31d14
|
|||
|
39a6a9d23e
|
|||
|
f716569aa7
|
|||
|
22ae634a87
|
|||
|
375bf6a982
|
|||
|
487386a7cc
|
|||
|
15097dcba4
|
|||
|
1d642d41b2
|
|||
|
80304f40d1
|
|||
|
c0e1531083
|
|||
|
529bef9679
|
|||
|
8928bba337
|
|||
|
c872fc6b90
|
1
.gitignore
vendored
1
.gitignore
vendored
@@ -8,3 +8,4 @@ missing
|
||||
__pycache__
|
||||
venv
|
||||
build
|
||||
.coverage
|
||||
|
||||
@@ -3,7 +3,7 @@ Version: 1.0
|
||||
Section: utils
|
||||
Priority: optional
|
||||
Architecture: all
|
||||
Depends: logrotate, python3 (>= 3.6), python3-psycopg2, python3-yaml, systemd
|
||||
Depends: logrotate, python3 (>= 3.6), python3-psycopg2, python3-requests, python3-yaml, systemd
|
||||
Maintainer: James Campbell <james@commandprompt.com>
|
||||
Homepage: https://www.commandprompt.com
|
||||
Description: A bridge to sit between monitoring tools and PostgreSQL
|
||||
|
||||
73
GENTOO/pgmon-1.0.2.ebuild
Normal file
73
GENTOO/pgmon-1.0.2.ebuild
Normal file
@@ -0,0 +1,73 @@
|
||||
# Copyright 2024 Gentoo Authors
|
||||
# Distributed under the terms of the GNU General Public License v2
|
||||
|
||||
EAPI=8
|
||||
|
||||
PYTHON_COMPAT=( python3_{6..13} )
|
||||
|
||||
inherit python-r1
|
||||
|
||||
DESCRIPTION="PostgreSQL monitoring bridge"
|
||||
HOMEPAGE="None"
|
||||
|
||||
LICENSE="BSD"
|
||||
SLOT="0"
|
||||
KEYWORDS="amd64"
|
||||
|
||||
SRC_URI="https://code2.shh-dot-com.org/james/${PN}/archive/v${PV}.tar.bz2 -> ${P}.tar.bz2"
|
||||
|
||||
IUSE="-systemd"
|
||||
|
||||
DEPEND="
|
||||
${PYTHON_DEPS}
|
||||
dev-python/psycopg:2
|
||||
dev-python/pyyaml
|
||||
app-admin/logrotate
|
||||
"
|
||||
RDEPEND="${DEPEND}"
|
||||
BDEPEND=""
|
||||
|
||||
RESTRICT="fetch"
|
||||
|
||||
#S="${WORKDIR}/${PN}"
|
||||
|
||||
pkg_nofetch() {
|
||||
einfo "Please download"
|
||||
einfo " - ${P}.tar.bz2"
|
||||
einfo "from ${HOMEPAGE} and place it in your DISTDIR directory."
|
||||
einfo "The file should be owned by portage:portage."
|
||||
}
|
||||
|
||||
src_compile() {
|
||||
true
|
||||
}
|
||||
|
||||
src_install() {
|
||||
# Install init script
|
||||
if ! use systemd ; then
|
||||
newinitd "openrc/pgmon.initd" pgmon
|
||||
newconfd "openrc/pgmon.confd" pgmon
|
||||
fi
|
||||
|
||||
# Install systemd unit
|
||||
if use systemd ; then
|
||||
systemd_dounit "systemd/pgmon.service"
|
||||
fi
|
||||
|
||||
# Install script
|
||||
exeinto /usr/bin
|
||||
newexe "src/pgmon.py" pgmon
|
||||
|
||||
# Install default config
|
||||
diropts -o root -g root -m 0755
|
||||
insinto /etc/pgmon
|
||||
doins "sample-config/pgmon.yml"
|
||||
doins "sample-config/pgmon-metrics.yml"
|
||||
|
||||
# Install logrotate config
|
||||
insinto /etc/logrotate.d
|
||||
newins "logrotate/pgmon.logrotate" pgmon
|
||||
|
||||
# Install man page
|
||||
doman manpages/pgmon.1
|
||||
}
|
||||
74
GENTOO/pgmon-1.0.3.ebuild
Normal file
74
GENTOO/pgmon-1.0.3.ebuild
Normal file
@@ -0,0 +1,74 @@
|
||||
# Copyright 2024 Gentoo Authors
|
||||
# Distributed under the terms of the GNU General Public License v2
|
||||
|
||||
EAPI=8
|
||||
|
||||
PYTHON_COMPAT=( python3_{6..13} )
|
||||
|
||||
inherit python-r1 systemd
|
||||
|
||||
DESCRIPTION="PostgreSQL monitoring bridge"
|
||||
HOMEPAGE="None"
|
||||
|
||||
LICENSE="BSD"
|
||||
SLOT="0"
|
||||
KEYWORDS="amd64"
|
||||
|
||||
SRC_URI="https://code2.shh-dot-com.org/james/${PN}/releases/download/v${PV}/${P}.tar.bz2"
|
||||
|
||||
IUSE="-systemd"
|
||||
|
||||
DEPEND="
|
||||
${PYTHON_DEPS}
|
||||
dev-python/psycopg:2
|
||||
dev-python/pyyaml
|
||||
dev-python/requests
|
||||
app-admin/logrotate
|
||||
"
|
||||
RDEPEND="${DEPEND}"
|
||||
BDEPEND=""
|
||||
|
||||
#RESTRICT="fetch"
|
||||
|
||||
#S="${WORKDIR}/${PN}"
|
||||
|
||||
#pkg_nofetch() {
|
||||
# einfo "Please download"
|
||||
# einfo " - ${P}.tar.bz2"
|
||||
# einfo "from ${HOMEPAGE} and place it in your DISTDIR directory."
|
||||
# einfo "The file should be owned by portage:portage."
|
||||
#}
|
||||
|
||||
src_compile() {
|
||||
true
|
||||
}
|
||||
|
||||
src_install() {
|
||||
# Install init script
|
||||
if ! use systemd ; then
|
||||
newinitd "openrc/pgmon.initd" pgmon
|
||||
newconfd "openrc/pgmon.confd" pgmon
|
||||
fi
|
||||
|
||||
# Install systemd unit
|
||||
if use systemd ; then
|
||||
systemd_dounit "systemd/pgmon.service"
|
||||
fi
|
||||
|
||||
# Install script
|
||||
exeinto /usr/bin
|
||||
newexe "src/pgmon.py" pgmon
|
||||
|
||||
# Install default config
|
||||
diropts -o root -g root -m 0755
|
||||
insinto /etc/pgmon
|
||||
doins "sample-config/pgmon.yml"
|
||||
doins "sample-config/pgmon-metrics.yml"
|
||||
|
||||
# Install logrotate config
|
||||
insinto /etc/logrotate.d
|
||||
newins "logrotate/pgmon.logrotate" pgmon
|
||||
|
||||
# Install man page
|
||||
doman manpages/pgmon.1
|
||||
}
|
||||
74
GENTOO/pgmon-1.0.4.ebuild
Normal file
74
GENTOO/pgmon-1.0.4.ebuild
Normal file
@@ -0,0 +1,74 @@
|
||||
# Copyright 2024 Gentoo Authors
|
||||
# Distributed under the terms of the GNU General Public License v2
|
||||
|
||||
EAPI=8
|
||||
|
||||
PYTHON_COMPAT=( python3_{6..13} )
|
||||
|
||||
inherit python-r1 systemd
|
||||
|
||||
DESCRIPTION="PostgreSQL monitoring bridge"
|
||||
HOMEPAGE="None"
|
||||
|
||||
LICENSE="BSD"
|
||||
SLOT="0"
|
||||
KEYWORDS="amd64"
|
||||
|
||||
SRC_URI="https://code2.shh-dot-com.org/james/${PN}/releases/download/v${PV}/${P}.tar.bz2"
|
||||
|
||||
IUSE="-systemd"
|
||||
|
||||
DEPEND="
|
||||
${PYTHON_DEPS}
|
||||
dev-python/psycopg:2
|
||||
dev-python/pyyaml
|
||||
dev-python/requests
|
||||
app-admin/logrotate
|
||||
"
|
||||
RDEPEND="${DEPEND}"
|
||||
BDEPEND=""
|
||||
|
||||
#RESTRICT="fetch"
|
||||
|
||||
#S="${WORKDIR}/${PN}"
|
||||
|
||||
#pkg_nofetch() {
|
||||
# einfo "Please download"
|
||||
# einfo " - ${P}.tar.bz2"
|
||||
# einfo "from ${HOMEPAGE} and place it in your DISTDIR directory."
|
||||
# einfo "The file should be owned by portage:portage."
|
||||
#}
|
||||
|
||||
src_compile() {
|
||||
true
|
||||
}
|
||||
|
||||
src_install() {
|
||||
# Install init script
|
||||
if ! use systemd ; then
|
||||
newinitd "openrc/pgmon.initd" pgmon
|
||||
newconfd "openrc/pgmon.confd" pgmon
|
||||
fi
|
||||
|
||||
# Install systemd unit
|
||||
if use systemd ; then
|
||||
systemd_dounit "systemd/pgmon.service"
|
||||
fi
|
||||
|
||||
# Install script
|
||||
exeinto /usr/bin
|
||||
newexe "src/pgmon.py" pgmon
|
||||
|
||||
# Install default config
|
||||
diropts -o root -g root -m 0755
|
||||
insinto /etc/pgmon
|
||||
doins "sample-config/pgmon.yml"
|
||||
doins "sample-config/pgmon-metrics.yml"
|
||||
|
||||
# Install logrotate config
|
||||
insinto /etc/logrotate.d
|
||||
newins "logrotate/pgmon.logrotate" pgmon
|
||||
|
||||
# Install man page
|
||||
doman manpages/pgmon.1
|
||||
}
|
||||
74
GENTOO/pgmon-1.1.0-rc1.ebuild
Normal file
74
GENTOO/pgmon-1.1.0-rc1.ebuild
Normal file
@@ -0,0 +1,74 @@
|
||||
# Copyright 2024 Gentoo Authors
|
||||
# Distributed under the terms of the GNU General Public License v2
|
||||
|
||||
EAPI=8
|
||||
|
||||
PYTHON_COMPAT=( python3_{6..13} )
|
||||
|
||||
inherit python-r1 systemd
|
||||
|
||||
DESCRIPTION="PostgreSQL monitoring bridge"
|
||||
HOMEPAGE="None"
|
||||
|
||||
LICENSE="BSD"
|
||||
SLOT="0"
|
||||
KEYWORDS="amd64"
|
||||
|
||||
SRC_URI="https://code2.shh-dot-com.org/james/${PN}/releases/download/v${PV}/${P}.tar.bz2"
|
||||
|
||||
IUSE="-systemd"
|
||||
|
||||
DEPEND="
|
||||
${PYTHON_DEPS}
|
||||
dev-python/psycopg:2
|
||||
dev-python/pyyaml
|
||||
dev-python/requests
|
||||
app-admin/logrotate
|
||||
"
|
||||
RDEPEND="${DEPEND}"
|
||||
BDEPEND=""
|
||||
|
||||
#RESTRICT="fetch"
|
||||
|
||||
#S="${WORKDIR}/${PN}"
|
||||
|
||||
#pkg_nofetch() {
|
||||
# einfo "Please download"
|
||||
# einfo " - ${P}.tar.bz2"
|
||||
# einfo "from ${HOMEPAGE} and place it in your DISTDIR directory."
|
||||
# einfo "The file should be owned by portage:portage."
|
||||
#}
|
||||
|
||||
src_compile() {
|
||||
true
|
||||
}
|
||||
|
||||
src_install() {
|
||||
# Install init script
|
||||
if ! use systemd ; then
|
||||
newinitd "openrc/pgmon.initd" pgmon
|
||||
newconfd "openrc/pgmon.confd" pgmon
|
||||
fi
|
||||
|
||||
# Install systemd unit
|
||||
if use systemd ; then
|
||||
systemd_dounit "systemd/pgmon.service"
|
||||
fi
|
||||
|
||||
# Install script
|
||||
exeinto /usr/bin
|
||||
newexe "src/pgmon.py" pgmon
|
||||
|
||||
# Install default config
|
||||
diropts -o root -g root -m 0755
|
||||
insinto /etc/pgmon
|
||||
doins "sample-config/pgmon.yml"
|
||||
doins "sample-config/pgmon-metrics.yml"
|
||||
|
||||
# Install logrotate config
|
||||
insinto /etc/logrotate.d
|
||||
newins "logrotate/pgmon.logrotate" pgmon
|
||||
|
||||
# Install man page
|
||||
doman manpages/pgmon.1
|
||||
}
|
||||
287
Makefile
287
Makefile
@@ -1,9 +1,32 @@
|
||||
# Package details
|
||||
PACKAGE_NAME := pgmon
|
||||
VERSION := 1.0.1
|
||||
|
||||
SCRIPT := src/$(PACKAGE_NAME).py
|
||||
|
||||
# Figure out the version components
|
||||
# Note: The release is for RPM packages, where prerelease releases are written as 0.<release>
|
||||
FULL_VERSION := $(shell grep -m 1 '^VERSION = ' "$(SCRIPT)" | sed -ne 's/.*"\(.*\)".*/\1/p')
|
||||
VERSION := $(shell echo $(FULL_VERSION) | sed -n 's/\(.*\)\(-rc.*\|$$\)/\1/p')
|
||||
RELEASE := $(shell echo $(FULL_VERSION) | sed -n 's/.*-rc\([0-9]\+\)$$/\1/p')
|
||||
|
||||
# Package version formatting
|
||||
ifeq ($(RELEASE),)
|
||||
RPM_RELEASE := 1
|
||||
RPM_VERSION := $(VERSION)-$(RPM_RELEASE)
|
||||
DEB_VERSION := $(VERSION)
|
||||
else
|
||||
RPM_RELEASE := 0.$(RELEASE)
|
||||
RPM_VERSION := $(VERSION)-$(RPM_RELEASE)
|
||||
DEB_VERSION := $(VERSION)~rc$(RELEASE)
|
||||
endif
|
||||
|
||||
# Python stuff
|
||||
PYTHON ?= python3
|
||||
PYLINT ?= pylint
|
||||
BLACK ?= black
|
||||
|
||||
HAVE_COVERAGE := $(shell $(PYTHON) -c 'import coverage' 2>/dev/null && echo yes)
|
||||
|
||||
# Where packages are built
|
||||
BUILD_DIR := build
|
||||
|
||||
@@ -11,32 +34,146 @@ BUILD_DIR := build
|
||||
SUPPORTED := ubuntu-20.04 \
|
||||
ubuntu-22.04 \
|
||||
ubuntu-24.04 \
|
||||
debian-10 \
|
||||
debian-11 \
|
||||
debian-12 \
|
||||
debian-13 \
|
||||
rockylinux-8 \
|
||||
rockylinux-9 \
|
||||
rockylinux-10 \
|
||||
oraclelinux-7 \
|
||||
amazonlinux-2 \
|
||||
amazonlinux-2023 \
|
||||
gentoo
|
||||
|
||||
##
|
||||
# These targets are the main ones to use for most things.
|
||||
##
|
||||
|
||||
.PHONY: all clean tgz test install
|
||||
.PHONY: all clean tgz lint format test query-tests template-coverage
|
||||
|
||||
all: package-all
|
||||
|
||||
# Dump information related to the current version
|
||||
version:
|
||||
@echo "full version=$(FULL_VERSION) version=$(VERSION) rel=$(RELEASE) rpm=$(RPM_VERSION) deb=$(DEB_VERSION)"
|
||||
|
||||
# Clean up the build directory
|
||||
clean:
|
||||
rm -rf $(BUILD_DIR)
|
||||
|
||||
# Check for lint
|
||||
lint:
|
||||
$(PYLINT) src/pgmon.py
|
||||
$(PYLINT) src/test_pgmon.py
|
||||
$(BLACK) --check --diff src/pgmon.py
|
||||
$(BLACK) --check --diff src/test_pgmon.py
|
||||
|
||||
# Format the code using black
|
||||
format:
|
||||
$(BLACK) src/pgmon.py
|
||||
$(BLACK) src/test_pgmon.py
|
||||
|
||||
# Run unit tests for the script (with coverage if it's available)
|
||||
test:
|
||||
ifeq ($(HAVE_COVERAGE),yes)
|
||||
cd src ; $(PYTHON) -m coverage run -m unittest && python3 -m coverage report -m
|
||||
else
|
||||
cd src ; $(PYTHON) -m unittest
|
||||
endif
|
||||
|
||||
# Run query tests
|
||||
query-tests:
|
||||
cd tests ; ./run-tests.sh
|
||||
|
||||
# Compare the sample metrics with the Zabbix template
|
||||
template-coverage:
|
||||
$(PYTHON) zabbix_templates/coverage.py sample-config/pgmon-metrics.yml zabbix_templates/pgmon_templates.yaml
|
||||
|
||||
# Create a tarball (openrc version)
|
||||
tgz:
|
||||
rm -rf $(BUILD_DIR)/tgz/root
|
||||
mkdir -p $(BUILD_DIR)/tgz/root
|
||||
$(MAKE) install-openrc DESTDIR=$(BUILD_DIR)/tgz/root
|
||||
tar -cz -f $(BUILD_DIR)/tgz/$(PACKAGE_NAME)-$(FULL_VERSION).tgz -C $(BUILD_DIR)/tgz/root .
|
||||
|
||||
|
||||
##
|
||||
# Install targets
|
||||
#
|
||||
# These are mostly used by the various package building images to actually install the script
|
||||
##
|
||||
|
||||
# Install the script at the specified base directory (common components)
|
||||
.PHONY: install-common
|
||||
install-common:
|
||||
# Set up directories
|
||||
mkdir -p $(DESTDIR)/etc/$(PACKAGE_NAME)
|
||||
mkdir -p ${DESTDIR}/etc/logrotate.d
|
||||
mkdir -p $(DESTDIR)/usr/bin
|
||||
mkdir -p $(DESTDIR)/usr/share/man/man1
|
||||
|
||||
# Install script
|
||||
cp $(SCRIPT) $(DESTDIR)/usr/bin/$(PACKAGE_NAME)
|
||||
chmod 755 $(DESTDIR)/usr/bin/$(PACKAGE_NAME)
|
||||
|
||||
# Install manpage
|
||||
cp manpages/* $(DESTDIR)/usr/share/man/man1/
|
||||
gzip -f -9 $(DESTDIR)/usr/share/man/man1/$(PACKAGE_NAME).1
|
||||
|
||||
# Install sample config
|
||||
cp sample-config/* $(DESTDIR)/etc/$(PACKAGE_NAME)/
|
||||
|
||||
# Install logrotate config
|
||||
cp logrotate/${PACKAGE_NAME}.logrotate ${DESTDIR}/etc/logrotate.d/${PACKAGE_NAME}
|
||||
|
||||
# Install for systemd
|
||||
.PHONY: install-systemd
|
||||
install-systemd:
|
||||
# Install the common stuff
|
||||
$(MAKE) install-common
|
||||
|
||||
# Set up directories
|
||||
mkdir -p $(DESTDIR)/lib/systemd/system
|
||||
|
||||
# Install systemd unit files
|
||||
cp systemd/* $(DESTDIR)/lib/systemd/system/
|
||||
|
||||
# Install for open-rc
|
||||
.PHONY: install-openrc
|
||||
install-openrc:
|
||||
# Install the common stuff
|
||||
$(MAKE) install-common
|
||||
|
||||
# Set up directories
|
||||
mkdir -p $(DESTDIR)/etc/init.d
|
||||
mkdir -p $(DESTDIR)/etc/conf.d
|
||||
|
||||
# Install init script
|
||||
cp openrc/pgmon.initd $(DESTDIR)/etc/init.d/pgmon
|
||||
chmod 755 $(DESTDIR)/etc/init.d/pgmon
|
||||
|
||||
# Install init script config file
|
||||
cp openrc/pgmon.confd $(DESTDIR)/etc/conf.d/pgmon
|
||||
|
||||
|
||||
##
|
||||
# Packaging targets
|
||||
#
|
||||
# These targets create packages for the supported distro versions
|
||||
##
|
||||
|
||||
# Build all packages
|
||||
.PHONY: package-all
|
||||
all: $(foreach distro_release, $(SUPPORTED), package-$(distro_release))
|
||||
package-all: all-package-images $(foreach distro_release, $(SUPPORTED), package-$(distro_release))
|
||||
|
||||
# Gentoo package (tar.gz) creation
|
||||
# Gentoo package (tar.bz2) creation
|
||||
.PHONY: package-gentoo
|
||||
package-gentoo:
|
||||
mkdir -p $(BUILD_DIR)/gentoo
|
||||
tar --transform "s,^\.,$(PACKAGE_NAME)-$(VERSION)," -acjf $(BUILD_DIR)/gentoo/$(PACKAGE_NAME)-$(VERSION).tar.bz2 --exclude $(BUILD_DIR) .
|
||||
tar --transform "s,^,$(PACKAGE_NAME)-$(FULL_VERSION)/," -acjf $(BUILD_DIR)/gentoo/$(PACKAGE_NAME)-$(FULL_VERSION).tar.bz2 --exclude .gitignore $(shell git ls-tree --full-tree --name-only -r HEAD)
|
||||
cp $(BUILD_DIR)/gentoo/$(PACKAGE_NAME)-$(FULL_VERSION).tar.bz2 $(BUILD_DIR)/
|
||||
|
||||
|
||||
# Create a deb package
|
||||
# Create a deb/rpm package
|
||||
.PHONY: package-%
|
||||
package-%: PARTS=$(subst -, ,$*)
|
||||
package-%: DISTRO=$(word 1, $(PARTS))
|
||||
@@ -49,83 +186,63 @@ package-%:
|
||||
--user $(shell id -u):$(shell id -g) \
|
||||
"$(DISTRO)-packager:$(RELEASE)"
|
||||
|
||||
# Create a tarball
|
||||
tgz:
|
||||
rm -rf $(BUILD_DIR)/tgz/root
|
||||
mkdir -p $(BUILD_DIR)/tgz/root
|
||||
$(MAKE) install DESTDIR=$(BUILD_DIR)/tgz/root
|
||||
tar -cz -f $(BUILD_DIR)/tgz/$(PACKAGE_NAME)-$(VERSION).tgz -C $(BUILD_DIR)/tgz/root .
|
||||
|
||||
# Clean up the build directory
|
||||
clean:
|
||||
rm -rf $(BUILD_DIR)
|
||||
|
||||
# Run unit tests for the script
|
||||
test:
|
||||
cd src ; python3 -m unittest
|
||||
|
||||
# Install the script at the specified base directory
|
||||
install:
|
||||
# Set up directories
|
||||
mkdir -p $(DESTDIR)/etc/$(PACKAGE_NAME)
|
||||
mkdir -p ${DESTDIR}/etc/logrotate.d
|
||||
mkdir -p $(DESTDIR)/lib/systemd/system
|
||||
mkdir -p $(DESTDIR)/usr/local/bin
|
||||
mkdir -p $(DESTDIR)/usr/share/man/man1
|
||||
|
||||
# Install script
|
||||
cp $(SCRIPT) $(DESTDIR)/usr/local/bin/$(PACKAGE_NAME)
|
||||
chmod 755 $(DESTDIR)/usr/local/bin/$(PACKAGE_NAME)
|
||||
|
||||
# Install manpage
|
||||
cp manpages/* $(DESTDIR)/usr/share/man/man1/
|
||||
gzip -f -9 $(DESTDIR)/usr/share/man/man1/$(PACKAGE_NAME).1
|
||||
|
||||
# Install sample config
|
||||
cp sample-config/* $(DESTDIR)/etc/$(PACKAGE_NAME)/
|
||||
|
||||
# Install systemd unit files
|
||||
cp systemd/* $(DESTDIR)/lib/systemd/system/
|
||||
|
||||
# Install logrotate config
|
||||
cp logrotate/${PACKAGE_NAME}.logrotate ${DESTDIR}/etc/logrotate.d/${PACKAGE_NAME}
|
||||
|
||||
##
|
||||
# Install test targets
|
||||
#
|
||||
# These targets test installing the package on the supported distro versions
|
||||
##
|
||||
|
||||
# Run all of the install tests
|
||||
.PHONY: install-tests debian-%-install-test rockylinux-%-install-test ubuntu-%-install-test
|
||||
.PHONY: install-tests debian-%-install-test rockylinux-%-install-test ubuntu-%-install-test amazonlinux-%-install-test gentoo-install-test
|
||||
install-tests: $(foreach distro_release, $(SUPPORTED), $(distro_release)-install-test)
|
||||
|
||||
|
||||
# Run a Debian install test
|
||||
debian-%-install-test:
|
||||
docker run --rm \
|
||||
-v ./$(BUILD_DIR):/output \
|
||||
debian:$* \
|
||||
bash -c 'apt-get update && apt-get install -y /output/$(PACKAGE_NAME)-$(VERSION)-debian-$*.deb'
|
||||
bash -c 'apt-get update && apt-get install -y /output/$(PACKAGE_NAME)-$(DEB_VERSION)-debian-$*.deb'
|
||||
|
||||
# Run a RedHat install test
|
||||
rockylinux-%-install-test:
|
||||
docker run --rm \
|
||||
-v ./$(BUILD_DIR):/output \
|
||||
rockylinux:$* \
|
||||
bash -c 'dnf makecache && dnf install -y /output/$(PACKAGE_NAME)-$(VERSION)-1.el$*.noarch.rpm'
|
||||
rockylinux/rockylinux:$* \
|
||||
bash -c 'dnf makecache && dnf install -y /output/$(PACKAGE_NAME)-$(RPM_VERSION).el$*.noarch.rpm'
|
||||
|
||||
# Run an Ubuntu install test
|
||||
ubuntu-%-install-test:
|
||||
docker run --rm \
|
||||
-v ./$(BUILD_DIR):/output \
|
||||
ubuntu:$* \
|
||||
bash -c 'apt-get update && apt-get install -y /output/$(PACKAGE_NAME)-$(VERSION)-ubuntu-$*.deb'
|
||||
bash -c 'apt-get update && apt-get install -y /output/$(PACKAGE_NAME)-$(DEB_VERSION)-ubuntu-$*.deb'
|
||||
|
||||
# Run an OracleLinux install test (this is for EL7 since CentOS7 images no longer exist)
|
||||
oraclelinux-%-install-test: MGR=$(intcmp $*, 7, yum, yum, dnf)
|
||||
oraclelinux-%-install-test:
|
||||
docker run --rm \
|
||||
-v ./$(BUILD_DIR):/output \
|
||||
oraclelinux:7 \
|
||||
bash -c 'yum makecache && yum install -y /output/$(PACKAGE_NAME)-$(VERSION)-1.el7.noarch.rpm'
|
||||
oraclelinux:$* \
|
||||
bash -c '$(MGR) makecache && $(MGR) install -y /output/$(PACKAGE_NAME)-$(RPM_VERSION).el7.noarch.rpm'
|
||||
|
||||
# Run a Amazon Linux install test
|
||||
# Note: AL2 is too old for dnf
|
||||
amazonlinux-%-install-test: MGR=$(intcmp $*, 2, yum, yum, dnf)
|
||||
amazonlinux-%-install-test:
|
||||
docker run --rm \
|
||||
-v ./$(BUILD_DIR):/output \
|
||||
amazonlinux:$* \
|
||||
bash -c '$(MGR) makecache && $(MGR) install -y /output/$(PACKAGE_NAME)-$(RPM_VERSION).amzn$*.noarch.rpm'
|
||||
|
||||
# Run a Gentoo install test
|
||||
gentoo-install-test:
|
||||
# May impliment this in the future, but would require additional headaches to set up a repo
|
||||
echo "This would take a while ... skipping for now"
|
||||
|
||||
|
||||
##
|
||||
# Container targets
|
||||
# Packaging image targets
|
||||
#
|
||||
# These targets build the docker images used to create packages
|
||||
##
|
||||
@@ -134,6 +251,11 @@ oraclelinux-%-install-test:
|
||||
.PHONY: all-package-images
|
||||
all-package-images: $(foreach distro_release, $(SUPPORTED), package-image-$(distro_release))
|
||||
|
||||
# Nothing needs to be created to build te tarballs for Gentoo
|
||||
.PHONY: package-image-gentoo
|
||||
package-image-gentoo:
|
||||
echo "Noting to do"
|
||||
|
||||
# Generic target for creating images that actually build the packages
|
||||
# The % is expected to be: distro-release
|
||||
# The build/.package-image-% target actually does the work and touches a state file to avoid excessive building
|
||||
@@ -149,38 +271,63 @@ package-image-%:
|
||||
# Inside-container targets
|
||||
#
|
||||
# These targets are used inside containers. They expect the repo to be mounted
|
||||
# at /src and the package manager specific build directory to be mounted at
|
||||
# /output.
|
||||
# at /src and the build directory to be mounted at /output.
|
||||
##
|
||||
|
||||
.PHONY: actually-package-debian-% actually-package-rockylinux-% actually-package-ubuntu-% actually-package-oraclelinux-%
|
||||
.PHONY: actually-package-debian-% actually-package-rockylinux-% actually-package-ubuntu-% actually-package-oraclelinux-% actually-package-amazonlinux-%
|
||||
|
||||
# Debian package creation
|
||||
actually-package-debian-%:
|
||||
$(MAKE) install DESTDIR=/output/debian-$*
|
||||
$(MAKE) install-systemd DESTDIR=/output/debian-$*
|
||||
cp -r --preserve=mode DEBIAN /output/debian-$*/
|
||||
dpkg-deb -Zgzip --build /output/debian-$* "/output/$(PACKAGE_NAME)-$(VERSION)-debian-$*.deb"
|
||||
dpkg-deb -Zgzip --build /output/debian-$* "/output/$(PACKAGE_NAME)-$(DEB_VERSION)-debian-$*.deb"
|
||||
|
||||
# RedHat package creation
|
||||
actually-package-rockylinux-%:
|
||||
mkdir -p /output/rockylinux-$*/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
|
||||
sed -e "s/@@VERSION@@/$(VERSION)/g" RPM/$(PACKAGE_NAME).spec > /output/rockylinux-$*/SPECS/$(PACKAGE_NAME).spec
|
||||
sed -e "s/@@VERSION@@/$(VERSION)/g" -e "s/@@RELEASE@@/$(RPM_RELEASE)/g" RPM/$(PACKAGE_NAME).spec > /output/rockylinux-$*/SPECS/$(PACKAGE_NAME).spec
|
||||
rpmbuild --define '_topdir /output/rockylinux-$*' \
|
||||
--define 'version $(VERSION)' \
|
||||
--define 'version $(RPM_VERSION)' \
|
||||
-bb /output/rockylinux-$*/SPECS/$(PACKAGE_NAME).spec
|
||||
cp /output/rockylinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(VERSION)-1.el$*.noarch.rpm /output/
|
||||
cp /output/rockylinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(RPM_VERSION).el$*.noarch.rpm /output/
|
||||
|
||||
# Ubuntu package creation
|
||||
actually-package-ubuntu-%:
|
||||
$(MAKE) install DESTDIR=/output/ubuntu-$*
|
||||
$(MAKE) install-systemd DESTDIR=/output/ubuntu-$*
|
||||
cp -r --preserve=mode DEBIAN /output/ubuntu-$*/
|
||||
dpkg-deb -Zgzip --build /output/ubuntu-$* "/output/$(PACKAGE_NAME)-$(VERSION)-ubuntu-$*.deb"
|
||||
dpkg-deb -Zgzip --build /output/ubuntu-$* "/output/$(PACKAGE_NAME)-$(DEB_VERSION)-ubuntu-$*.deb"
|
||||
|
||||
# OracleLinux package creation
|
||||
# Note: This needs to work inside OEL7, so we can't use intcmp
|
||||
actually-package-oraclelinux-7:
|
||||
mkdir -p /output/oraclelinux-7/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
|
||||
sed -e "s/@@VERSION@@/$(VERSION)/g" -e "s/@@RELEASE@@/$(RPM_RELEASE)/g" RPM/$(PACKAGE_NAME)-el7.spec > /output/oraclelinux-7/SPECS/$(PACKAGE_NAME).spec
|
||||
rpmbuild --define '_topdir /output/oraclelinux-7' \
|
||||
--define 'version $(RPM_VERSION)' \
|
||||
-bb /output/oraclelinux-7/SPECS/$(PACKAGE_NAME).spec
|
||||
cp /output/oraclelinux-7/RPMS/noarch/$(PACKAGE_NAME)-$(RPM_VERSION).el7.noarch.rpm /output/
|
||||
actually-package-oraclelinux-%:
|
||||
mkdir -p /output/oraclelinux-$*/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
|
||||
sed -e "s/@@VERSION@@/$(VERSION)/g" RPM/$(PACKAGE_NAME)-el7.spec > /output/oraclelinux-$*/SPECS/$(PACKAGE_NAME).spec
|
||||
sed -e "s/@@VERSION@@/$(VERSION)/g" -e "s/@@RELEASE@@/$(RPM_RELEASE)/g" RPM/$(PACKAGE_NAME).spec > /output/oraclelinux-$*/SPECS/$(PACKAGE_NAME).spec
|
||||
rpmbuild --define '_topdir /output/oraclelinux-$*' \
|
||||
--define 'version $(VERSION)' \
|
||||
--define 'version $(RPM_VERSION)' \
|
||||
-bb /output/oraclelinux-$*/SPECS/$(PACKAGE_NAME).spec
|
||||
cp /output/oraclelinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(VERSION)-1.el$*.noarch.rpm /output/
|
||||
cp /output/oraclelinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(RPM_VERSION).el$*.noarch.rpm /output/
|
||||
|
||||
# Amazon Linux package creation
|
||||
# Note: This needs to work inside AL2, so we can't use intcmp
|
||||
actually-package-amazonlinux-2:
|
||||
mkdir -p /output/amazonlinux-2/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
|
||||
sed -e "s/@@VERSION@@/$(VERSION)/g" -e "s/@@RELEASE@@/$(RPM_RELEASE)/g" RPM/$(PACKAGE_NAME)-el7.spec > /output/amazonlinux-2/SPECS/$(PACKAGE_NAME).spec
|
||||
rpmbuild --define '_topdir /output/amazonlinux-2' \
|
||||
--define 'version $(RPM_VERSION)' \
|
||||
-bb /output/amazonlinux-2/SPECS/$(PACKAGE_NAME).spec
|
||||
cp /output/amazonlinux-2/RPMS/noarch/$(PACKAGE_NAME)-$(RPM_VERSION).amzn2.noarch.rpm /output/
|
||||
actually-package-amazonlinux-%:
|
||||
mkdir -p /output/amazonlinux-$*/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
|
||||
sed -e "s/@@VERSION@@/$(VERSION)/g" -e "s/@@RELEASE@@/$(RPM_RELEASE)/g" RPM/$(PACKAGE_NAME).spec > /output/amazonlinux-$*/SPECS/$(PACKAGE_NAME).spec
|
||||
rpmbuild --define '_topdir /output/amazonlinux-$*' \
|
||||
--define 'version $(RPM_VERSION)' \
|
||||
-bb /output/amazonlinux-$*/SPECS/$(PACKAGE_NAME).spec
|
||||
cp /output/amazonlinux-$*/RPMS/noarch/$(PACKAGE_NAME)-$(RPM_VERSION).amzn$*.noarch.rpm /output/
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
Name: pgmon
|
||||
Version: @@VERSION@@
|
||||
Release: 1%{?dist}
|
||||
Release: @@RELEASE@@%{?dist}
|
||||
Summary: A bridge to sit between monitoring tools and PostgreSQL
|
||||
|
||||
License: MIT
|
||||
URL: https://www.commandprompt.com
|
||||
|
||||
BuildArch: noarch
|
||||
Requires: logrotate, python, python-psycopg2, PyYAML, systemd
|
||||
Requires: logrotate, python, python-psycopg2, PyYAML, python-requests, systemd
|
||||
|
||||
%description
|
||||
A bridge to sit between monitoring tools and PostgreSQL
|
||||
@@ -19,7 +19,7 @@ A bridge to sit between monitoring tools and PostgreSQL
|
||||
# Do nothing since we have nothing to build
|
||||
|
||||
%install
|
||||
make -C /src install DESTDIR=%{buildroot}
|
||||
make -C /src install-systemd DESTDIR=%{buildroot}
|
||||
|
||||
%files
|
||||
/etc/logrotate.d/pgmon
|
||||
@@ -28,7 +28,7 @@ make -C /src install DESTDIR=%{buildroot}
|
||||
/etc/pgmon/pgmon-service.conf
|
||||
/lib/systemd/system/pgmon.service
|
||||
/lib/systemd/system/pgmon@.service
|
||||
/usr/local/bin/pgmon
|
||||
/usr/bin/pgmon
|
||||
/usr/share/man/man1/pgmon.1.gz
|
||||
|
||||
%post
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
Name: pgmon
|
||||
Version: @@VERSION@@
|
||||
Release: 1%{?dist}
|
||||
Release: @@RELEASE@@%{?dist}
|
||||
Summary: A bridge to sit between monitoring tools and PostgreSQL
|
||||
|
||||
License: MIT
|
||||
URL: https://www.commandprompt.com
|
||||
|
||||
BuildArch: noarch
|
||||
Requires: logrotate, python3, python3-psycopg2, python3-pyyaml, systemd
|
||||
Requires: logrotate, python3, python3-psycopg2, python3-pyyaml, python3-requests, systemd
|
||||
|
||||
%description
|
||||
A bridge to sit between monitoring tools and PostgreSQL
|
||||
@@ -19,7 +19,7 @@ A bridge to sit between monitoring tools and PostgreSQL
|
||||
# Do nothing since we have nothing to build
|
||||
|
||||
%install
|
||||
make -C /src install DESTDIR=%{buildroot}
|
||||
make -C /src install-systemd DESTDIR=%{buildroot}
|
||||
|
||||
%files
|
||||
/etc/logrotate.d/pgmon
|
||||
@@ -28,7 +28,7 @@ make -C /src install DESTDIR=%{buildroot}
|
||||
/etc/pgmon/pgmon-service.conf
|
||||
/lib/systemd/system/pgmon.service
|
||||
/lib/systemd/system/pgmon@.service
|
||||
/usr/local/bin/pgmon
|
||||
/usr/bin/pgmon
|
||||
/usr/share/man/man1/pgmon.1.gz
|
||||
|
||||
%post
|
||||
|
||||
22
docker/Dockerfile-amazonlinux
Normal file
22
docker/Dockerfile-amazonlinux
Normal file
@@ -0,0 +1,22 @@
|
||||
# Dockerfile.rpm
|
||||
|
||||
ARG DISTRO=amazonlinux
|
||||
ARG RELEASE=2023
|
||||
|
||||
FROM ${DISTRO}:${RELEASE}
|
||||
|
||||
ARG DISTRO
|
||||
ARG RELEASE
|
||||
|
||||
RUN if [ ${RELEASE} -le 2 ] ; then MGR=yum ; else MGR=dnf ; fi && \
|
||||
${MGR} install -y \
|
||||
rpm-build \
|
||||
make \
|
||||
&& ${MGR} clean all
|
||||
|
||||
RUN echo -e "#!/bin/bash\nmake actually-package-${DISTRO}-${RELEASE}" > /init.sh \
|
||||
&& chmod 755 /init.sh
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
CMD ["/bin/bash", "/init.sh"]
|
||||
@@ -8,11 +8,12 @@ FROM ${DISTRO}:${RELEASE}
|
||||
ARG DISTRO
|
||||
ARG RELEASE
|
||||
|
||||
RUN yum install -y \
|
||||
RUN if [ ${RELEASE} -le 7 ] ; then MGR=yum ; else MGR=dnf ; fi && \
|
||||
${MGR} install -y \
|
||||
rpm-build \
|
||||
make \
|
||||
oracle-epel-release-el7 \
|
||||
&& yum clean all
|
||||
oracle-epel-release-el${RELEASE} \
|
||||
&& ${MGR} clean all
|
||||
|
||||
RUN echo -e "#!/bin/bash\nmake actually-package-${DISTRO}-${RELEASE}" > /init.sh \
|
||||
&& chmod 755 /init.sh
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
ARG DISTRO=rockylinux
|
||||
ARG RELEASE=9
|
||||
|
||||
FROM ${DISTRO}:${RELEASE}
|
||||
FROM rockylinux/${DISTRO}:${RELEASE}
|
||||
|
||||
ARG DISTRO
|
||||
ARG RELEASE
|
||||
|
||||
@@ -19,6 +19,6 @@ start_pre() {
|
||||
}
|
||||
|
||||
command="/usr/bin/pgmon"
|
||||
command_args="'$CONFIG_FILE'"
|
||||
command_args="-c '$CONFIG_FILE'"
|
||||
command_background="true"
|
||||
command_user="${PGMON_USER}:${PGMON_GROUP}"
|
||||
|
||||
4
requirements-dev.yml
Normal file
4
requirements-dev.yml
Normal file
@@ -0,0 +1,4 @@
|
||||
-r requirements.txt
|
||||
testcontainers[postgresql]
|
||||
pytest
|
||||
black
|
||||
@@ -1,55 +1,395 @@
|
||||
metrics:
|
||||
##
|
||||
# Discovery metrics
|
||||
##
|
||||
discover_dbs:
|
||||
type: set
|
||||
query:
|
||||
0: SELECT datname AS dbname FROM pg_database
|
||||
0: >
|
||||
SELECT datname AS dbname
|
||||
FROM pg_database
|
||||
|
||||
# Note: If the user lacks sufficient privileges, these fields will be NULL.
|
||||
# The WHERE clause is intended to prevent Zabbix from discovering a
|
||||
# connection it cannot monitor. Ideally this would generate an error
|
||||
# instead.
|
||||
discover_rep:
|
||||
type: set
|
||||
query:
|
||||
0: SELECT client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') AS repid, client_addr, state FROM pg_stat_replication
|
||||
0: >
|
||||
SELECT host(client_addr) || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') AS repid,
|
||||
application_name,
|
||||
client_addr,
|
||||
state
|
||||
FROM pg_stat_replication
|
||||
WHERE state IS NOT NULL
|
||||
|
||||
discover_slots:
|
||||
type: set
|
||||
query:
|
||||
0: SELECT slot_name, plugin, slot_type, database, temporary, active FROM pg_replication_slots
|
||||
90400: >
|
||||
SELECT slot_name,
|
||||
plugin,
|
||||
slot_type,
|
||||
database,
|
||||
false as temporary,
|
||||
active
|
||||
FROM pg_replication_slots
|
||||
100000: >
|
||||
SELECT slot_name,
|
||||
plugin,
|
||||
slot_type,
|
||||
database,
|
||||
temporary,
|
||||
active
|
||||
FROM pg_replication_slots
|
||||
|
||||
|
||||
##
|
||||
# cluster-wide metrics
|
||||
##
|
||||
version:
|
||||
type: value
|
||||
query:
|
||||
0: SHOW server_version_num
|
||||
|
||||
max_frozen_age:
|
||||
type: row
|
||||
query:
|
||||
0: >
|
||||
SELECT max(age(datfrozenxid)) AS xid_age,
|
||||
NULL AS mxid_age
|
||||
FROM pg_database
|
||||
90600: >
|
||||
SELECT max(age(datfrozenxid)) AS xid_age,
|
||||
max(mxid_age(datminmxid)) AS mxid_age
|
||||
FROM pg_database
|
||||
|
||||
bgwriter:
|
||||
type: row
|
||||
query:
|
||||
0: >
|
||||
SELECT checkpoints_timed,
|
||||
checkpoints_req,
|
||||
checkpoint_write_time,
|
||||
checkpoint_sync_time,
|
||||
buffers_checkpoint,
|
||||
buffers_clean,
|
||||
maxwritten_clean,
|
||||
buffers_backend,
|
||||
buffers_backend_fsync,
|
||||
buffers_alloc
|
||||
FROM pg_stat_bgwriter
|
||||
170000: >
|
||||
SELECT cp.num_timed AS checkpoints_timed,
|
||||
cp.num_requested AS checkpoints_req,
|
||||
cp.write_time AS checkpoint_write_time,
|
||||
cp.sync_time AS checkpoint_sync_time,
|
||||
cp.buffers_written AS buffers_checkpoint,
|
||||
bg.buffers_clean AS buffers_clean,
|
||||
bg.maxwritten_clean AS maxwritten_clean,
|
||||
NULL AS buffers_backend,
|
||||
NULL AS buffers_backend_fsync,
|
||||
bg.buffers_alloc AS buffers_alloc
|
||||
FROM pg_stat_bgwriter bg
|
||||
CROSS JOIN pg_stat_checkpointer cp
|
||||
|
||||
io_per_backend:
|
||||
type: set
|
||||
query:
|
||||
160000: >
|
||||
SELECT backend_type,
|
||||
COALESCE(SUM(reads * op_bytes), 0) AS reads,
|
||||
COALESCE(SUM(read_time), 0) AS read_time,
|
||||
COALESCE(SUM(writes * op_bytes), 0) AS writes,
|
||||
COALESCE(SUM(write_time), 0) AS write_time,
|
||||
COALESCE(SUM(writebacks * op_bytes), 0) AS writebacks,
|
||||
COALESCE(SUM(writeback_time), 0) AS writeback_time,
|
||||
COALESCE(SUM(extends * op_bytes), 0) AS extends,
|
||||
COALESCE(SUM(extend_time), 0) AS extend_time,
|
||||
COALESCE(SUM(op_bytes), 0) AS op_bytes,
|
||||
COALESCE(SUM(hits), 0) AS hits,
|
||||
COALESCE(SUM(evictions), 0) AS evictions,
|
||||
COALESCE(SUM(reuses), 0) AS reuses,
|
||||
COALESCE(SUM(fsyncs), 0) AS fsyncs,
|
||||
COALESCE(SUM(fsync_time), 0) AS fsync_time
|
||||
FROM pg_stat_io
|
||||
GROUP BY backend_type
|
||||
|
||||
temp_files:
|
||||
type: row
|
||||
query:
|
||||
120000: >
|
||||
SELECT count(*) AS count,
|
||||
coalesce(sum(size), 0) AS size
|
||||
FROM pg_ls_tmpdir()
|
||||
WHERE name LIKE 'pgsql_tmp%%'
|
||||
90500: >
|
||||
SELECT count(*) AS count,
|
||||
coalesce(sum((pg_stat_file(name)).size), 0) AS size
|
||||
FROM pg_ls_dir('base/pgsql_tmp/', true, false) AS name
|
||||
WHERE name LIKE 'pgsql_tmp%%'
|
||||
0: >
|
||||
SELECT count(*) AS count,
|
||||
coalesce(sum((pg_stat_file(files.name)).size), 0) AS size
|
||||
FROM (
|
||||
SELECT name
|
||||
FROM pg_ls_dir('base') AS name
|
||||
WHERE name = 'pgsql_tmp') AS dir
|
||||
LEFT JOIN (
|
||||
SELECT name
|
||||
FROM pg_ls_dir('base/pgsql_tmp/') AS name
|
||||
WHERE name LIKE 'pgsql_tmp%%') AS files
|
||||
ON dir.name IS NOT NULL
|
||||
|
||||
|
||||
ready_archive_files:
|
||||
type: value
|
||||
query:
|
||||
0: SELECT max(age(datfrozenxid)) FROM pg_database
|
||||
120000: >
|
||||
SELECT count(*) AS count
|
||||
FROM pg_ls_archive_statusdir()
|
||||
WHERE name LIKE '%%.ready'
|
||||
100000: >
|
||||
SELECT count(*) AS count
|
||||
FROM pg_ls_dir('pg_wal/archive_status/', true, false) AS name
|
||||
WHERE name LIKE '%%.ready'
|
||||
90500: >
|
||||
SELECT count(*) AS count
|
||||
FROM pg_ls_dir('pg_xlog/archive_status/', true, false) AS name
|
||||
WHERE name LIKE '%%.ready'
|
||||
0: >
|
||||
SELECT
|
||||
CASE WHEN (
|
||||
SELECT count(*)
|
||||
FROM pg_ls_dir('pg_xlog/') AS f
|
||||
WHERE f = 'archive_status') = 0
|
||||
THEN (
|
||||
SELECT 0 AS count
|
||||
)
|
||||
ELSE (
|
||||
SELECT count(*) AS count
|
||||
FROM pg_ls_dir('pg_xlog/archive_status/') AS name
|
||||
WHERE name LIKE '%%.ready')
|
||||
END
|
||||
|
||||
|
||||
##
|
||||
# Per-database metrics
|
||||
##
|
||||
db_stats:
|
||||
type: row
|
||||
query:
|
||||
0: SELECT numbackends, xact_commit, xact_rollback, blks_read, blks_hit, tup_returned, tup_fetched, tup_inserted, tup_updated, tup_deleted, conflicts, temp_files, temp_bytes, deadlocks, blk_read_time, blk_write_time, extract('epoch' from stats_reset)::float FROM pg_stat_database WHERE datname = %(dbname)s
|
||||
140000: SELECT numbackends, xact_commit, xact_rollback, blks_read, blks_hit, tup_returned, tup_fetched, tup_inserted, tup_updated, tup_deleted, conflicts, temp_files, temp_bytes, deadlocks, COALESCE(checksum_failures, 0) AS checksum_failures, blk_read_time, blk_write_time, session_time, active_time, idle_in_transaction_time, sessions, sessions_abandoned, sessions_fatal, sessions_killed, extract('epoch' from stats_reset)::float FROM pg_stat_database WHERE datname = %(dbname)s
|
||||
0: >
|
||||
SELECT numbackends,
|
||||
xact_commit,
|
||||
xact_rollback,
|
||||
blks_read,
|
||||
blks_hit,
|
||||
tup_returned,
|
||||
tup_fetched,
|
||||
tup_inserted,
|
||||
tup_updated,
|
||||
tup_deleted,
|
||||
conflicts,
|
||||
temp_files,
|
||||
temp_bytes,
|
||||
deadlocks,
|
||||
NULL AS checksum_failures,
|
||||
blk_read_time,
|
||||
blk_write_time,
|
||||
NULL AS session_time,
|
||||
NULL AS active_time,
|
||||
NULL AS idle_in_transaction_time,
|
||||
NULL AS sessions,
|
||||
NULL AS sessions_abandoned,
|
||||
NULL AS sessions_fatal,
|
||||
NULL AS sessions_killed,
|
||||
extract('epoch' from stats_reset) AS stats_reset
|
||||
FROM pg_stat_database WHERE datname = %(dbname)s
|
||||
140000: >
|
||||
SELECT numbackends,
|
||||
xact_commit,
|
||||
xact_rollback,
|
||||
blks_read,
|
||||
blks_hit,
|
||||
tup_returned,
|
||||
tup_fetched,
|
||||
tup_inserted,
|
||||
tup_updated,
|
||||
tup_deleted,
|
||||
conflicts,
|
||||
temp_files,
|
||||
temp_bytes,
|
||||
deadlocks,
|
||||
COALESCE(checksum_failures, 0) AS checksum_failures,
|
||||
blk_read_time,
|
||||
blk_write_time,
|
||||
session_time,
|
||||
active_time,
|
||||
idle_in_transaction_time,
|
||||
sessions,
|
||||
sessions_abandoned,
|
||||
sessions_fatal,
|
||||
sessions_killed,
|
||||
extract('epoch' from stats_reset) AS stats_reset
|
||||
FROM pg_stat_database WHERE datname = %(dbname)s
|
||||
test_args:
|
||||
dbname: postgres
|
||||
|
||||
hit_ratios:
|
||||
type: row
|
||||
query:
|
||||
0: >
|
||||
SELECT sum(heap_blks_read)::float / NULLIF(sum(heap_blks_read + heap_blks_hit), 0) AS avg_heap_hit_ratio,
|
||||
sum(idx_blks_hit)::float / NULLIF(sum(idx_blks_read + idx_blks_hit), 0) AS avg_idx_hit_ratio,
|
||||
sum(toast_blks_hit)::float / NULLIF(sum(toast_blks_read + toast_blks_hit), 0) AS avg_toast_hit_ratio,
|
||||
sum(tidx_blks_hit)::float / NULLIF(sum(tidx_blks_read + tidx_blks_hit), 0) AS avg_tidx_hit_ratio
|
||||
FROM pg_statio_all_tables
|
||||
test_args:
|
||||
dbname: postgres
|
||||
|
||||
activity:
|
||||
type: set
|
||||
query:
|
||||
0: >
|
||||
SELECT
|
||||
states.state,
|
||||
COALESCE(a.backend_count, 0) AS backend_count,
|
||||
COALESCE(a.max_state_time, 0) AS max_state_time
|
||||
FROM (
|
||||
SELECT state,
|
||||
count(*) AS backend_count,
|
||||
COALESCE(EXTRACT(EPOCH FROM now() - min(state_change)), 0) AS max_state_time
|
||||
FROM pg_stat_activity
|
||||
WHERE datname = %(dbname)s
|
||||
GROUP BY state
|
||||
) AS a
|
||||
RIGHT JOIN
|
||||
unnest(ARRAY[
|
||||
'active',
|
||||
'idle',
|
||||
'idle in transaction',
|
||||
'idle in transaction (aborted)',
|
||||
'fastpath function call',
|
||||
'disabled'
|
||||
]) AS states(state)
|
||||
USING (state);
|
||||
test_args:
|
||||
dbname: postgres
|
||||
|
||||
sequence_usage:
|
||||
type: value
|
||||
query:
|
||||
# 9.2 lacks lateral joins, the pg_sequence_last_value function, and the pg_sequences view
|
||||
# 0: >
|
||||
# SELECT COALESCE(MAX(pg_sequence_last_value(c.oid)::float / (pg_sequence_parameters(oid)).maximum_value), 0) AS max_usage
|
||||
# FROM pg_class c
|
||||
# WHERE c.relkind = 'S'
|
||||
# 9.3 - 9.6 lacks the pg_sequence_last_value function, and pg_sequences view
|
||||
# 90300: >
|
||||
# SELECT COALESCE(MAX(pg_sequence_last_value(c.oid)::float / s.maximum_value), 0) AS max_usage
|
||||
# FROM pg_class c
|
||||
# CROSS JOIN LATERAL pg_sequence_parameters(c.oid) AS s
|
||||
# WHERE c.relkind = 'S'
|
||||
100000: SELECT COALESCE(MAX(last_value::float / max_value), 0) AS max_usage FROM pg_sequences;
|
||||
test_args:
|
||||
dbname: postgres
|
||||
|
||||
sequence_visibility:
|
||||
type: row
|
||||
query:
|
||||
100000: >
|
||||
SELECT COUNT(*) FILTER (WHERE has_sequence_privilege(c.oid, 'SELECT,USAGE')) AS visible_sequences,
|
||||
COUNT(*) AS total_sequences
|
||||
FROM pg_class AS c
|
||||
WHERE relkind = 'S'
|
||||
|
||||
locks:
|
||||
type: row
|
||||
query:
|
||||
0: >
|
||||
SELECT COUNT(*) AS total,
|
||||
SUM(CASE WHEN granted THEN 1 ELSE 0 END) AS granted
|
||||
FROM pg_locks
|
||||
90400: >
|
||||
SELECT COUNT(*) AS total,
|
||||
COUNT(*) FILTER (WHERE granted) AS granted
|
||||
FROM pg_locks
|
||||
|
||||
|
||||
##
|
||||
# Per-replication metrics
|
||||
##
|
||||
rep_stats:
|
||||
type: row
|
||||
query:
|
||||
90400: >
|
||||
SELECT pid, usename,
|
||||
EXTRACT(EPOCH FROM backend_start) AS backend_start,
|
||||
state,
|
||||
pg_xlog_location_diff(pg_current_xlog_location(), sent_location) AS sent_lsn,
|
||||
pg_xlog_location_diff(pg_current_xlog_location(), write_location) AS write_lsn,
|
||||
pg_xlog_location_diff(pg_current_xlog_location(), flush_location) AS flush_lsn,
|
||||
pg_xlog_location_diff(pg_current_xlog_location(), replay_location) AS replay_lsn,
|
||||
NULL AS write_lag,
|
||||
NULL AS flush_lag,
|
||||
NULL AS replay_lag,
|
||||
sync_state
|
||||
FROM pg_stat_replication
|
||||
WHERE host(client_addr) || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = %(repid)s
|
||||
100000: >
|
||||
SELECT pid, usename,
|
||||
EXTRACT(EPOCH FROM backend_start) AS backend_start,
|
||||
state,
|
||||
pg_wal_lsn_diff(pg_current_wal_lsn(), sent_lsn) AS sent_lsn,
|
||||
pg_wal_lsn_diff(pg_current_wal_lsn(), write_lsn) AS write_lsn,
|
||||
pg_wal_lsn_diff(pg_current_wal_lsn(), flush_lsn) AS flush_lsn,
|
||||
pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) AS replay_lsn,
|
||||
COALESCE(EXTRACT(EPOCH FROM write_lag), 0) AS write_lag,
|
||||
COALESCE(EXTRACT(EPOCH FROM flush_lag), 0) AS flush_lag,
|
||||
COALESCE(EXTRACT(EPOCH FROM replay_lag), 0) AS replay_lag,
|
||||
sync_state
|
||||
FROM pg_stat_replication
|
||||
WHERE host(client_addr) || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = %(repid)s
|
||||
test_args:
|
||||
repid: 127.0.0.1_test_rep
|
||||
|
||||
|
||||
##
|
||||
# Per-slot metrics
|
||||
##
|
||||
slot_stats:
|
||||
type: row
|
||||
query:
|
||||
90400: >
|
||||
SELECT NULL as active_pid,
|
||||
xmin,
|
||||
pg_xlog_location_diff(pg_current_xlog_location(), restart_lsn) AS restart_bytes,
|
||||
NULL AS confirmed_flush_bytes
|
||||
FROM pg_replication_slots WHERE slot_name = %(slot)s
|
||||
90600: >
|
||||
SELECT active_pid,
|
||||
xmin,
|
||||
pg_xlog_location_diff(pg_current_xlog_location(), restart_lsn) AS restart_bytes,
|
||||
pg_xlog_location_diff(pg_current_xlog_location(), confirmed_flush_lsn) AS confirmed_flush_bytes
|
||||
FROM pg_replication_slots WHERE slot_name = %(slot)s
|
||||
100000: >
|
||||
SELECT active_pid,
|
||||
xmin,
|
||||
pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) AS restart_bytes,
|
||||
pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn) AS confirmed_flush_bytes
|
||||
FROM pg_replication_slots WHERE slot_name = %(slot)s
|
||||
test_args:
|
||||
slot: test_slot
|
||||
|
||||
|
||||
##
|
||||
# Debugging
|
||||
##
|
||||
ntables:
|
||||
type: value
|
||||
query:
|
||||
0: SELECT count(*) AS ntables FROM pg_stat_user_tables
|
||||
|
||||
# Per-replication metrics
|
||||
rep_stats:
|
||||
type: row
|
||||
query:
|
||||
0: SELECT * FROM pg_stat_database WHERE client_addr || '_' || regexp_replace(application_name, '[ ,]', '_', 'g') = '{repid}'
|
||||
|
||||
# Debugging
|
||||
sleep:
|
||||
type: value
|
||||
query:
|
||||
0: SELECT now(), pg_sleep(5);
|
||||
|
||||
# Per-slot metrics
|
||||
slot_stats:
|
||||
type: row
|
||||
query:
|
||||
0: SELECT active_pid, xmin, pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) AS restart_bytes, pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn) AS confirmed_flush_bytes FROM pg_replication_slots WHERE slot_name = '{slot}'
|
||||
#sequence_usage:
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# The address the agent binds to
|
||||
#address: 127.0.0.1
|
||||
|
||||
# The port the agent listens on for requests
|
||||
#port: 5400
|
||||
|
||||
@@ -26,6 +29,9 @@
|
||||
# Default database to connect to when none is specified for a metric
|
||||
#dbname: 'postgres'
|
||||
|
||||
# SSL connection mode
|
||||
#ssl_mode: require
|
||||
|
||||
# Timeout for getting a connection slot from a pool
|
||||
#pool_slot_timeout: 5
|
||||
|
||||
|
||||
729
src/pgmon.py
729
src/pgmon.py
@@ -1,85 +1,149 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
pgmon is a monitoring intermediary that sits between a PostgreSQL cluster and a monitoring systen
|
||||
that is capable of parsing JSON responses over an HTTP connection.
|
||||
"""
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
|
||||
import yaml
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
|
||||
import sys
|
||||
import signal
|
||||
import argparse
|
||||
import logging
|
||||
import re
|
||||
|
||||
from decimal import Decimal
|
||||
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from http.server import ThreadingHTTPServer
|
||||
|
||||
from threading import Lock
|
||||
|
||||
import yaml
|
||||
|
||||
import psycopg2
|
||||
from psycopg2.extras import RealDictCursor
|
||||
from psycopg2.pool import ThreadedConnectionPool
|
||||
|
||||
from contextlib import contextmanager
|
||||
import requests
|
||||
|
||||
import signal
|
||||
from threading import Thread, Lock, Semaphore
|
||||
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from http.server import ThreadingHTTPServer
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
VERSION = "1.1.0-rc1"
|
||||
|
||||
VERSION = "1.0.1"
|
||||
|
||||
# Configuration
|
||||
config = {}
|
||||
class Context:
|
||||
"""
|
||||
The global context for connections, config, version, nad IPC
|
||||
"""
|
||||
|
||||
# Dictionary of current PostgreSQL connection pools
|
||||
connections_lock = Lock()
|
||||
connections = {}
|
||||
# Configuration
|
||||
config = {}
|
||||
|
||||
# Dictionary of unhappy databases. Keys are database names, value is the time
|
||||
# the database was determined to be unhappy plus the cooldown setting. So,
|
||||
# basically it's the time when we should try to connect to the database again.
|
||||
unhappy_cooldown = {}
|
||||
# Dictionary of current PostgreSQL connection pools
|
||||
connections_lock = Lock()
|
||||
connections = {}
|
||||
|
||||
# Version information
|
||||
cluster_version = None
|
||||
cluster_version_next_check = None
|
||||
cluster_version_lock = Lock()
|
||||
# Dictionary of unhappy databases. Keys are database names, value is the time
|
||||
# the database was determined to be unhappy plus the cooldown setting. So,
|
||||
# basically it's the time when we should try to connect to the database again.
|
||||
unhappy_cooldown = {}
|
||||
|
||||
# Running state (used to gracefully shut down)
|
||||
running = True
|
||||
# Version information
|
||||
cluster_version = None
|
||||
cluster_version_next_check = None
|
||||
cluster_version_lock = Lock()
|
||||
|
||||
# The http server object
|
||||
httpd = None
|
||||
# PostgreSQL latest version information
|
||||
latest_version = None
|
||||
latest_version_next_check = None
|
||||
latest_version_lock = Lock()
|
||||
release_supported = None
|
||||
|
||||
# Where the config file lives
|
||||
config_file = None
|
||||
# Running state (used to gracefully shut down)
|
||||
running = True
|
||||
|
||||
# Configure logging
|
||||
log = logging.getLogger(__name__)
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s - %(levelname)s - %(filename)s: %(funcName)s() line %(lineno)d: %(message)s"
|
||||
)
|
||||
console_log_handler = logging.StreamHandler()
|
||||
console_log_handler.setFormatter(formatter)
|
||||
log.addHandler(console_log_handler)
|
||||
# The http server object
|
||||
httpd = None
|
||||
|
||||
# Where the config file lives
|
||||
config_file = None
|
||||
|
||||
# Configure logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@classmethod
|
||||
def init_logging(cls):
|
||||
"""
|
||||
Actually initialize the logging framework. Since we don't ever instantiate the Context
|
||||
class, this provides a way to make a few modifications to the log handler.
|
||||
"""
|
||||
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s - %(levelname)s - %(filename)s: "
|
||||
"%(funcName)s() line %(lineno)d: %(message)s"
|
||||
)
|
||||
console_log_handler = logging.StreamHandler()
|
||||
console_log_handler.setFormatter(formatter)
|
||||
cls.log.addHandler(console_log_handler)
|
||||
|
||||
|
||||
# Error types
|
||||
class ConfigError(Exception):
|
||||
pass
|
||||
"""
|
||||
Error type for all config related errors.
|
||||
"""
|
||||
|
||||
|
||||
class DisconnectedError(Exception):
|
||||
pass
|
||||
"""
|
||||
Error indicating a previously active connection to the database has been disconnected.
|
||||
"""
|
||||
|
||||
|
||||
class UnhappyDBError(Exception):
|
||||
pass
|
||||
"""
|
||||
Error indicating that a database the code has been asked to connect to is on the unhappy list.
|
||||
"""
|
||||
|
||||
|
||||
class UnknownMetricError(Exception):
|
||||
"""
|
||||
Error indicating that an undefined metric was requested.
|
||||
"""
|
||||
|
||||
|
||||
class MetricVersionError(Exception):
|
||||
pass
|
||||
"""
|
||||
Error indicating that there is no suitable query for a metric that was requested for the
|
||||
version of PostgreSQL being monitored.
|
||||
"""
|
||||
|
||||
|
||||
class LatestVersionCheckError(Exception):
|
||||
"""
|
||||
Error indicating that there was a problem retrieving or parsing the latest version information.
|
||||
"""
|
||||
|
||||
|
||||
class InvalidDataError(Exception):
|
||||
"""
|
||||
Error indicating query results were somehow invalid
|
||||
"""
|
||||
|
||||
|
||||
# Default config settings
|
||||
default_config = {
|
||||
DEFAULT_CONFIG = {
|
||||
# The address the agent binds to
|
||||
"address": "127.0.0.1",
|
||||
# The port the agent listens on for requests
|
||||
"port": 5400,
|
||||
# Min PostgreSQL connection pool size (per database)
|
||||
@@ -98,6 +162,8 @@ default_config = {
|
||||
"dbport": 5432,
|
||||
# Default database to connect to when none is specified for a metric
|
||||
"dbname": "postgres",
|
||||
# SSL connection mode
|
||||
"ssl_mode": "require",
|
||||
# Timeout for getting a connection slot from a pool
|
||||
"pool_slot_timeout": 5,
|
||||
# PostgreSQL connection timeout (seconds)
|
||||
@@ -107,6 +173,8 @@ default_config = {
|
||||
"reconnect_cooldown": 30,
|
||||
# How often to check the version of PostgreSQL (seconds)
|
||||
"version_check_period": 300,
|
||||
# How often to check the latest supported version of PostgreSQL (seconds)
|
||||
"latest_version_check_period": 86400,
|
||||
# Metrics
|
||||
"metrics": {},
|
||||
}
|
||||
@@ -151,27 +219,16 @@ def update_deep(d1, d2):
|
||||
return d1
|
||||
|
||||
|
||||
def read_config(path, included=False):
|
||||
def validate_metric(path, name, metric):
|
||||
"""
|
||||
Read a config file.
|
||||
Validate a metric definition from a given file. If any query definitions come from external
|
||||
files, the metric dict will be updated with the actual query.
|
||||
|
||||
params:
|
||||
path: path to the file to read
|
||||
included: is this file included by another file?
|
||||
Params:
|
||||
path: path to the file which contains this definition
|
||||
name: name of the metric
|
||||
metric: the dictionary containing the metric definition
|
||||
"""
|
||||
# Read config file
|
||||
log.info("Reading log file: {}".format(path))
|
||||
with open(path, "r") as f:
|
||||
try:
|
||||
cfg = yaml.safe_load(f)
|
||||
except yaml.parser.ParserError as e:
|
||||
raise ConfigError("Inavlid config file: {}: {}".format(path, e))
|
||||
|
||||
# Since we use it a few places, get the base directory from the config
|
||||
config_base = os.path.dirname(path)
|
||||
|
||||
# Read any external queries and validate metric definitions
|
||||
for name, metric in cfg.get("metrics", {}).items():
|
||||
# Validate return types
|
||||
try:
|
||||
if metric["type"] not in ["value", "row", "column", "set"]:
|
||||
@@ -180,14 +237,14 @@ def read_config(path, included=False):
|
||||
metric["type"], name, path
|
||||
)
|
||||
)
|
||||
except KeyError:
|
||||
except KeyError as e:
|
||||
raise ConfigError(
|
||||
"No type specified for metric {} in {}".format(name, path)
|
||||
)
|
||||
) from e
|
||||
|
||||
# Ensure queries exist
|
||||
query_dict = metric.get("query", {})
|
||||
if type(query_dict) is not dict:
|
||||
if not isinstance(query_dict, dict):
|
||||
raise ConfigError(
|
||||
"Query definition should be a dictionary, got: {} for metric {} in {}".format(
|
||||
query_dict, name, path
|
||||
@@ -198,22 +255,47 @@ def read_config(path, included=False):
|
||||
raise ConfigError("Missing queries for metric {} in {}".format(name, path))
|
||||
|
||||
# Read external sql files and validate version keys
|
||||
config_base = os.path.dirname(path)
|
||||
for vers, query in metric["query"].items():
|
||||
try:
|
||||
int(vers)
|
||||
except:
|
||||
except Exception as e:
|
||||
raise ConfigError(
|
||||
"Invalid version: {} for metric {} in {}".format(vers, name, path)
|
||||
)
|
||||
) from e
|
||||
|
||||
# Read in the external query and update the definition in the metricdictionary
|
||||
if query.startswith("file:"):
|
||||
query_path = query[5:]
|
||||
if not query_path.startswith("/"):
|
||||
query_path = os.path.join(config_base, query_path)
|
||||
with open(query_path, "r") as f:
|
||||
with open(query_path, "r", encoding="utf-8") as f:
|
||||
metric["query"][vers] = f.read()
|
||||
|
||||
|
||||
def read_config(path, included=False):
|
||||
"""
|
||||
Read a config file.
|
||||
|
||||
params:
|
||||
path: path to the file to read
|
||||
included: is this file included by another file?
|
||||
"""
|
||||
|
||||
# Read config file
|
||||
Context.log.info("Reading log file: %s", path)
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
try:
|
||||
cfg = yaml.safe_load(f)
|
||||
except yaml.parser.ParserError as e:
|
||||
raise ConfigError("Inavlid config file: {}: {}".format(path, e)) from e
|
||||
|
||||
# Read any external queries and validate metric definitions
|
||||
for name, metric in cfg.get("metrics", {}).items():
|
||||
validate_metric(path, name, metric)
|
||||
|
||||
# Read any included config files
|
||||
config_base = os.path.dirname(path)
|
||||
for inc in cfg.get("include", []):
|
||||
# Prefix relative paths with the directory from the current config
|
||||
if not inc.startswith("/"):
|
||||
@@ -224,14 +306,14 @@ def read_config(path, included=False):
|
||||
# config
|
||||
if included:
|
||||
return cfg
|
||||
else:
|
||||
|
||||
new_config = {}
|
||||
update_deep(new_config, default_config)
|
||||
update_deep(new_config, DEFAULT_CONFIG)
|
||||
update_deep(new_config, cfg)
|
||||
|
||||
# Minor sanity checks
|
||||
if len(new_config["metrics"]) == 0:
|
||||
log.error("No metrics are defined")
|
||||
Context.log.error("No metrics are defined")
|
||||
raise ConfigError("No metrics defined")
|
||||
|
||||
# Validate the new log level before changing the config
|
||||
@@ -244,14 +326,17 @@ def read_config(path, included=False):
|
||||
]:
|
||||
raise ConfigError("Invalid log level: {}".format(new_config["log_level"]))
|
||||
|
||||
global config
|
||||
config = new_config
|
||||
Context.config = new_config
|
||||
|
||||
# Apply changes to log level
|
||||
log.setLevel(logging.getLevelName(config["log_level"].upper()))
|
||||
Context.log.setLevel(logging.getLevelName(Context.config["log_level"].upper()))
|
||||
|
||||
# Return the config (mostly to make pylint happy, but also in case I opt to remove the side
|
||||
# effect and make this more functional.
|
||||
return Context.config
|
||||
|
||||
|
||||
def signal_handler(sig, frame):
|
||||
def signal_handler(sig, frame): # pylint: disable=unused-argument
|
||||
"""
|
||||
Function for handling signals
|
||||
|
||||
@@ -262,19 +347,22 @@ def signal_handler(sig, frame):
|
||||
|
||||
# Signal everything to shut down
|
||||
if sig in [signal.SIGINT, signal.SIGTERM, signal.SIGQUIT]:
|
||||
log.info("Shutting down ...")
|
||||
global running
|
||||
running = False
|
||||
if httpd is not None:
|
||||
httpd.socket.close()
|
||||
Context.log.info("Shutting down ...")
|
||||
Context.running = False
|
||||
if Context.httpd is not None:
|
||||
Context.httpd.socket.close()
|
||||
|
||||
# Signal a reload
|
||||
if sig == signal.SIGHUP:
|
||||
log.warning("Received config reload signal")
|
||||
read_config(config_file)
|
||||
Context.log.warning("Received config reload signal")
|
||||
read_config(Context.config_file)
|
||||
|
||||
|
||||
class ConnectionPool(ThreadedConnectionPool):
|
||||
"""
|
||||
Threaded connection pool that has a context manager.
|
||||
"""
|
||||
|
||||
def __init__(self, dbname, minconn, maxconn, *args, **kwargs):
|
||||
# Make sure dbname isn't different in the kwargs
|
||||
kwargs["dbname"] = dbname
|
||||
@@ -283,7 +371,14 @@ class ConnectionPool(ThreadedConnectionPool):
|
||||
self.name = dbname
|
||||
|
||||
@contextmanager
|
||||
def connection(self, timeout=None):
|
||||
def connection(self, timeout):
|
||||
"""
|
||||
Connection context manager for our connection pool. This will attempt to retrieve a
|
||||
connection until the timeout is reached.
|
||||
|
||||
Params:
|
||||
timeout: how long to keep trying to get a connection bedore giving up
|
||||
"""
|
||||
conn = None
|
||||
timeout_time = datetime.now() + timedelta(timeout)
|
||||
# We will continue to try to get a connection slot until we time out
|
||||
@@ -307,33 +402,37 @@ class ConnectionPool(ThreadedConnectionPool):
|
||||
def get_pool(dbname):
|
||||
"""
|
||||
Get a database connection pool.
|
||||
|
||||
Params:
|
||||
dbname: the name of the database for which a connection pool should be returned.
|
||||
"""
|
||||
# Check if the db is unhappy and wants to be left alone
|
||||
if dbname in unhappy_cooldown:
|
||||
if unhappy_cooldown[dbname] > datetime.now():
|
||||
if dbname in Context.unhappy_cooldown:
|
||||
if Context.unhappy_cooldown[dbname] > datetime.now():
|
||||
raise UnhappyDBError()
|
||||
|
||||
# Create a connection pool if it doesn't already exist
|
||||
if dbname not in connections:
|
||||
with connections_lock:
|
||||
if dbname not in Context.connections:
|
||||
with Context.connections_lock:
|
||||
# Make sure nobody created the pool while we were waiting on the
|
||||
# lock
|
||||
if dbname not in connections:
|
||||
log.info("Creating connection pool for: {}".format(dbname))
|
||||
connections[dbname] = ConnectionPool(
|
||||
if dbname not in Context.connections:
|
||||
Context.log.info("Creating connection pool for: %s", dbname)
|
||||
# Actually create the connection pool
|
||||
Context.connections[dbname] = ConnectionPool(
|
||||
dbname,
|
||||
int(config["min_pool_size"]),
|
||||
int(config["max_pool_size"]),
|
||||
int(Context.config["min_pool_size"]),
|
||||
int(Context.config["max_pool_size"]),
|
||||
application_name="pgmon",
|
||||
host=config["dbhost"],
|
||||
port=config["dbport"],
|
||||
user=config["dbuser"],
|
||||
connect_timeout=int(config["connect_timeout"]),
|
||||
sslmode="require",
|
||||
host=Context.config["dbhost"],
|
||||
port=Context.config["dbport"],
|
||||
user=Context.config["dbuser"],
|
||||
connect_timeout=int(Context.config["connect_timeout"]),
|
||||
sslmode=Context.config["ssl_mode"],
|
||||
)
|
||||
# Clear the unhappy indicator if present
|
||||
unhappy_cooldown.pop(dbname, None)
|
||||
return connections[dbname]
|
||||
Context.unhappy_cooldown.pop(dbname, None)
|
||||
return Context.connections[dbname]
|
||||
|
||||
|
||||
def handle_connect_failure(pool):
|
||||
@@ -341,8 +440,8 @@ def handle_connect_failure(pool):
|
||||
Mark the database as being unhappy so we can leave it alone for a while
|
||||
"""
|
||||
dbname = pool.name
|
||||
unhappy_cooldown[dbname] = datetime.now() + timedelta(
|
||||
seconds=int(config["reconnect_cooldown"])
|
||||
Context.unhappy_cooldown[dbname] = datetime.now() + timedelta(
|
||||
seconds=int(Context.config["reconnect_cooldown"])
|
||||
)
|
||||
|
||||
|
||||
@@ -366,31 +465,73 @@ def get_query(metric, version):
|
||||
raise MetricVersionError("Missing metric query for PostgreSQL {}".format(version))
|
||||
|
||||
|
||||
def json_encode_special(obj):
|
||||
"""
|
||||
Encoder function to handle types the standard JSON package doesn't know what
|
||||
to do with
|
||||
"""
|
||||
if isinstance(obj, Decimal):
|
||||
return float(obj)
|
||||
raise TypeError("Cannot serialize object of {}".format(type(obj)))
|
||||
|
||||
|
||||
def json_encode_result(return_type, res):
|
||||
"""
|
||||
Return a json string encoding of the results of a query.
|
||||
|
||||
params:
|
||||
return_type: the expected structure to return. One of:
|
||||
value, row, column, set
|
||||
res: the query results
|
||||
|
||||
returns: a json string form of the results
|
||||
|
||||
raises:
|
||||
ConfigError: when an invalid return_type is given
|
||||
InvalidDataError: when the query results don't match the return type
|
||||
"""
|
||||
try:
|
||||
if return_type == "value":
|
||||
if len(res) == 0:
|
||||
return ""
|
||||
return str(list(res[0].values())[0])
|
||||
|
||||
if return_type == "row":
|
||||
return json.dumps(
|
||||
res[0] if len(res) > 0 else {}, default=json_encode_special
|
||||
)
|
||||
|
||||
if return_type == "column":
|
||||
return json.dumps(
|
||||
[list(r.values())[0] for r in res], default=json_encode_special
|
||||
)
|
||||
|
||||
if return_type == "set":
|
||||
return json.dumps(res, default=json_encode_special)
|
||||
except IndexError as e:
|
||||
raise InvalidDataError(e) from e
|
||||
|
||||
# If we got to this point, the return type is invalid
|
||||
raise ConfigError("Invalid query return type: {}".format(return_type))
|
||||
|
||||
|
||||
def run_query_no_retry(pool, return_type, query, args):
|
||||
"""
|
||||
Run the query with no explicit retry code
|
||||
"""
|
||||
with pool.connection(float(config["connect_timeout"])) as conn:
|
||||
with pool.connection(float(Context.config["connect_timeout"])) as conn:
|
||||
try:
|
||||
with conn.cursor(cursor_factory=RealDictCursor) as curs:
|
||||
curs.execute(query, args)
|
||||
res = curs.fetchall()
|
||||
|
||||
if return_type == "value":
|
||||
return str(list(res[0].values())[0])
|
||||
elif return_type == "row":
|
||||
return json.dumps(res[0])
|
||||
elif return_type == "column":
|
||||
return json.dumps([list(r.values())[0] for r in res])
|
||||
elif return_type == "set":
|
||||
return json.dumps(res)
|
||||
except:
|
||||
return json_encode_result(return_type, res)
|
||||
except Exception as e:
|
||||
dbname = pool.name
|
||||
if dbname in unhappy_cooldown:
|
||||
raise UnhappyDBError()
|
||||
elif conn.broken:
|
||||
raise DisconnectedError()
|
||||
else:
|
||||
if dbname in Context.unhappy_cooldown:
|
||||
raise UnhappyDBError() from e
|
||||
if conn.closed != 0:
|
||||
raise DisconnectedError() from e
|
||||
raise
|
||||
|
||||
|
||||
@@ -412,7 +553,7 @@ def run_query(pool, return_type, query, args):
|
||||
try:
|
||||
return run_query_no_retry(pool, return_type, query, args)
|
||||
except DisconnectedError:
|
||||
log.warning("Stale PostgreSQL connection found ... trying again")
|
||||
Context.log.warning("Stale PostgreSQL connection found ... trying again")
|
||||
# This sleep is an annoying hack to give the pool workers time to
|
||||
# actually mark the connection, otherwise it can be given back in the
|
||||
# next connection() call
|
||||
@@ -420,9 +561,9 @@ def run_query(pool, return_type, query, args):
|
||||
time.sleep(1)
|
||||
try:
|
||||
return run_query_no_retry(pool, return_type, query, args)
|
||||
except:
|
||||
except Exception as e:
|
||||
handle_connect_failure(pool)
|
||||
raise UnhappyDBError()
|
||||
raise UnhappyDBError() from e
|
||||
|
||||
|
||||
def get_cluster_version():
|
||||
@@ -430,40 +571,225 @@ def get_cluster_version():
|
||||
Get the PostgreSQL version if we don't already know it, or if it's been
|
||||
too long sice the last time it was checked.
|
||||
"""
|
||||
global cluster_version
|
||||
global cluster_version_next_check
|
||||
|
||||
# If we don't know the version or it's past the recheck time, get the
|
||||
# version from the database. Only one thread needs to do this, so they all
|
||||
# try to grab the lock, and then make sure nobody else beat them to it.
|
||||
if (
|
||||
cluster_version is None
|
||||
or cluster_version_next_check is None
|
||||
or cluster_version_next_check < datetime.now()
|
||||
Context.cluster_version is None
|
||||
or Context.cluster_version_next_check is None
|
||||
or Context.cluster_version_next_check < datetime.now()
|
||||
):
|
||||
with cluster_version_lock:
|
||||
with Context.cluster_version_lock:
|
||||
# Only check if nobody already got the version before us
|
||||
if (
|
||||
cluster_version is None
|
||||
or cluster_version_next_check is None
|
||||
or cluster_version_next_check < datetime.now()
|
||||
Context.cluster_version is None
|
||||
or Context.cluster_version_next_check is None
|
||||
or Context.cluster_version_next_check < datetime.now()
|
||||
):
|
||||
log.info("Checking PostgreSQL cluster version")
|
||||
pool = get_pool(config["dbname"])
|
||||
cluster_version = int(
|
||||
Context.log.info("Checking PostgreSQL cluster version")
|
||||
pool = get_pool(Context.config["dbname"])
|
||||
Context.cluster_version = int(
|
||||
run_query(pool, "value", "SHOW server_version_num", None)
|
||||
)
|
||||
cluster_version_next_check = datetime.now() + timedelta(
|
||||
seconds=int(config["version_check_period"])
|
||||
Context.cluster_version_next_check = datetime.now() + timedelta(
|
||||
seconds=int(Context.config["version_check_period"])
|
||||
)
|
||||
log.info("Got PostgreSQL cluster version: {}".format(cluster_version))
|
||||
log.debug(
|
||||
"Next PostgreSQL cluster version check will be after: {}".format(
|
||||
cluster_version_next_check
|
||||
Context.log.info(
|
||||
"Got PostgreSQL cluster version: %s", Context.cluster_version
|
||||
)
|
||||
Context.log.debug(
|
||||
"Next PostgreSQL cluster version check will be after: %s",
|
||||
Context.cluster_version_next_check,
|
||||
)
|
||||
|
||||
return cluster_version
|
||||
return Context.cluster_version
|
||||
|
||||
|
||||
def version_num_to_release(version_num):
|
||||
"""
|
||||
Extract the revease from a version_num.
|
||||
|
||||
In other words, this converts things like:
|
||||
90603 => 9.6
|
||||
130010 => 13
|
||||
"""
|
||||
if version_num // 10000 < 10:
|
||||
return version_num // 10000 + (version_num % 10000 // 100 / 10)
|
||||
return version_num // 10000
|
||||
|
||||
|
||||
def parse_version_rss(raw_rss, release):
|
||||
"""
|
||||
Parse the raw RSS from the versions.rss feed to extract the latest version of
|
||||
PostgreSQL that's availabe for the cluster being monitored.
|
||||
|
||||
This sets these Context variables:
|
||||
latest_version
|
||||
release_supported
|
||||
|
||||
It is expected that the caller already holds the latest_version_lock lock.
|
||||
|
||||
params:
|
||||
raw_rss: The raw rss text from versions.rss
|
||||
release: The PostgreSQL release we care about (ex: 9.2, 14)
|
||||
"""
|
||||
|
||||
# Regular expressions for parsing the RSS document
|
||||
version_line = re.compile(
|
||||
r".*?([0-9][0-9.]+) is the latest release in the {} series.*".format(release)
|
||||
)
|
||||
unsupported_line = re.compile(r"^This version is unsupported")
|
||||
|
||||
# Loop through the RSS until we find the current release
|
||||
release_found = False
|
||||
for line in raw_rss.splitlines():
|
||||
m = version_line.match(line)
|
||||
if m:
|
||||
# Note that we found the version we were looking for
|
||||
release_found = True
|
||||
|
||||
# Convert the version to version_num format
|
||||
version = m.group(1)
|
||||
parts = list(map(int, version.split(".")))
|
||||
if parts[0] < 10:
|
||||
Context.latest_version = int(
|
||||
"{}{:02}{:02}".format(parts[0], parts[1], parts[2])
|
||||
)
|
||||
else:
|
||||
Context.latest_version = int("{}00{:02}".format(parts[0], parts[1]))
|
||||
elif release_found:
|
||||
# The next line after the version tells if the version is supported
|
||||
if unsupported_line.match(line):
|
||||
Context.release_supported = False
|
||||
else:
|
||||
Context.release_supported = True
|
||||
break
|
||||
|
||||
# Make sure we actually found it
|
||||
if not release_found:
|
||||
raise LatestVersionCheckError("Current release ({}) not found".format(release))
|
||||
|
||||
Context.log.info(
|
||||
"Got latest PostgreSQL version: %s supported=%s",
|
||||
Context.latest_version,
|
||||
Context.release_supported,
|
||||
)
|
||||
Context.log.debug(
|
||||
"Next latest PostgreSQL version check will be after: %s",
|
||||
Context.latest_version_next_check,
|
||||
)
|
||||
|
||||
|
||||
def get_latest_version():
|
||||
"""
|
||||
Get the latest supported version of the major PostgreSQL release running on the server being
|
||||
monitored.
|
||||
"""
|
||||
|
||||
# If we don't know the latest version or it's past the recheck time, get the
|
||||
# version from the PostgreSQL RSS feed. Only one thread needs to do this, so
|
||||
# they all try to grab the lock, and then make sure nobody else beat them to it.
|
||||
if (
|
||||
Context.latest_version is None
|
||||
or Context.latest_version_next_check is None
|
||||
or Context.latest_version_next_check < datetime.now()
|
||||
):
|
||||
# Note: we get the cluster version here before grabbing the latest_version_lock
|
||||
# lock so it's not held while trying to talk with the DB.
|
||||
release = version_num_to_release(get_cluster_version())
|
||||
|
||||
with Context.latest_version_lock:
|
||||
# Only check if nobody already got the version before us
|
||||
if (
|
||||
Context.latest_version is None
|
||||
or Context.latest_version_next_check is None
|
||||
or Context.latest_version_next_check < datetime.now()
|
||||
):
|
||||
Context.log.info("Checking latest PostgreSQL version")
|
||||
Context.latest_version_next_check = datetime.now() + timedelta(
|
||||
seconds=int(Context.config["latest_version_check_period"])
|
||||
)
|
||||
|
||||
# Grab the RSS feed
|
||||
raw_rss = requests.get(
|
||||
"https://www.postgresql.org/versions.rss", timeout=30
|
||||
)
|
||||
if raw_rss.status_code != 200:
|
||||
raise LatestVersionCheckError("code={}".format(raw_rss.status_code))
|
||||
|
||||
# Parse the RSS body and set Context variables
|
||||
parse_version_rss(raw_rss.text, release)
|
||||
|
||||
return Context.latest_version
|
||||
|
||||
|
||||
def sample_metric(dbname, metric_name, args, retry=True):
|
||||
"""
|
||||
Run the appropriate query for the named metric against the specified database
|
||||
"""
|
||||
# Get the metric definition
|
||||
try:
|
||||
metric = Context.config["metrics"][metric_name]
|
||||
except KeyError as e:
|
||||
raise UnknownMetricError("Unknown metric: {}".format(metric_name)) from e
|
||||
|
||||
# Get the connection pool for the database, or create one if it doesn't
|
||||
# already exist.
|
||||
pool = get_pool(dbname)
|
||||
|
||||
# Identify the PostgreSQL version
|
||||
version = get_cluster_version()
|
||||
|
||||
# Get the query version
|
||||
query = get_query(metric, version)
|
||||
|
||||
# Execute the quert
|
||||
if retry:
|
||||
return run_query(pool, metric["type"], query, args)
|
||||
return run_query_no_retry(pool, metric["type"], query, args)
|
||||
|
||||
|
||||
def test_queries():
|
||||
"""
|
||||
Run all of the metric queries against a database and check the results
|
||||
"""
|
||||
# We just use the default db for tests
|
||||
dbname = Context.config["dbname"]
|
||||
# Loop through all defined metrics.
|
||||
for name, metric in Context.config["metrics"].items():
|
||||
# If the metric has arguments to use while testing, grab those
|
||||
args = metric.get("test_args", {})
|
||||
print(
|
||||
"Testing {} [{}]".format(
|
||||
name,
|
||||
", ".join(["{}={}".format(key, value) for key, value in args.items()]),
|
||||
)
|
||||
)
|
||||
# When testing against a docker container, we may end up connecting
|
||||
# before the service is truly up (it restarts during the initialization
|
||||
# phase). To cope with this, we'll allow a few connection failures.
|
||||
tries = 5
|
||||
while True:
|
||||
# Run the query without the ability to retry
|
||||
try:
|
||||
res = sample_metric(dbname, name, args, retry=False)
|
||||
break
|
||||
except MetricVersionError:
|
||||
res = "Unsupported for this version"
|
||||
break
|
||||
except psycopg2.OperationalError as e:
|
||||
print("Error encountered, {} tries left: {}".format(tries, e))
|
||||
if tries <= 0:
|
||||
raise
|
||||
time.sleep(1)
|
||||
tries -= 1
|
||||
# Compare the result to the provided sample results
|
||||
# TODO
|
||||
print("{} -> {}".format(name, res))
|
||||
# Return the number of errors
|
||||
# TODO
|
||||
return 0
|
||||
|
||||
|
||||
class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
|
||||
@@ -476,9 +802,8 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
|
||||
"""
|
||||
Override to suppress standard request logging
|
||||
"""
|
||||
pass
|
||||
|
||||
def do_GET(self):
|
||||
def do_GET(self): # pylint: disable=invalid-name
|
||||
"""
|
||||
Handle a request. This is just a wrapper around the actual handler
|
||||
code to keep things more readable.
|
||||
@@ -486,7 +811,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
|
||||
try:
|
||||
self._handle_request()
|
||||
except BrokenPipeError:
|
||||
log.error("Client disconnected, exiting handler")
|
||||
Context.log.error("Client disconnected, exiting handler")
|
||||
|
||||
def _handle_request(self):
|
||||
"""
|
||||
@@ -494,72 +819,52 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
|
||||
"""
|
||||
# Parse the URL
|
||||
parsed_path = urlparse(self.path)
|
||||
name = parsed_path.path.strip("/")
|
||||
metric_name = parsed_path.path.strip("/")
|
||||
parsed_query = parse_qs(parsed_path.query)
|
||||
|
||||
if name == "agent_version":
|
||||
if metric_name == "agent_version":
|
||||
self._reply(200, VERSION)
|
||||
return
|
||||
|
||||
elif metric_name == "latest_version_info":
|
||||
try:
|
||||
get_latest_version()
|
||||
self._reply(
|
||||
200,
|
||||
json.dumps(
|
||||
{
|
||||
"latest": Context.latest_version,
|
||||
"supported": 1 if Context.release_supported else 0,
|
||||
}
|
||||
),
|
||||
)
|
||||
except LatestVersionCheckError as e:
|
||||
Context.log.error(
|
||||
"Failed to retrieve latest version information: %s", e
|
||||
)
|
||||
self._reply(503, "Failed to retrieve latest version info")
|
||||
else:
|
||||
# Note: parse_qs returns the values as a list. Since we always expect
|
||||
# single values, just grab the first from each.
|
||||
args = {key: values[0] for key, values in parsed_query.items()}
|
||||
|
||||
# Get the metric definition
|
||||
try:
|
||||
metric = config["metrics"][name]
|
||||
except KeyError:
|
||||
log.error("Unknown metric: {}".format(name))
|
||||
self._reply(404, "Unknown metric")
|
||||
return
|
||||
|
||||
# Get the dbname. If none was provided, use the default from the
|
||||
# config.
|
||||
dbname = args.get("dbname", config["dbname"])
|
||||
dbname = args.get("dbname", Context.config["dbname"])
|
||||
|
||||
# Get the connection pool for the database, or create one if it doesn't
|
||||
# already exist.
|
||||
# Sample the metric
|
||||
try:
|
||||
pool = get_pool(dbname)
|
||||
except UnhappyDBError:
|
||||
log.info("Database {} is unhappy, please be patient".format(dbname))
|
||||
self._reply(503, "Database unavailable")
|
||||
return
|
||||
|
||||
# Identify the PostgreSQL version
|
||||
try:
|
||||
version = get_cluster_version()
|
||||
except UnhappyDBError:
|
||||
return
|
||||
except Exception as e:
|
||||
if dbname in unhappy_cooldown:
|
||||
log.info("Database {} is unhappy, please be patient".format(dbname))
|
||||
self._reply(503, "Database unavailable")
|
||||
else:
|
||||
log.error("Failed to get PostgreSQL version: {}".format(e))
|
||||
self._reply(500, "Error getting DB version")
|
||||
return
|
||||
|
||||
# Get the query version
|
||||
try:
|
||||
query = get_query(metric, version)
|
||||
except KeyError:
|
||||
log.error("Failed to find a version of {} for {}".format(name, version))
|
||||
self._reply(200, sample_metric(dbname, metric_name, args))
|
||||
except UnknownMetricError:
|
||||
Context.log.error("Unknown metric: %s", metric_name)
|
||||
self._reply(404, "Unknown metric")
|
||||
except MetricVersionError:
|
||||
Context.log.error("Failed to find an query version for %s", metric_name)
|
||||
self._reply(404, "Unsupported version")
|
||||
return
|
||||
|
||||
# Execute the quert
|
||||
try:
|
||||
self._reply(200, run_query(pool, metric["type"], query, args))
|
||||
return
|
||||
except Exception as e:
|
||||
if dbname in unhappy_cooldown:
|
||||
log.info("Database {} is unhappy, please be patient".format(dbname))
|
||||
except UnhappyDBError:
|
||||
Context.log.info("Database %s is unhappy, please be patient", dbname)
|
||||
self._reply(503, "Database unavailable")
|
||||
else:
|
||||
log.error("Error running query: {}".format(e))
|
||||
self._reply(500, "Error running query")
|
||||
return
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
Context.log.error("Error running query: %s", e)
|
||||
self._reply(500, "Unexpected error: {}".format(e))
|
||||
|
||||
def _reply(self, code, content):
|
||||
"""
|
||||
@@ -572,41 +877,63 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
|
||||
self.wfile.write(bytes(content, "utf-8"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
def main():
|
||||
"""
|
||||
Main application routine
|
||||
"""
|
||||
|
||||
# Initialize the logging framework
|
||||
Context.init_logging()
|
||||
|
||||
# Handle cli args
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="pgmon", description="A PostgreSQL monitoring agent"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"config_file",
|
||||
"-c",
|
||||
"--config_file",
|
||||
default="pgmon.yml",
|
||||
nargs="?",
|
||||
help="The config file to read (default: %(default)s)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-t", "--test", action="store_true", help="Run query tests and exit"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Set the config file path
|
||||
config_file = args.config_file
|
||||
Context.config_file = args.config_file
|
||||
|
||||
# Read the config file
|
||||
read_config(config_file)
|
||||
read_config(Context.config_file)
|
||||
|
||||
# Run query tests and exit if test mode is enabled
|
||||
if args.test:
|
||||
if test_queries() > 0:
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
# Set up the http server to receive requests
|
||||
server_address = ("127.0.0.1", config["port"])
|
||||
httpd = ThreadingHTTPServer(server_address, SimpleHTTPRequestHandler)
|
||||
server_address = (Context.config["address"], Context.config["port"])
|
||||
Context.httpd = ThreadingHTTPServer(server_address, SimpleHTTPRequestHandler)
|
||||
|
||||
# Set up the signal handler
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGHUP, signal_handler)
|
||||
|
||||
# Handle requests.
|
||||
log.info("Listening on port {}...".format(config["port"]))
|
||||
while running:
|
||||
httpd.handle_request()
|
||||
Context.log.info("Listening on port %s...", Context.config["port"])
|
||||
while Context.running:
|
||||
Context.httpd.handle_request()
|
||||
|
||||
# Clean up PostgreSQL connections
|
||||
# TODO: Improve this ... not sure it actually closes all the connections cleanly
|
||||
for pool in connections.values():
|
||||
for pool in Context.connections.values():
|
||||
pool.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,7 +7,7 @@ After=network.target
|
||||
[Service]
|
||||
EnvironmentFile=/etc/pgmon/%i-service.conf
|
||||
User=${SERVICE_USER:-postgres}
|
||||
ExecStart=/usr/local/bin/pgmon /etc/pgmon/%i.yml
|
||||
ExecStart=/usr/bin/pgmon -c /etc/pgmon/%i.yml
|
||||
ExecReload=kill -HUP $MAINPID
|
||||
Restart=on-failure
|
||||
Type=exec
|
||||
|
||||
23
tests/Dockerfile
Normal file
23
tests/Dockerfile
Normal file
@@ -0,0 +1,23 @@
|
||||
FROM alpine:3.21
|
||||
|
||||
RUN apk update && \
|
||||
apk add py3-psycopg2 \
|
||||
py3-requests \
|
||||
py3-yaml \
|
||||
tini
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY src/pgmon.py /app/
|
||||
|
||||
COPY sample-config/pgmon-metrics.yml /app/
|
||||
|
||||
COPY tests/test-config.yml /app/
|
||||
|
||||
COPY --chmod=0600 --chown=postgres:postgres tests/pgpass /root/.pgpass
|
||||
|
||||
ENTRYPOINT ["tini", "--"]
|
||||
|
||||
EXPOSE 5400
|
||||
|
||||
CMD ["/app/pgmon.py", "-c", "/app/test-config.yml", "--test"]
|
||||
32
tests/docker-compose.yml
Normal file
32
tests/docker-compose.yml
Normal file
@@ -0,0 +1,32 @@
|
||||
---
|
||||
|
||||
services:
|
||||
agent:
|
||||
image: pgmon
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: tests/Dockerfile
|
||||
ports:
|
||||
- :5400
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
||||
db:
|
||||
image: "postgres:${PGTAG:-17-bookworm}"
|
||||
ports:
|
||||
- :5432
|
||||
environment:
|
||||
POSTGRES_PASSWORD: secret
|
||||
healthcheck:
|
||||
#test: [ "CMD", "pg_isready", "-U", "postgres" ]
|
||||
test: [ "CMD-SHELL", "pg_controldata $(psql -At -U postgres -c 'show data_directory') | grep -q 'in production'" ]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 40
|
||||
command: >
|
||||
postgres -c ssl=on
|
||||
-c ssl_cert_file='/etc/ssl/certs/ssl-cert-snakeoil.pem'
|
||||
-c ssl_key_file='/etc/ssl/private/ssl-cert-snakeoil.key'
|
||||
-c listen_addresses='*'
|
||||
|
||||
1
tests/pgpass
Normal file
1
tests/pgpass
Normal file
@@ -0,0 +1 @@
|
||||
db:5432:*:postgres:secret
|
||||
66
tests/run-tests.sh
Executable file
66
tests/run-tests.sh
Executable file
@@ -0,0 +1,66 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Versions to test
|
||||
versions=( $@ )
|
||||
|
||||
# If we weren't given any versions, test them all
|
||||
if [ ${#versions[@]} -eq 0 ]
|
||||
then
|
||||
versions=( 9.2 9.4 9.6 10 11 12 13 14 15 16 17 )
|
||||
fi
|
||||
|
||||
# Image tags to use
|
||||
declare -A images=()
|
||||
images["9.2"]='9.2'
|
||||
images["9.3"]='9.3'
|
||||
images["9.4"]='9.4'
|
||||
images["9.5"]='9.5'
|
||||
images["9.6"]='9.6-bullseye'
|
||||
images["10"]='10-bullseye'
|
||||
images["11"]='11-bookworm'
|
||||
images["12"]='12-bookworm'
|
||||
images["13"]='13-bookworm'
|
||||
images["14"]='14-bookworm'
|
||||
images["15"]='15-bookworm'
|
||||
images["16"]='16-bookworm'
|
||||
images["17"]='17-bookworm'
|
||||
images["18"]='18-trixie'
|
||||
|
||||
declare -A results=()
|
||||
|
||||
# Make sure everything's down to start with
|
||||
docker compose down
|
||||
|
||||
# Make sure our agent container is up to date
|
||||
docker compose build agent
|
||||
|
||||
for version in "${versions[@]}"
|
||||
do
|
||||
echo
|
||||
echo "Testing: PostgreSQL ${version}"
|
||||
|
||||
# Specify the version we're testing against
|
||||
export PGTAG="${images["$version"]}"
|
||||
|
||||
# Start the containers
|
||||
docker compose up --exit-code-from=agent agent
|
||||
rc=$?
|
||||
|
||||
results["$version"]=$rc
|
||||
|
||||
# Destroy the containers
|
||||
docker compose down
|
||||
done
|
||||
|
||||
echo
|
||||
echo
|
||||
for v in "${versions[@]}"
|
||||
do
|
||||
case "${results["$v"]}" in
|
||||
0) msg="OK" ;;
|
||||
1) msg="Query failure detected" ;;
|
||||
18) msg="Docker image error: 18" ;;
|
||||
*) msg="Unexpected error: ${results["$v"]}" ;;
|
||||
esac
|
||||
echo "$v -> $msg"
|
||||
done
|
||||
17
tests/test-config.yml
Normal file
17
tests/test-config.yml
Normal file
@@ -0,0 +1,17 @@
|
||||
---
|
||||
|
||||
# Bind to all interfaces so we can submit requests from outside the test container
|
||||
address: 0.0.0.0
|
||||
|
||||
# We always just connect to the db container
|
||||
dbhost: db
|
||||
dbport: 5432
|
||||
dbuser: postgres
|
||||
|
||||
# The SSL cipher parameters are too old in the 9.2 container, so we allow the tests
|
||||
# to be run without encryption
|
||||
ssl_mode: prefer
|
||||
|
||||
# Pull in the standard metrics
|
||||
include:
|
||||
- pgmon-metrics.yml
|
||||
129
zabbix_templates/coverage.py
Executable file
129
zabbix_templates/coverage.py
Executable file
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Compare the items defined in a Zabbix template with the metrics defined in a config file.
|
||||
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
# Special built in metrics
|
||||
SPECIAL = {
|
||||
"agent_version",
|
||||
"latest_version_info"
|
||||
}
|
||||
|
||||
|
||||
class NonMetricItemError(Exception):
|
||||
"""
|
||||
A given item does not directly use a metric
|
||||
"""
|
||||
|
||||
|
||||
def read_metrics(file):
|
||||
"""
|
||||
Read the metrics from a config file and return the list of names
|
||||
|
||||
params:
|
||||
file: the name of the file to read
|
||||
|
||||
returns:
|
||||
list of metric named defined in the file
|
||||
|
||||
raises:
|
||||
yaml.parser.ParserError: invalid yaml file
|
||||
"""
|
||||
names = set()
|
||||
config = None
|
||||
|
||||
with open(file, "r", encoding="utf-8") as f:
|
||||
config = yaml.safe_load(f)
|
||||
|
||||
try:
|
||||
for m in config["metrics"].keys():
|
||||
names.add(m)
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
return names
|
||||
|
||||
|
||||
def extract_metric(item):
|
||||
"""
|
||||
Extract the metric from an item definition
|
||||
|
||||
params:
|
||||
item: the item/discovery/prototype definition dict
|
||||
|
||||
returns:
|
||||
the name of the metric used in the item
|
||||
|
||||
raises:
|
||||
NonMetricItemError: the item does not directly use a metric
|
||||
"""
|
||||
try:
|
||||
if item["type"] == "HTTP_AGENT":
|
||||
url = item["url"]
|
||||
if url.startswith("http://localhost:{$AGENT_PORT}"):
|
||||
return url.split("/")[-1]
|
||||
except KeyError:
|
||||
raise NonMetricItemError()
|
||||
|
||||
raise NonMetricItemError()
|
||||
|
||||
|
||||
def read_template(file):
|
||||
"""
|
||||
Read the items from a Zabbix template and return the list of metric names
|
||||
|
||||
params:
|
||||
file: the name of the file to read
|
||||
|
||||
returns:
|
||||
list of metric named used in the file
|
||||
|
||||
raises:
|
||||
yaml.parser.ParserError: invalid yaml file
|
||||
"""
|
||||
names = set()
|
||||
config = None
|
||||
|
||||
with open(file, "r", encoding="utf-8") as f:
|
||||
config = yaml.safe_load(f)
|
||||
|
||||
try:
|
||||
for template in config["zabbix_export"]["templates"]:
|
||||
for item in template["items"]:
|
||||
try:
|
||||
names.add(extract_metric(item))
|
||||
except NonMetricItemError:
|
||||
pass
|
||||
|
||||
for rule in template["discovery_rules"]:
|
||||
try:
|
||||
names.add(extract_metric(rule))
|
||||
except NonMetricItemError:
|
||||
pass
|
||||
|
||||
for proto in rule["item_prototypes"]:
|
||||
try:
|
||||
names.add(extract_metric(proto))
|
||||
except NonMetricItemError:
|
||||
pass
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
return names
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
config_file = sys.argv[1]
|
||||
config_metrics = read_metrics(config_file)
|
||||
|
||||
template_file = sys.argv[2]
|
||||
template_metrics = read_template(template_file) - SPECIAL
|
||||
|
||||
config_only = config_metrics - template_metrics
|
||||
template_only = template_metrics - config_metrics
|
||||
|
||||
print("Config only: {}".format(sorted(list(config_only))))
|
||||
print("Template only: {}".format(sorted(list(template_only))))
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user