From 85fbe5cb9067b5ecc02d785f1b5c6fe9b8b138c9 Mon Sep 17 00:00:00 2001 From: paddy Date: Tue, 4 May 2021 18:18:17 +0100 Subject: [PATCH 1/5] (SUP-2372) reset pgrepack task added --- tasks/reset_pgrepack_schema.json | 5 +++++ tasks/reset_pgrepack_schema.sh | 11 +++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tasks/reset_pgrepack_schema.json create mode 100644 tasks/reset_pgrepack_schema.sh diff --git a/tasks/reset_pgrepack_schema.json b/tasks/reset_pgrepack_schema.json new file mode 100644 index 0000000..a0b509c --- /dev/null +++ b/tasks/reset_pgrepack_schema.json @@ -0,0 +1,5 @@ +{ + "puppet_task_version": 1, + "supports_noop": false, + "description": "This task will remove and recreate the pg_repack extension" +} diff --git a/tasks/reset_pgrepack_schema.sh b/tasks/reset_pgrepack_schema.sh new file mode 100644 index 0000000..d2b28f5 --- /dev/null +++ b/tasks/reset_pgrepack_schema.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Puppet Task Name: reset_pgrepack_schema + +#Remove the pg_repack extension +su - pe-postgres -s '/bin/bash' -c '/opt/puppetlabs/server/bin/psql -d pe-puppetdb -c "DROP EXTENSION pg_repack CASCADE"' + +#Then, recreate the pg_repack extension +su - pe-postgres -s '/bin/bash' -c '/opt/puppetlabs/server/bin/psql -d pe-puppetdb -c "CREATE EXTENSION pg_repack"' + +echo " -- reset_pgrepack_scheme Task ended: $(date +%s) --" \ No newline at end of file From 930b000d1ef74159a3a07d183bf884d9782ea405 Mon Sep 17 00:00:00 2001 From: paddy Date: Tue, 4 May 2021 18:25:34 +0100 Subject: [PATCH 2/5] newline added to end of task --- tasks/reset_pgrepack_schema.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/reset_pgrepack_schema.sh b/tasks/reset_pgrepack_schema.sh index d2b28f5..0fe7cb6 100644 --- a/tasks/reset_pgrepack_schema.sh +++ b/tasks/reset_pgrepack_schema.sh @@ -8,4 +8,4 @@ su - pe-postgres -s '/bin/bash' -c '/opt/puppetlabs/server/bin/psql -d pe-puppet #Then, recreate the pg_repack extension su - pe-postgres -s '/bin/bash' -c '/opt/puppetlabs/server/bin/psql -d pe-puppetdb -c "CREATE EXTENSION pg_repack"' -echo " -- reset_pgrepack_scheme Task ended: $(date +%s) --" \ No newline at end of file +echo " -- reset_pgrepack_scheme Task ended: $(date +%s) --" From 3a52625db9303d1f297572f8d949683745e62f15 Mon Sep 17 00:00:00 2001 From: paddy Date: Fri, 7 May 2021 17:58:41 +0100 Subject: [PATCH 3/5] Add some error checking for the PE-PostgreSQL service --- tasks/reset_pgrepack_schema.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tasks/reset_pgrepack_schema.sh b/tasks/reset_pgrepack_schema.sh index 0fe7cb6..2c4ab59 100644 --- a/tasks/reset_pgrepack_schema.sh +++ b/tasks/reset_pgrepack_schema.sh @@ -2,10 +2,15 @@ # Puppet Task Name: reset_pgrepack_schema -#Remove the pg_repack extension -su - pe-postgres -s '/bin/bash' -c '/opt/puppetlabs/server/bin/psql -d pe-puppetdb -c "DROP EXTENSION pg_repack CASCADE"' +#Determine if PE Postgres is available +if puppet resource service pe-postgresql | grep -q running; then + #Remove the pg_repack extension + su - pe-postgres -s '/bin/bash' -c '/opt/puppetlabs/server/bin/psql -d pe-puppetdb -c "DROP EXTENSION pg_repack CASCADE"' -#Then, recreate the pg_repack extension -su - pe-postgres -s '/bin/bash' -c '/opt/puppetlabs/server/bin/psql -d pe-puppetdb -c "CREATE EXTENSION pg_repack"' + #Then, recreate the pg_repack extension + su - pe-postgres -s '/bin/bash' -c '/opt/puppetlabs/server/bin/psql -d pe-puppetdb -c "CREATE EXTENSION pg_repack"' +else + echo " -- No running PE-PostgreSQL instance found, please run this task against your Primary or PE-PostgreSQL server --" +fi echo " -- reset_pgrepack_scheme Task ended: $(date +%s) --" From d2721dcf34bf26ff7bc313d01dfcaa059109343c Mon Sep 17 00:00:00 2001 From: paddy Date: Mon, 10 May 2021 18:24:55 +0100 Subject: [PATCH 4/5] Read me updated with task info --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 043ce7a..fe80643 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,16 @@ This module provides cron jobs to VACUUM FULL tables in the `pe-puppetdb` databa With PE 2018.1.7 and 2019.0.2 and newer, this module uses `pg_repack` instead of `VACUUM FULL`. +Please note that when using `pg_repack` as part of the pe_databases module, unclean exits can leave behind the schema when otherwise it should have been cleaned up. This can result in the messages similar to the following: + +``` +INFO: repacking table "public.fact_paths" +WARNING: the table "public.fact_paths" already has a trigger called "repack_trigger" +DETAIL: The trigger was probably installed during a previous attempt to run pg_repack on the table which was interrupted and for some reason failed to clean up the temporary objects. Please drop the trigger or drop and recreate the pg_repack extension altogether to remove all the temporary objects left over. +``` + +The module now contains a task `reset_pgrepack_schema` to mitigate this issue. This needs to be run against your Primary or Postgrsql server to resolve this and it will drop and recreate the extension, removing the temporary objects. + ### Vacuuming Generally speaking, PostgreSQL keeps itself in good shape with a process called [auto vacuuming](https://www.postgresql.org/docs/11/runtime-config-autovacuum.html). From a7763da7bbab6937d4e5bfe9569126cdf35368df Mon Sep 17 00:00:00 2001 From: martyewings Date: Thu, 13 May 2021 11:34:02 +0100 Subject: [PATCH 5/5] added in bash helper --- files/common.sh | 37 ++++++++++++++++++++++++++++++++ tasks/reset_pgrepack_schema.json | 5 ++++- tasks/reset_pgrepack_schema.sh | 12 +++++++---- 3 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 files/common.sh diff --git a/files/common.sh b/files/common.sh new file mode 100644 index 0000000..8af0b0f --- /dev/null +++ b/files/common.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# TODO: helper task? + +# Exit with an error message and error code, defaulting to 1 +fail() { + # Print a stderr: entry if there were anything printed to stderr + if [[ -s $_tmp ]]; then + # Hack to try and output valid json by replacing newlines with spaces. + echo "{ \"status\": \"error\", \"message\": \"$1\", \"stderr\": \"$(tr '\n' ' ' <$_tmp)\" }" + else + echo "{ \"status\": \"error\", \"message\": \"$1\" }" + fi + + exit ${2:-1} +} + +success() { + echo "$1" + exit 0 +} + +# Test for colors. If unavailable, unset variables are ok +if tput colors &>/dev/null; then + green="$(tput setaf 2)" + red="$(tput setaf 1)" + reset="$(tput sgr0)" +fi + +_tmp="$(mktemp)" +exec 2>>"$_tmp" + +# Use indirection to munge PT_ environment variables +# e.g. "$PT_version" becomes "$version" +for v in ${!PT_*}; do + declare "${v#*PT_}"="${!v}" +done diff --git a/tasks/reset_pgrepack_schema.json b/tasks/reset_pgrepack_schema.json index a0b509c..10622c9 100644 --- a/tasks/reset_pgrepack_schema.json +++ b/tasks/reset_pgrepack_schema.json @@ -1,5 +1,8 @@ { "puppet_task_version": 1, "supports_noop": false, - "description": "This task will remove and recreate the pg_repack extension" + "description": "This task will remove and recreate the pg_repack extension", + "implementations": [ + {"name": "reset_pgrepack_schema.sh", "requirements": ["shell"], "files": ["pe_databases/files/common.sh"], "input_method": "environment"} + ] } diff --git a/tasks/reset_pgrepack_schema.sh b/tasks/reset_pgrepack_schema.sh index 2c4ab59..84eb0bf 100644 --- a/tasks/reset_pgrepack_schema.sh +++ b/tasks/reset_pgrepack_schema.sh @@ -1,16 +1,20 @@ #!/bin/sh # Puppet Task Name: reset_pgrepack_schema +declare PT__installdir +# shellcheck disable=SC1090 +source "$PT__installdir/pe_databases/files/common.sh" #Determine if PE Postgres is available if puppet resource service pe-postgresql | grep -q running; then #Remove the pg_repack extension - su - pe-postgres -s '/bin/bash' -c '/opt/puppetlabs/server/bin/psql -d pe-puppetdb -c "DROP EXTENSION pg_repack CASCADE"' + su - pe-postgres -s '/bin/bash' -c '/opt/puppetlabs/server/bin/psql -d pe-puppetdb -c "DROP EXTENSION pg_repack CASCADE"' || fail "unable to drop pg_repack extension" #Then, recreate the pg_repack extension - su - pe-postgres -s '/bin/bash' -c '/opt/puppetlabs/server/bin/psql -d pe-puppetdb -c "CREATE EXTENSION pg_repack"' + su - pe-postgres -s '/bin/bash' -c '/opt/puppetlabs/server/bin/psql -d pe-puppetdb -c "CREATE EXTENSION pg_repack"' || fail "unable to create pg_repack estenstion" else - echo " -- No running PE-PostgreSQL instance found, please run this task against your Primary or PE-PostgreSQL server --" + fail "No running PE-PostgreSQL instance found, please run this task against your Primary or PE-PostgreSQL server" fi -echo " -- reset_pgrepack_scheme Task ended: $(date +%s) --" + +success '{ "status": "success" }'