Open
Description
Describe the Bug
pg_repack aborts and kill other pids?
Expected Behavior
pg_repack should terminate without killing other queries
Steps to Reproduce
I'm not sure how to reproduce. maybe with a primary and too slow disks.
Environment
- PE 2019.8.9
- one Primary with three compilers
Additional Context
pe_databases-catalogs
[root@primary ~]# journalctl --unit pe_databases-catalogs --no-pager
-- Logs begin at Mon 2022-03-21 08:58:06 CET, end at Wed 2022-03-23 16:07:58 CET. --
Mar 23 15:48:53 primary systemd[1]: Starting Service to repack PE database tables...
Mar 23 15:48:53 primary pg_repack[21463]: NOTICE: Setting up workers.conns
Mar 23 15:48:53 primary pg_repack[21463]: INFO: repacking table "public.catalog_inputs"
Mar 23 15:48:53 primary pg_repack[21463]: LOG: Initial worker 0 to build index: CREATE UNIQUE INDEX index_124998945 ON repack.table_17744 USING btree (type, name, certname_id) TABLESPACE "pe-puppetdb"
Mar 23 15:48:53 primary pg_repack[21463]: LOG: Initial worker 1 to build index: CREATE INDEX index_17755 ON repack.table_17744 USING btree (certname_id) TABLESPACE "pe-puppetdb"
Mar 23 15:48:53 primary pg_repack[21463]: LOG: Command finished in worker 0: CREATE UNIQUE INDEX index_124998945 ON repack.table_17744 USING btree (type, name, certname_id) TABLESPACE "pe-puppetdb"
Mar 23 15:48:53 primary pg_repack[21463]: LOG: Command finished in worker 1: CREATE INDEX index_17755 ON repack.table_17744 USING btree (certname_id) TABLESPACE "pe-puppetdb"
Mar 23 15:48:53 primary pg_repack[21463]: NOTICE: Waiting for 5 transactions to finish. First PID: 20238
Mar 23 15:48:54 primary pg_repack[21463]: NOTICE: Waiting for 4 transactions to finish. First PID: 20238
Mar 23 15:48:55 primary pg_repack[21463]: NOTICE: Waiting for 4 transactions to finish. First PID: 20238
Mar 23 15:48:56 primary pg_repack[21463]: NOTICE: Waiting for 4 transactions to finish. First PID: 20238
Mar 23 15:48:57 primary pg_repack[21463]: NOTICE: Waiting for 3 transactions to finish. First PID: 20238
Mar 23 15:48:58 primary pg_repack[21463]: NOTICE: Waiting for 3 transactions to finish. First PID: 20238
Mar 23 15:48:59 primary pg_repack[21463]: NOTICE: Waiting for 3 transactions to finish. First PID: 20238
Mar 23 15:49:00 primary pg_repack[21463]: NOTICE: Waiting for 3 transactions to finish. First PID: 20238
Mar 23 15:49:01 primary pg_repack[21463]: NOTICE: Waiting for 3 transactions to finish. First PID: 20238
Mar 23 15:49:02 primary pg_repack[21463]: NOTICE: Waiting for 3 transactions to finish. First PID: 20238
Mar 23 15:49:03 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:04 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:05 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:06 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:07 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:08 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:09 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:10 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:11 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:12 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:13 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:14 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:15 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:16 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:17 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:18 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:19 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:20 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:21 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:22 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:23 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:24 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 625
Mar 23 15:49:25 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 5745
Mar 23 15:49:26 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 5745
Mar 23 15:49:27 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 5745
Mar 23 15:49:28 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 5745
Mar 23 15:49:29 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 5745
Mar 23 15:49:30 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 5745
Mar 23 15:49:31 primary pg_repack[21463]: INFO: repacking table "public.catalog_resources"
Mar 23 15:50:32 primary pg_repack[21463]: WARNING: canceling conflicted backends
Mar 23 15:50:43 primary pg_repack[21463]: LOG: Initial worker 0 to build index: CREATE INDEX index_17510 ON repack.table_17499 USING btree (type, title) TABLESPACE "pe-puppetdb"
Mar 23 15:50:43 primary pg_repack[21463]: LOG: Initial worker 1 to build index: CREATE INDEX index_17507 ON repack.table_17499 USING btree (exported) TABLESPACE "pe-puppetdb" WHERE (exported = true)
Mar 23 15:50:43 primary pg_repack[21463]: LOG: Initial worker 2 to build index: CREATE INDEX index_17508 ON repack.table_17499 USING btree (type) TABLESPACE "pe-puppetdb"
Mar 23 15:50:43 primary pg_repack[21463]: LOG: Initial worker 3 to build index: CREATE INDEX index_17509 ON repack.table_17499 USING btree (resource) TABLESPACE "pe-puppetdb"
Mar 23 15:50:43 primary pg_repack[21463]: LOG: Command finished in worker 1: CREATE INDEX index_17507 ON repack.table_17499 USING btree (exported) TABLESPACE "pe-puppetdb" WHERE (exported = true)
Mar 23 15:50:43 primary pg_repack[21463]: LOG: Assigning worker 1 to build index #4: CREATE INDEX index_17511 ON repack.table_17499 USING btree (encode(resource, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:50:44 primary pg_repack[21463]: LOG: Command finished in worker 3: CREATE INDEX index_17509 ON repack.table_17499 USING btree (resource) TABLESPACE "pe-puppetdb"
Mar 23 15:50:44 primary pg_repack[21463]: LOG: Assigning worker 3 to build index #5: CREATE UNIQUE INDEX index_17534 ON repack.table_17499 USING btree (certname_id, type, title) TABLESPACE "pe-puppetdb"
Mar 23 15:50:45 primary pg_repack[21463]: LOG: Command finished in worker 2: CREATE INDEX index_17508 ON repack.table_17499 USING btree (type) TABLESPACE "pe-puppetdb"
Mar 23 15:50:45 primary pg_repack[21463]: LOG: Assigning worker 2 to build index #6: CREATE INDEX index_18303 ON repack.table_17499 USING gin (file gin_trgm_ops) TABLESPACE "pe-puppetdb" WHERE (file IS NOT NULL)
Mar 23 15:50:47 primary pg_repack[21463]: LOG: Command finished in worker 1: CREATE INDEX index_17511 ON repack.table_17499 USING btree (encode(resource, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:50:53 primary pg_repack[21463]: LOG: Command finished in worker 0: CREATE INDEX index_17510 ON repack.table_17499 USING btree (type, title) TABLESPACE "pe-puppetdb"
Mar 23 15:50:53 primary pg_repack[21463]: LOG: Command finished in worker 3: CREATE UNIQUE INDEX index_17534 ON repack.table_17499 USING btree (certname_id, type, title) TABLESPACE "pe-puppetdb"
Mar 23 15:51:22 primary pg_repack[21463]: LOG: Command finished in worker 2: CREATE INDEX index_18303 ON repack.table_17499 USING gin (file gin_trgm_ops) TABLESPACE "pe-puppetdb" WHERE (file IS NOT NULL)
Mar 23 15:51:23 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:24 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:25 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:26 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:27 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:28 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:29 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:30 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:31 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:32 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:33 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:34 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:35 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:36 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:37 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:38 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:39 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:40 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:41 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:42 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:43 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:44 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:45 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:46 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:47 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:48 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:49 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:50 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:51 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:52 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:53 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:54 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:55 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:56 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:57 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:58 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:51:59 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:00 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:01 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:02 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:03 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:04 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:05 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:06 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:07 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:08 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:09 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:10 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:11 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:12 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:13 primary pg_repack[21463]: NOTICE: Waiting for 2 transactions to finish. First PID: 20238
Mar 23 15:52:17 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:18 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:19 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:20 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:21 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:22 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:23 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:24 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:25 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:26 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:27 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:28 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:29 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:30 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:31 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:32 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:33 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:34 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:35 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:36 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:37 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:52:38 primary pg_repack[21463]: NOTICE: Waiting for 1 transactions to finish. First PID: 20238
Mar 23 15:53:00 primary pg_repack[21463]: INFO: repacking table "public.catalogs"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Initial worker 0 to build index: CREATE INDEX index_17303 ON repack.table_17271 USING btree (producer_timestamp) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Initial worker 1 to build index: CREATE INDEX index_17304 ON repack.table_17271 USING btree (environment_id) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Initial worker 2 to build index: CREATE UNIQUE INDEX index_17309 ON repack.table_17271 USING btree (id) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Initial worker 3 to build index: CREATE INDEX index_17491 ON repack.table_17271 USING btree (((transaction_uuid)::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Command finished in worker 2: CREATE UNIQUE INDEX index_17309 ON repack.table_17271 USING btree (id) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Assigning worker 2 to build index #4: CREATE INDEX index_17542 ON repack.table_17271 USING btree (encode(hash, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Command finished in worker 0: CREATE INDEX index_17303 ON repack.table_17271 USING btree (producer_timestamp) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Assigning worker 0 to build index #5: CREATE INDEX index_17543 ON repack.table_17271 USING btree (certname) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Command finished in worker 1: CREATE INDEX index_17304 ON repack.table_17271 USING btree (environment_id) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Assigning worker 1 to build index #6: CREATE INDEX index_17578 ON repack.table_17271 USING btree (producer_id) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Command finished in worker 3: CREATE INDEX index_17491 ON repack.table_17271 USING btree (((transaction_uuid)::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Assigning worker 3 to build index #7: CREATE INDEX index_17683 ON repack.table_17271 USING btree (job_id) TABLESPACE "pe-puppetdb" WHERE (job_id IS NOT NULL)
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Command finished in worker 2: CREATE INDEX index_17542 ON repack.table_17271 USING btree (encode(hash, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Command finished in worker 1: CREATE INDEX index_17578 ON repack.table_17271 USING btree (producer_id) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Command finished in worker 3: CREATE INDEX index_17683 ON repack.table_17271 USING btree (job_id) TABLESPACE "pe-puppetdb" WHERE (job_id IS NOT NULL)
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Command finished in worker 0: CREATE INDEX index_17543 ON repack.table_17271 USING btree (certname) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: INFO: repacking table "public.certnames"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Initial worker 0 to build index: CREATE INDEX index_17679 ON repack.table_17432 USING btree (latest_report_timestamp) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Initial worker 1 to build index: CREATE UNIQUE INDEX index_17618 ON repack.table_17432 USING btree (latest_report_id) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Initial worker 2 to build index: CREATE UNIQUE INDEX index_17441 ON repack.table_17432 USING btree (certname) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Initial worker 3 to build index: CREATE UNIQUE INDEX index_17439 ON repack.table_17432 USING btree (id) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Command finished in worker 3: CREATE UNIQUE INDEX index_17439 ON repack.table_17432 USING btree (id) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Command finished in worker 1: CREATE UNIQUE INDEX index_17618 ON repack.table_17432 USING btree (latest_report_id) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Command finished in worker 0: CREATE INDEX index_17679 ON repack.table_17432 USING btree (latest_report_timestamp) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: LOG: Command finished in worker 2: CREATE UNIQUE INDEX index_17441 ON repack.table_17432 USING btree (certname) TABLESPACE "pe-puppetdb"
Mar 23 15:53:00 primary pg_repack[21463]: INFO: repacking table "public.edges"
Mar 23 15:53:10 primary systemd[1]: Started Service to repack PE database tables.
pe_databases-facts
[root@primary ~]# journalctl --unit pe_databases-facts --no-pager
-- Logs begin at Mon 2022-03-21 08:58:06 CET, end at Wed 2022-03-23 16:10:00 CET. --
Mar 23 15:59:14 primary systemd[1]: Starting Service to repack PE database tables...
Mar 23 15:59:14 primary pg_repack[32716]: NOTICE: Setting up workers.conns
Mar 23 15:59:14 primary pg_repack[32716]: INFO: repacking table "public.fact_paths"
Mar 23 15:59:14 primary pg_repack[32716]: LOG: Initial worker 0 to build index: CREATE INDEX index_18301 ON repack.table_16994 USING gist (path gist_trgm_ops) TABLESPACE "pe-puppetdb"
Mar 23 15:59:14 primary pg_repack[32716]: LOG: Initial worker 1 to build index: CREATE UNIQUE INDEX index_17729 ON repack.table_16994 USING btree (path, value_type_id) TABLESPACE "pe-puppetdb"
Mar 23 15:59:14 primary pg_repack[32716]: LOG: Initial worker 2 to build index: CREATE INDEX index_17114 ON repack.table_16994 USING btree (name) TABLESPACE "pe-puppetdb"
Mar 23 15:59:14 primary pg_repack[32716]: LOG: Initial worker 3 to build index: CREATE UNIQUE INDEX index_17088 ON repack.table_16994 USING btree (id) TABLESPACE "pe-puppetdb"
Mar 23 15:59:15 primary pg_repack[32716]: LOG: Command finished in worker 3: CREATE UNIQUE INDEX index_17088 ON repack.table_16994 USING btree (id) TABLESPACE "pe-puppetdb"
Mar 23 15:59:15 primary pg_repack[32716]: LOG: Command finished in worker 2: CREATE INDEX index_17114 ON repack.table_16994 USING btree (name) TABLESPACE "pe-puppetdb"
Mar 23 15:59:15 primary pg_repack[32716]: LOG: Command finished in worker 1: CREATE UNIQUE INDEX index_17729 ON repack.table_16994 USING btree (path, value_type_id) TABLESPACE "pe-puppetdb"
Mar 23 15:59:16 primary pg_repack[32716]: LOG: Command finished in worker 0: CREATE INDEX index_18301 ON repack.table_16994 USING gist (path gist_trgm_ops) TABLESPACE "pe-puppetdb"
Mar 23 15:59:16 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:17 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:18 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:19 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:20 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:21 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:22 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:23 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:24 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:25 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:26 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:27 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:28 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:29 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:30 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:31 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:32 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:33 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:34 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:35 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:36 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:37 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:38 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:39 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:40 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:41 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:42 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:43 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:44 primary pg_repack[32716]: NOTICE: Waiting for 3 transactions to finish. First PID: 27887
Mar 23 15:59:45 primary pg_repack[32716]: NOTICE: Waiting for 2 transactions to finish. First PID: 27887
Mar 23 15:59:46 primary pg_repack[32716]: NOTICE: Waiting for 2 transactions to finish. First PID: 27887
Mar 23 15:59:47 primary pg_repack[32716]: NOTICE: Waiting for 2 transactions to finish. First PID: 27887
Mar 23 15:59:48 primary pg_repack[32716]: NOTICE: Waiting for 2 transactions to finish. First PID: 27887
Mar 23 15:59:49 primary pg_repack[32716]: NOTICE: Waiting for 2 transactions to finish. First PID: 27887
Mar 23 15:59:50 primary pg_repack[32716]: NOTICE: Waiting for 2 transactions to finish. First PID: 27887
Mar 23 15:59:51 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 15:59:52 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 15:59:53 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 15:59:54 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 15:59:55 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 15:59:56 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 15:59:57 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 15:59:58 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 15:59:59 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:00 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:01 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:02 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:03 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:04 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:05 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:06 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:07 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:08 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:09 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:10 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:11 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:12 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:13 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:14 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:15 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:16 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:17 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:18 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:19 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:20 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:21 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:22 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:23 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:24 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:25 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:26 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:27 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:28 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:29 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:30 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:31 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:32 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:33 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:34 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:35 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:36 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:37 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:38 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:39 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:40 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:41 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:42 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:43 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:44 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:45 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:46 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:47 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:48 primary pg_repack[32716]: NOTICE: Waiting for 1 transactions to finish. First PID: 7431
Mar 23 16:00:50 primary pg_repack[32716]: INFO: repacking table "public.factsets"
Mar 23 16:01:51 primary pg_repack[32716]: WARNING: canceling conflicted backends
Mar 23 16:01:52 primary pg_repack[32716]: LOG: Initial worker 0 to build index: CREATE INDEX index_17728 ON repack.table_17231 USING gin (((stable || volatile)) jsonb_path_ops) TABLESPACE "pe-puppetdb"
Mar 23 16:01:52 primary pg_repack[32716]: LOG: Initial worker 1 to build index: CREATE INDEX index_17577 ON repack.table_17231 USING btree (producer_id) TABLESPACE "pe-puppetdb"
Mar 23 16:01:52 primary pg_repack[32716]: LOG: Initial worker 2 to build index: CREATE UNIQUE INDEX index_17498 ON repack.table_17231 USING btree (encode(hash, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 16:01:52 primary pg_repack[32716]: LOG: Initial worker 3 to build index: CREATE UNIQUE INDEX index_17390 ON repack.table_17231 USING btree (certname) TABLESPACE "pe-puppetdb"
Mar 23 16:01:52 primary pg_repack[32716]: LOG: Command finished in worker 1: CREATE INDEX index_17577 ON repack.table_17231 USING btree (producer_id) TABLESPACE "pe-puppetdb"
Mar 23 16:01:52 primary pg_repack[32716]: LOG: Assigning worker 1 to build index #4: CREATE UNIQUE INDEX index_17378 ON repack.table_17231 USING btree (id) TABLESPACE "pe-puppetdb"
Mar 23 16:01:52 primary pg_repack[32716]: LOG: Command finished in worker 2: CREATE UNIQUE INDEX index_17498 ON repack.table_17231 USING btree (encode(hash, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 16:01:52 primary pg_repack[32716]: LOG: Command finished in worker 3: CREATE UNIQUE INDEX index_17390 ON repack.table_17231 USING btree (certname) TABLESPACE "pe-puppetdb"
Mar 23 16:01:52 primary pg_repack[32716]: LOG: Command finished in worker 1: CREATE UNIQUE INDEX index_17378 ON repack.table_17231 USING btree (id) TABLESPACE "pe-puppetdb"
Mar 23 16:01:53 primary pg_repack[32716]: LOG: Command finished in worker 0: CREATE INDEX index_17728 ON repack.table_17231 USING gin (((stable || volatile)) jsonb_path_ops) TABLESPACE "pe-puppetdb"
Mar 23 16:01:54 primary systemd[1]: Started Service to repack PE database tables.
pe_databases-other
[root@primary ~]# journalctl --unit pe_databases-other --no-pager
-- Logs begin at Mon 2022-03-21 08:58:06 CET, end at Wed 2022-03-23 16:11:16 CET. --
Mar 23 15:10:49 primary systemd[1]: Starting Service to repack PE database tables...
Mar 23 15:10:49 primary pg_repack[23170]: NOTICE: Setting up workers.conns
Mar 23 15:10:50 primary pg_repack[23170]: INFO: repacking table "public.producers"
Mar 23 15:10:50 primary pg_repack[23170]: LOG: Initial worker 0 to build index: CREATE UNIQUE INDEX index_17556 ON repack.table_17548 USING btree (name) TABLESPACE "pe-puppetdb"
Mar 23 15:10:50 primary pg_repack[23170]: LOG: Initial worker 1 to build index: CREATE UNIQUE INDEX index_17554 ON repack.table_17548 USING btree (id) TABLESPACE "pe-puppetdb"
Mar 23 15:10:50 primary pg_repack[23170]: LOG: Command finished in worker 1: CREATE UNIQUE INDEX index_17554 ON repack.table_17548 USING btree (id) TABLESPACE "pe-puppetdb"
Mar 23 15:10:50 primary pg_repack[23170]: LOG: Command finished in worker 0: CREATE UNIQUE INDEX index_17556 ON repack.table_17548 USING btree (name) TABLESPACE "pe-puppetdb"
Mar 23 15:10:50 primary pg_repack[23170]: NOTICE: Waiting for 1 transactions to finish. First PID: 22717
Mar 23 15:10:51 primary pg_repack[23170]: INFO: repacking table "public.resource_params"
Mar 23 15:11:18 primary pg_repack[23170]: LOG: Initial worker 0 to build index: CREATE INDEX index_17496 ON repack.table_17257 USING btree (encode(resource, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:11:18 primary pg_repack[23170]: LOG: Initial worker 1 to build index: CREATE INDEX index_17324 ON repack.table_17257 USING btree (name) TABLESPACE "pe-puppetdb"
Mar 23 15:11:18 primary pg_repack[23170]: LOG: Initial worker 2 to build index: CREATE INDEX index_17323 ON repack.table_17257 USING btree (resource) TABLESPACE "pe-puppetdb"
Mar 23 15:11:18 primary pg_repack[23170]: LOG: Initial worker 3 to build index: CREATE UNIQUE INDEX index_17321 ON repack.table_17257 USING btree (resource, name) TABLESPACE "pe-puppetdb"
Mar 23 15:11:31 primary pg_repack[23170]: LOG: Command finished in worker 2: CREATE INDEX index_17323 ON repack.table_17257 USING btree (resource) TABLESPACE "pe-puppetdb"
Mar 23 15:11:34 primary pg_repack[23170]: LOG: Command finished in worker 1: CREATE INDEX index_17324 ON repack.table_17257 USING btree (name) TABLESPACE "pe-puppetdb"
Mar 23 15:11:39 primary pg_repack[23170]: LOG: Command finished in worker 3: CREATE UNIQUE INDEX index_17321 ON repack.table_17257 USING btree (resource, name) TABLESPACE "pe-puppetdb"
Mar 23 15:11:45 primary pg_repack[23170]: LOG: Command finished in worker 0: CREATE INDEX index_17496 ON repack.table_17257 USING btree (encode(resource, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:12:46 primary pg_repack[23170]: WARNING: canceling conflicted backends
Mar 23 15:12:47 primary pg_repack[23170]: WARNING: canceling conflicted backends
Mar 23 15:12:48 primary pg_repack[23170]: WARNING: canceling conflicted backends
Mar 23 15:14:03 primary pg_repack[23170]: WARNING: canceling conflicted backends
Mar 23 15:14:04 primary pg_repack[23170]: WARNING: canceling conflicted backends
Mar 23 15:14:05 primary pg_repack[23170]: WARNING: canceling conflicted backends
Mar 23 15:14:06 primary pg_repack[23170]: WARNING: canceling conflicted backends
Mar 23 15:14:07 primary pg_repack[23170]: INFO: repacking table "public.resource_params_cache"
Mar 23 15:14:43 primary pg_repack[23170]: LOG: Initial worker 0 to build index: CREATE INDEX index_17664 ON repack.table_17598 USING gin (parameters) TABLESPACE "pe-puppetdb"
Mar 23 15:14:43 primary pg_repack[23170]: LOG: Initial worker 1 to build index: CREATE INDEX index_17616 ON repack.table_17598 USING btree (encode(resource, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:14:43 primary pg_repack[23170]: LOG: Initial worker 2 to build index: CREATE UNIQUE INDEX index_17604 ON repack.table_17598 USING btree (resource) TABLESPACE "pe-puppetdb"
Mar 23 15:14:44 primary pg_repack[23170]: LOG: Command finished in worker 2: CREATE UNIQUE INDEX index_17604 ON repack.table_17598 USING btree (resource) TABLESPACE "pe-puppetdb"
Mar 23 15:14:47 primary pg_repack[23170]: LOG: Command finished in worker 1: CREATE INDEX index_17616 ON repack.table_17598 USING btree (encode(resource, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:15:00 primary pg_repack[23170]: LOG: Command finished in worker 0: CREATE INDEX index_17664 ON repack.table_17598 USING gin (parameters) TABLESPACE "pe-puppetdb"
Mar 23 15:16:01 primary pg_repack[23170]: WARNING: canceling conflicted backends
Mar 23 15:16:02 primary pg_repack[23170]: WARNING: canceling conflicted backends
Mar 23 15:16:03 primary pg_repack[23170]: WARNING: canceling conflicted backends
Mar 23 15:16:04 primary pg_repack[23170]: WARNING: canceling conflicted backends
Mar 23 15:16:06 primary systemd[1]: Started Service to repack PE database tables.
Mar 23 15:30:42 primary systemd[1]: Starting Service to repack PE database tables...
Mar 23 15:30:42 primary pg_repack[17145]: NOTICE: Setting up workers.conns
Mar 23 15:30:42 primary pg_repack[17145]: INFO: repacking table "public.producers"
Mar 23 15:31:04 primary pg_repack[17145]: LOG: Initial worker 0 to build index: CREATE UNIQUE INDEX index_17556 ON repack.table_17548 USING btree (name) TABLESPACE "pe-puppetdb"
Mar 23 15:31:04 primary pg_repack[17145]: LOG: Initial worker 1 to build index: CREATE UNIQUE INDEX index_17554 ON repack.table_17548 USING btree (id) TABLESPACE "pe-puppetdb"
Mar 23 15:31:04 primary pg_repack[17145]: LOG: Command finished in worker 1: CREATE UNIQUE INDEX index_17554 ON repack.table_17548 USING btree (id) TABLESPACE "pe-puppetdb"
Mar 23 15:31:04 primary pg_repack[17145]: LOG: Command finished in worker 0: CREATE UNIQUE INDEX index_17556 ON repack.table_17548 USING btree (name) TABLESPACE "pe-puppetdb"
Mar 23 15:31:04 primary pg_repack[17145]: INFO: repacking table "public.resource_params"
Mar 23 15:32:04 primary pg_repack[17145]: LOG: Initial worker 0 to build index: CREATE INDEX index_17496 ON repack.table_17257 USING btree (encode(resource, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:32:04 primary pg_repack[17145]: LOG: Initial worker 1 to build index: CREATE INDEX index_17324 ON repack.table_17257 USING btree (name) TABLESPACE "pe-puppetdb"
Mar 23 15:32:04 primary pg_repack[17145]: LOG: Initial worker 2 to build index: CREATE INDEX index_17323 ON repack.table_17257 USING btree (resource) TABLESPACE "pe-puppetdb"
Mar 23 15:32:04 primary pg_repack[17145]: LOG: Initial worker 3 to build index: CREATE UNIQUE INDEX index_17321 ON repack.table_17257 USING btree (resource, name) TABLESPACE "pe-puppetdb"
Mar 23 15:32:09 primary pg_repack[17145]: LOG: Command finished in worker 2: CREATE INDEX index_17323 ON repack.table_17257 USING btree (resource) TABLESPACE "pe-puppetdb"
Mar 23 15:32:09 primary pg_repack[17145]: LOG: Command finished in worker 1: CREATE INDEX index_17324 ON repack.table_17257 USING btree (name) TABLESPACE "pe-puppetdb"
Mar 23 15:32:11 primary pg_repack[17145]: LOG: Command finished in worker 3: CREATE UNIQUE INDEX index_17321 ON repack.table_17257 USING btree (resource, name) TABLESPACE "pe-puppetdb"
Mar 23 15:32:15 primary pg_repack[17145]: LOG: Command finished in worker 0: CREATE INDEX index_17496 ON repack.table_17257 USING btree (encode(resource, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:32:17 primary pg_repack[17145]: INFO: repacking table "public.resource_params_cache"
Mar 23 15:33:18 primary pg_repack[17145]: WARNING: canceling conflicted backends
Mar 23 15:33:19 primary pg_repack[17145]: WARNING: canceling conflicted backends
Mar 23 15:33:26 primary pg_repack[17145]: LOG: Initial worker 0 to build index: CREATE INDEX index_17664 ON repack.table_17598 USING gin (parameters) TABLESPACE "pe-puppetdb"
Mar 23 15:33:26 primary pg_repack[17145]: LOG: Initial worker 1 to build index: CREATE INDEX index_17616 ON repack.table_17598 USING btree (encode(resource, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:33:26 primary pg_repack[17145]: LOG: Initial worker 2 to build index: CREATE UNIQUE INDEX index_17604 ON repack.table_17598 USING btree (resource) TABLESPACE "pe-puppetdb"
Mar 23 15:33:27 primary pg_repack[17145]: LOG: Command finished in worker 2: CREATE UNIQUE INDEX index_17604 ON repack.table_17598 USING btree (resource) TABLESPACE "pe-puppetdb"
Mar 23 15:33:28 primary pg_repack[17145]: LOG: Command finished in worker 1: CREATE INDEX index_17616 ON repack.table_17598 USING btree (encode(resource, 'hex'::text)) TABLESPACE "pe-puppetdb"
Mar 23 15:33:36 primary pg_repack[17145]: LOG: Command finished in worker 0: CREATE INDEX index_17664 ON repack.table_17598 USING gin (parameters) TABLESPACE "pe-puppetdb"
Mar 23 15:33:36 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:37 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:38 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:39 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:40 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:41 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:42 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:43 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:44 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:45 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:46 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:47 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:48 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:49 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:50 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:51 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:53 primary pg_repack[17145]: NOTICE: Waiting for 2 transactions to finish. First PID: 27537
Mar 23 15:33:54 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:33:55 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:33:56 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:33:57 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:33:58 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:33:59 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:00 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:01 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:02 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:03 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:05 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:06 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:07 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:08 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:09 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:10 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:11 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:12 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:13 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:14 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:15 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:16 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:17 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:18 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:19 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:20 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:21 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:22 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:23 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:24 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:25 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:26 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:27 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:28 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:29 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:30 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:31 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:32 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:33 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:34 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:35 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:36 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:37 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:38 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:39 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:41 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:42 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:43 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:44 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:45 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:46 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:47 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:48 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:49 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:50 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:51 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:52 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:53 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:54 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:55 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:56 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:57 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:58 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:34:59 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:35:00 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:35:01 primary pg_repack[17145]: NOTICE: Waiting for 1 transactions to finish. First PID: 8383
Mar 23 15:36:04 primary pg_repack[17145]: WARNING: canceling conflicted backends
Mar 23 15:36:05 primary pg_repack[17145]: WARNING: canceling conflicted backends
Mar 23 15:36:06 primary pg_repack[17145]: WARNING: canceling conflicted backends
Mar 23 15:36:53 primary systemd[1]: Started Service to repack PE database tables.
log from one of the puppetdbs
2022-03-23T15:50:32.041+01:00 ERROR [p.p.middleware] #error {
:cause ERROR: canceling statement due to user request
:via
[{:type org.postgresql.util.PSQLException
:message ERROR: canceling statement due to user request
:at [org.postgresql.core.v3.QueryExecutorImpl receiveErrorResponse QueryExecutorImpl.java 2532]}]
:trace
[[org.postgresql.core.v3.QueryExecutorImpl receiveErrorResponse QueryExecutorImpl.java 2532]
[org.postgresql.core.v3.QueryExecutorImpl processResults QueryExecutorImpl.java 2267]
[org.postgresql.core.v3.QueryExecutorImpl execute QueryExecutorImpl.java 312]
[org.postgresql.jdbc.PgStatement executeInternal PgStatement.java 448]
[org.postgresql.jdbc.PgStatement execute PgStatement.java 369]
[org.postgresql.jdbc.PgPreparedStatement executeWithFlags PgPreparedStatement.java 153]
[org.postgresql.jdbc.PgPreparedStatement executeQuery PgPreparedStatement.java 103]
[com.zaxxer.hikari.pool.ProxyPreparedStatement executeQuery ProxyPreparedStatement.java 52]
[com.zaxxer.hikari.pool.HikariProxyPreparedStatement executeQuery HikariProxyPreparedStatement.java -1]
[puppetlabs.puppetdb.jdbc$call_with_array_converted_query_rows$fn__28273 invoke jdbc.clj 315]
[clojure.java.jdbc$db_transaction_STAR_ invokeStatic jdbc.clj 860]
[clojure.java.jdbc$db_transaction_STAR_ invoke jdbc.clj 776]
[puppetlabs.puppetdb.jdbc$call_with_array_converted_query_rows invokeStatic jdbc.clj 307]
[puppetlabs.puppetdb.jdbc$call_with_array_converted_query_rows invoke jdbc.clj 290]
[puppetlabs.puppetdb.jdbc$call_with_array_converted_query_rows invokeStatic jdbc.clj 303]
[puppetlabs.puppetdb.jdbc$call_with_array_converted_query_rows invoke jdbc.clj 290]
[puppetlabs.puppetdb.query_eng$deprecated_produce_streaming_body$fn__35401$fn__35403$fn__35410 invoke query_eng.clj 450]
[puppetlabs.puppetdb.jdbc$with_transacted_connection_fn$fn__28406$fn__28409 invoke jdbc.clj 528]
[clojure.java.jdbc$db_transaction_STAR_ invokeStatic jdbc.clj 807]
[clojure.java.jdbc$db_transaction_STAR_ invoke jdbc.clj 776]
[puppetlabs.puppetdb.jdbc$with_transacted_connection_fn$fn__28406 invoke jdbc.clj 527]
[puppetlabs.puppetdb.jdbc$retry_sql$attempt__28400 invoke jdbc.clj 486]
[puppetlabs.puppetdb.jdbc$retry_sql invokeStatic jdbc.clj 496]
[puppetlabs.puppetdb.jdbc$retry_sql invoke jdbc.clj 477]
[puppetlabs.puppetdb.jdbc$with_transacted_connection_fn invokeStatic jdbc.clj 526]
[puppetlabs.puppetdb.jdbc$with_transacted_connection_fn invoke jdbc.clj 518]
[puppetlabs.puppetdb.query_eng$deprecated_produce_streaming_body$fn__35401$fn__35403 invoke query_eng.clj 449]
[[ring.util.io](http://ring.util.io/)$piped_input_stream$fn__212 invoke io.clj 28]
[clojure.core$binding_conveyor_fn$fn__5754 invoke core.clj 2030]
[clojure.lang.AFn call AFn.java 18]
[java.util.concurrent.FutureTask run FutureTask.java 264]
[java.util.concurrent.ThreadPoolExecutor runWorker ThreadPoolExecutor.java 1128]
[java.util.concurrent.ThreadPoolExecutor$Worker run ThreadPoolExecutor.java 628]
[java.lang.Thread run Thread.java 829]]}
2022-03-23T15:50:32.041+01:00 ERROR [p.p.middleware] #error {
:cause ERROR: canceling statement due to user request
:via
[{:type org.postgresql.util.PSQLException
:message ERROR: canceling statement due to user request
:at [org.postgresql.core.v3.QueryExecutorImpl receiveErrorResponse QueryExecutorImpl.java 2532]}]
:trace
[[org.postgresql.core.v3.QueryExecutorImpl receiveErrorResponse QueryExecutorImpl.java 2532]
[org.postgresql.core.v3.QueryExecutorImpl processResults QueryExecutorImpl.java 2267]
[org.postgresql.core.v3.QueryExecutorImpl execute QueryExecutorImpl.java 312]
[org.postgresql.jdbc.PgStatement executeInternal PgStatement.java 448]
[org.postgresql.jdbc.PgStatement execute PgStatement.java 369]
[org.postgresql.jdbc.PgPreparedStatement executeWithFlags PgPreparedStatement.java 153]
[org.postgresql.jdbc.PgPreparedStatement executeQuery PgPreparedStatement.java 103]
[com.zaxxer.hikari.pool.ProxyPreparedStatement executeQuery ProxyPreparedStatement.java 52]
[com.zaxxer.hikari.pool.HikariProxyPreparedStatement executeQuery HikariProxyPreparedStatement.java -1]
[puppetlabs.puppetdb.jdbc$call_with_array_converted_query_rows$fn__28273 invoke jdbc.clj 315]
[clojure.java.jdbc$db_transaction_STAR_ invokeStatic jdbc.clj 860]
[clojure.java.jdbc$db_transaction_STAR_ invoke jdbc.clj 776]
[puppetlabs.puppetdb.jdbc$call_with_array_converted_query_rows invokeStatic jdbc.clj 307]
[puppetlabs.puppetdb.jdbc$call_with_array_converted_query_rows invoke jdbc.clj 290]
[puppetlabs.puppetdb.jdbc$call_with_array_converted_query_rows invokeStatic jdbc.clj 303]
[puppetlabs.puppetdb.jdbc$call_with_array_converted_query_rows invoke jdbc.clj 290]
[puppetlabs.puppetdb.query_eng$deprecated_produce_streaming_body$fn__35401$fn__35403$fn__35410 invoke query_eng.clj 450]
[puppetlabs.puppetdb.jdbc$with_transacted_connection_fn$fn__28406$fn__28409 invoke jdbc.clj 528]
[clojure.java.jdbc$db_transaction_STAR_ invokeStatic jdbc.clj 807]
[clojure.java.jdbc$db_transaction_STAR_ invoke jdbc.clj 776]
[puppetlabs.puppetdb.jdbc$with_transacted_connection_fn$fn__28406 invoke jdbc.clj 527]
[puppetlabs.puppetdb.jdbc$retry_sql$attempt__28400 invoke jdbc.clj 486]
[puppetlabs.puppetdb.jdbc$retry_sql invokeStatic jdbc.clj 496]
[puppetlabs.puppetdb.jdbc$retry_sql invoke jdbc.clj 477]
[puppetlabs.puppetdb.jdbc$with_transacted_connection_fn invokeStatic jdbc.clj 526]
[puppetlabs.puppetdb.jdbc$with_transacted_connection_fn invoke jdbc.clj 518]
[puppetlabs.puppetdb.query_eng$deprecated_produce_streaming_body$fn__35401$fn__35403 invoke query_eng.clj 449]
[[ring.util.io](http://ring.util.io/)$piped_input_stream$fn__212 invoke io.clj 28]
[clojure.core$binding_conveyor_fn$fn__5754 invoke core.clj 2030]
[clojure.lang.AFn call AFn.java 18]
[java.util.concurrent.FutureTask run FutureTask.java 264]
[java.util.concurrent.ThreadPoolExecutor runWorker ThreadPoolExecutor.java 1128]
[java.util.concurrent.ThreadPoolExecutor$Worker run ThreadPoolExecutor.java 628]
[java.lang.Thread run Thread.java 829]]}