From bfc7e4de2c04aa36c3ca566c3ba53efbed995738 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 4 Dec 2018 16:08:49 +0100 Subject: [PATCH 01/10] add mongodb+srv:// support --- mongodb_consistent_backup/Common/DB.py | 6 +++--- mongodb_consistent_backup/Common/MongoUri.py | 21 ++++++++++++++++++-- requirements.txt | 1 + 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 44c42326..8629ade1 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -3,7 +3,7 @@ from bson.codec_options import CodecOptions from inspect import currentframe, getframeinfo from pymongo import DESCENDING, CursorType, MongoClient -from pymongo.errors import ConnectionFailure, OperationFailure, ServerSelectionTimeoutError +from pymongo.errors import ConfigurationError, ConnectionFailure, OperationFailure, ServerSelectionTimeoutError from ssl import CERT_REQUIRED, CERT_NONE from time import sleep @@ -63,8 +63,8 @@ def do_ssl_insecure(self): def client_opts(self): opts = { "connect": self.do_connect, - "host": self.uri.hosts(), "connectTimeoutMS": self.conn_timeout, + "host": self.uri.hosts(), "serverSelectionTimeoutMS": self.conn_timeout, "maxPoolSize": 1, } @@ -113,7 +113,7 @@ def connect(self): conn = MongoClient(**self.client_opts()) if self.do_connect: conn['admin'].command({"ping": 1}) - except (ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e: + except (ConfigurationError, ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e: logging.error("Unable to connect to %s! Error: %s" % (self.uri, e)) raise DBConnectionError(e) if conn is not None: diff --git a/mongodb_consistent_backup/Common/MongoUri.py b/mongodb_consistent_backup/Common/MongoUri.py index c9b65a6f..1ae13757 100644 --- a/mongodb_consistent_backup/Common/MongoUri.py +++ b/mongodb_consistent_backup/Common/MongoUri.py @@ -1,4 +1,7 @@ +import re + from Util import validate_hostname +from mongodb_consistent_backup.Errors import Error, OperationError class MongoAddr: @@ -28,6 +31,8 @@ def __init__(self, url, default_port=27017, replset=None): self.parse() def hosts(self): + if self.srv: + return self.url if len(self.addrs) > 0: hosts = [] for addr in self.addrs: @@ -35,6 +40,8 @@ def hosts(self): return ",".join(hosts) def str(self): + if self.srv: + return self.url string = self.hosts() if self.replset: string = "%s/%s" % (self.replset, string) @@ -44,8 +51,18 @@ def __str__(self): return self.str() def parse(self): - if "/" in self.url: - self.replset, self.url = self.url.split("/") + # allow mongodb+srv:// URI + if self.url.startswith("mongodb+srv://"): + if "replicaSet=" not in self.url: + raise OperationError("replicaSet=X flag required when using mongodb+srv:// URI") + rsSearch = re.search('replicaSet=(\S+)(&.+)?$', self.url, re.IGNORECASE) + if not rsSearch: + raise OperationError("cannot find required replicaSet=X flag in mongodb+srv:// URI") + self.replset = rsSearch.group(1) + self.srv = True + return True + + self.replset, self.url = self.url.split("/") for url in self.url.split(","): addr = MongoAddr() addr.replset = self.replset diff --git a/requirements.txt b/requirements.txt index 0913ce40..14507168 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ yconf==0.3.4 google_compute_engine==2.7.2 progress==1.3 py-zabbix==1.1.3 +dnspython==1.15 From 8ef4b33ca09734bf9be7681f17cd4b447103cd76 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 4 Dec 2018 21:12:37 +0100 Subject: [PATCH 02/10] Cleanup --- Dockerfile | 2 +- .../Backup/Mongodump/Mongodump.py | 3 +++ .../Backup/Mongodump/MongodumpThread.py | 20 +++++++++++++++---- mongodb_consistent_backup/Common/DB.py | 2 +- mongodb_consistent_backup/Common/MongoUri.py | 2 +- mongodb_consistent_backup/Main.py | 12 +++++++++++ scripts/build.sh | 2 +- 7 files changed, 35 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2fbfcb18..d2ac0e41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM centos:centos7 MAINTAINER Tim Vaillancourt RUN yum install -y https://www.percona.com/redir/downloads/percona-release/redhat/latest/percona-release-0.1-6.noarch.rpm epel-release && \ - yum install -y Percona-Server-MongoDB-34-tools zbackup && yum clean all + yum install -y Percona-Server-MongoDB-36-tools zbackup && yum clean all ADD build/rpm/RPMS/x86_64/mongodb_consistent_backup*.el7.x86_64.rpm / RUN yum localinstall -y /mongodb_consistent_backup*.el7.x86_64.rpm && \ diff --git a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py index 61284e15..4e160bdf 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py +++ b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py @@ -1,5 +1,6 @@ import os import logging +import pprint from math import floor from subprocess import check_output @@ -124,6 +125,8 @@ def threads(self, threads=None): def run(self): self.timer.start(self.timer_name) + pprint.pprint(self.replsets) + # backup a secondary from each shard: for shard in self.replsets: secondary = self.replsets[shard].find_secondary() diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 29e0f76d..145554ac 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -138,12 +138,24 @@ def wait(self): def mongodump_cmd(self): mongodump_uri = self.uri.get() mongodump_cmd = [self.binary] - mongodump_flags = [ - "--host=%s" % mongodump_uri.host, - "--port=%s" % str(mongodump_uri.port), + mongodump_flags = [] + + # --host/--port (suport mongodb+srv:// too) + if self.uri.srv: + if not self.is_version_gte("3.6.0"): + logging.fatal("Mongodump must be >= 3.6.0 to use mongodb+srv:// URIs") + sys.exit(1) + mongodump_flags.append("--host=%s" % self.uri.url) + else: + mongodump_flags.extend([ + "--host=%s" % mongodump_uri.host, + "--port=%s" % str(mongodump_uri.port) + ]) + + mongodump_flags.extend([ "--oplog", "--out=%s/dump" % self.backup_dir - ] + ]) # --numParallelCollections if self.threads > 0: diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 8629ade1..e8d2aa9c 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -63,8 +63,8 @@ def do_ssl_insecure(self): def client_opts(self): opts = { "connect": self.do_connect, + "host": self.uri.hosts(), "connectTimeoutMS": self.conn_timeout, - "host": self.uri.hosts(), "serverSelectionTimeoutMS": self.conn_timeout, "maxPoolSize": 1, } diff --git a/mongodb_consistent_backup/Common/MongoUri.py b/mongodb_consistent_backup/Common/MongoUri.py index 1ae13757..86193062 100644 --- a/mongodb_consistent_backup/Common/MongoUri.py +++ b/mongodb_consistent_backup/Common/MongoUri.py @@ -1,7 +1,7 @@ import re from Util import validate_hostname -from mongodb_consistent_backup.Errors import Error, OperationError +from mongodb_consistent_backup.Errors import OperationError class MongoAddr: diff --git a/mongodb_consistent_backup/Main.py b/mongodb_consistent_backup/Main.py index f84d23dd..cac562b6 100644 --- a/mongodb_consistent_backup/Main.py +++ b/mongodb_consistent_backup/Main.py @@ -3,6 +3,8 @@ import signal import sys +import pprint + from datetime import datetime from multiprocessing import current_process, Event, Manager @@ -263,7 +265,15 @@ def run(self): self.db ) replset_name = self.replset.get_rs_name() + + # REMOVEME + logging.info("replset name: %s" % replset_name) + replset_dir = os.path.join(self.backup_directory, replset_name) + + # REMOVEME + logging.info("replset dir: %s" % replset_dir) + self.replsets[replset_name] = self.replset state = StateBackupReplset(replset_dir, self.config, self.backup_time, replset_name) state.load_state(self.replset.summary()) @@ -271,6 +281,8 @@ def run(self): except Exception, e: self.exception("Problem getting shard secondaries! Error: %s" % e, e) + pprint.pprint(self.replsets) + # run backup try: self.backup = Backup( diff --git a/scripts/build.sh b/scripts/build.sh index 364cbec2..b67cafa2 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -91,7 +91,7 @@ if [ -d ${srcdir} ]; then pip_flags="--download-cache=${pipdir}" ${venvdir}/bin/python2.7 ${venvdir}/bin/pip --help | grep -q '\-\-cache\-dir' [ $? = 0 ] && pip_flags="--cache-dir=${pipdir}" - ${venvdir}/bin/python2.7 ${venvdir}/bin/pip install ${pip_flags} pex requests + ${venvdir}/bin/python2.7 ${venvdir}/bin/pip install ${pip_flags} pex==1.4 requests if [ $? -gt 0 ]; then echo "Failed to install pex utility for building!" exit 1 From cb93ba83e753cdee4122d9c2f07487809279aa78 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 5 Dec 2018 14:36:23 +0100 Subject: [PATCH 03/10] Remove test logging --- Makefile | 2 +- .../Backup/Mongodump/Mongodump.py | 34 +++++++++---------- mongodb_consistent_backup/Common/MongoUri.py | 4 ++- mongodb_consistent_backup/Main.py | 12 ------- mongodb_consistent_backup/Pipeline/Stage.py | 1 + 5 files changed, 22 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index f179c132..40a84ae9 100644 --- a/Makefile +++ b/Makefile @@ -110,7 +110,7 @@ bin/mongodb-consistent-backup.debian9.$(ARCH): debian9: bin/mongodb-consistent-backup.debian9.$(ARCH) docker: build/rpm/RPMS/$(ARCH)/$(NAME)-$(VERSION)-$(RELEASE).el7.$(ARCH).rpm - docker build --no-cache --tag $(DOCKER_TAG) . + docker build --tag $(DOCKER_TAG) . docker tag $(DOCKER_TAG) $(NAME):latest docker run --rm -i $(DOCKER_TAG) --version diff --git a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py index 4e160bdf..ce9d3200 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py +++ b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py @@ -1,6 +1,5 @@ import os import logging -import pprint from math import floor from subprocess import check_output @@ -125,24 +124,25 @@ def threads(self, threads=None): def run(self): self.timer.start(self.timer_name) - pprint.pprint(self.replsets) - # backup a secondary from each shard: for shard in self.replsets: - secondary = self.replsets[shard].find_secondary() - mongo_uri = secondary['uri'] - self.states[shard] = OplogState(self.manager, mongo_uri) - thread = MongodumpThread( - self.states[shard], - mongo_uri, - self.timer, - self.config, - self.backup_dir, - self.version, - self.threads(), - self.do_gzip() - ) - self.dump_threads.append(thread) + try: + secondary = self.replsets[shard].find_secondary() + mongo_uri = secondary['uri'] + self.states[shard] = OplogState(self.manager, mongo_uri) + thread = MongodumpThread( + self.states[shard], + mongo_uri, + self.timer, + self.config, + self.backup_dir, + self.version, + self.threads(), + self.do_gzip() + ) + self.dump_threads.append(thread) + except Exception: + raise OperationError("Failed to get secondary for shard %s: %s" % (shard, e)) if not len(self.dump_threads) > 0: raise OperationError('No backup threads started!') diff --git a/mongodb_consistent_backup/Common/MongoUri.py b/mongodb_consistent_backup/Common/MongoUri.py index 86193062..a0293e10 100644 --- a/mongodb_consistent_backup/Common/MongoUri.py +++ b/mongodb_consistent_backup/Common/MongoUri.py @@ -25,6 +25,7 @@ def __init__(self, url, default_port=27017, replset=None): self.default_port = default_port self.replset = replset + self.srv = False self.addrs = [] self.addr_idx = 0 @@ -62,7 +63,8 @@ def parse(self): self.srv = True return True - self.replset, self.url = self.url.split("/") + if "/" in self.url: + self.replset, self.url = self.url.split("/") for url in self.url.split(","): addr = MongoAddr() addr.replset = self.replset diff --git a/mongodb_consistent_backup/Main.py b/mongodb_consistent_backup/Main.py index cac562b6..f84d23dd 100644 --- a/mongodb_consistent_backup/Main.py +++ b/mongodb_consistent_backup/Main.py @@ -3,8 +3,6 @@ import signal import sys -import pprint - from datetime import datetime from multiprocessing import current_process, Event, Manager @@ -265,15 +263,7 @@ def run(self): self.db ) replset_name = self.replset.get_rs_name() - - # REMOVEME - logging.info("replset name: %s" % replset_name) - replset_dir = os.path.join(self.backup_directory, replset_name) - - # REMOVEME - logging.info("replset dir: %s" % replset_dir) - self.replsets[replset_name] = self.replset state = StateBackupReplset(replset_dir, self.config, self.backup_time, replset_name) state.load_state(self.replset.summary()) @@ -281,8 +271,6 @@ def run(self): except Exception, e: self.exception("Problem getting shard secondaries! Error: %s" % e, e) - pprint.pprint(self.replsets) - # run backup try: self.backup = Backup( diff --git a/mongodb_consistent_backup/Pipeline/Stage.py b/mongodb_consistent_backup/Pipeline/Stage.py index d8e86155..6893d91c 100644 --- a/mongodb_consistent_backup/Pipeline/Stage.py +++ b/mongodb_consistent_backup/Pipeline/Stage.py @@ -84,6 +84,7 @@ def run(self): data = self._task.run() self.stopped = True except Exception, e: + logging.error("State %s returned error: %s" % (self.stage, e)) raise OperationError(e) finally: self.running = False From f71d13492ad6eba017fd65d7fa538d4c5801a938 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 5 Dec 2018 14:43:17 +0100 Subject: [PATCH 04/10] Move raise -> log+raise --- mongodb_consistent_backup/Backup/Mongodump/Mongodump.py | 3 ++- mongodb_consistent_backup/Pipeline/Stage.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py index ce9d3200..197578ef 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py +++ b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py @@ -142,7 +142,8 @@ def run(self): ) self.dump_threads.append(thread) except Exception: - raise OperationError("Failed to get secondary for shard %s: %s" % (shard, e)) + logging.error("Failed to get secondary for shard %s: %s" % (shard, e)) + raise e if not len(self.dump_threads) > 0: raise OperationError('No backup threads started!') diff --git a/mongodb_consistent_backup/Pipeline/Stage.py b/mongodb_consistent_backup/Pipeline/Stage.py index 6893d91c..58f4e01c 100644 --- a/mongodb_consistent_backup/Pipeline/Stage.py +++ b/mongodb_consistent_backup/Pipeline/Stage.py @@ -84,7 +84,7 @@ def run(self): data = self._task.run() self.stopped = True except Exception, e: - logging.error("State %s returned error: %s" % (self.stage, e)) + logging.error("State %s returned error: %s" % (self.stage, e)) raise OperationError(e) finally: self.running = False From 82f37d9b666e167f7c3c8ddb4daef5433ab42a32 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 5 Dec 2018 15:07:37 +0100 Subject: [PATCH 05/10] Fix URI check + build.sh --- mongodb_consistent_backup/Common/MongoUri.py | 6 ++---- scripts/build.sh | 9 ++++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/mongodb_consistent_backup/Common/MongoUri.py b/mongodb_consistent_backup/Common/MongoUri.py index a0293e10..2acaaba5 100644 --- a/mongodb_consistent_backup/Common/MongoUri.py +++ b/mongodb_consistent_backup/Common/MongoUri.py @@ -54,11 +54,9 @@ def __str__(self): def parse(self): # allow mongodb+srv:// URI if self.url.startswith("mongodb+srv://"): - if "replicaSet=" not in self.url: - raise OperationError("replicaSet=X flag required when using mongodb+srv:// URI") - rsSearch = re.search('replicaSet=(\S+)(&.+)?$', self.url, re.IGNORECASE) + rsSearch = re.search('replicaSet=(\S+)(&.+)?$', self.url) if not rsSearch: - raise OperationError("cannot find required replicaSet=X flag in mongodb+srv:// URI") + raise OperationError("replicaSet=X flag required when using mongodb+srv:// URI") self.replset = rsSearch.group(1) self.srv = True return True diff --git a/scripts/build.sh b/scripts/build.sh index b67cafa2..09af6c85 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -91,7 +91,14 @@ if [ -d ${srcdir} ]; then pip_flags="--download-cache=${pipdir}" ${venvdir}/bin/python2.7 ${venvdir}/bin/pip --help | grep -q '\-\-cache\-dir' [ $? = 0 ] && pip_flags="--cache-dir=${pipdir}" - ${venvdir}/bin/python2.7 ${venvdir}/bin/pip install ${pip_flags} pex==1.4 requests + ${venvdir}/bin/python2.7 ${venvdir}/bin/pip install ${pip_flags} "requests" + if [ $? -gt 0 ]; then + echo "Failed to install 'requests'!" + exit 1 + fi + + # build fails on Pex 1.5+ + ${venvdir}/bin/python2.7 ${venvdir}/bin/pip install ${pip_flags} "pex<=1.4" if [ $? -gt 0 ]; then echo "Failed to install pex utility for building!" exit 1 From f6b5b12da107a1fc00979208078629fe6af7db9a Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 5 Dec 2018 15:29:30 +0100 Subject: [PATCH 06/10] Fix missing 'e' in except-line --- mongodb_consistent_backup/Backup/Mongodump/Mongodump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py index 197578ef..af0974a1 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py +++ b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py @@ -141,7 +141,7 @@ def run(self): self.do_gzip() ) self.dump_threads.append(thread) - except Exception: + except Exception, e: logging.error("Failed to get secondary for shard %s: %s" % (shard, e)) raise e From c6007de659bcb6a4b015ee310bf841a4b19642f4 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 5 Dec 2018 15:52:38 +0100 Subject: [PATCH 07/10] revert 3.6 tools --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d2ac0e41..2fbfcb18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM centos:centos7 MAINTAINER Tim Vaillancourt RUN yum install -y https://www.percona.com/redir/downloads/percona-release/redhat/latest/percona-release-0.1-6.noarch.rpm epel-release && \ - yum install -y Percona-Server-MongoDB-36-tools zbackup && yum clean all + yum install -y Percona-Server-MongoDB-34-tools zbackup && yum clean all ADD build/rpm/RPMS/x86_64/mongodb_consistent_backup*.el7.x86_64.rpm / RUN yum localinstall -y /mongodb_consistent_backup*.el7.x86_64.rpm && \ From d2608ee598a62c8aba954bbbe24f54537d26333e Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 5 Dec 2018 15:57:29 +0100 Subject: [PATCH 08/10] 1.4.0 -> 1.4.1 version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 88c5fb89..347f5833 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.0 +1.4.1 From fdeaf47e5dd326ed4246bfc9405bfee385594a15 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 5 Dec 2018 15:58:33 +0100 Subject: [PATCH 09/10] Fix flake8 warn --- mongodb_consistent_backup/Common/MongoUri.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Common/MongoUri.py b/mongodb_consistent_backup/Common/MongoUri.py index 2acaaba5..6f625efb 100644 --- a/mongodb_consistent_backup/Common/MongoUri.py +++ b/mongodb_consistent_backup/Common/MongoUri.py @@ -54,7 +54,7 @@ def __str__(self): def parse(self): # allow mongodb+srv:// URI if self.url.startswith("mongodb+srv://"): - rsSearch = re.search('replicaSet=(\S+)(&.+)?$', self.url) + rsSearch = re.search(r'replicaSet=(\S+)(&.+)?$', self.url) if not rsSearch: raise OperationError("replicaSet=X flag required when using mongodb+srv:// URI") self.replset = rsSearch.group(1) From b0639f1a9878ea9fdb6c0065c67d82b62c89b19a Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 5 Dec 2018 16:11:20 +0100 Subject: [PATCH 10/10] Add --no-cache back --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 40a84ae9..f179c132 100644 --- a/Makefile +++ b/Makefile @@ -110,7 +110,7 @@ bin/mongodb-consistent-backup.debian9.$(ARCH): debian9: bin/mongodb-consistent-backup.debian9.$(ARCH) docker: build/rpm/RPMS/$(ARCH)/$(NAME)-$(VERSION)-$(RELEASE).el7.$(ARCH).rpm - docker build --tag $(DOCKER_TAG) . + docker build --no-cache --tag $(DOCKER_TAG) . docker tag $(DOCKER_TAG) $(NAME):latest docker run --rm -i $(DOCKER_TAG) --version