diff --git a/VERSION b/VERSION index 88c5fb89..347f5833 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.0 +1.4.1 diff --git a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py index 61284e15..af0974a1 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py +++ b/mongodb_consistent_backup/Backup/Mongodump/Mongodump.py @@ -126,20 +126,24 @@ def run(self): # backup a secondary from each shard: for shard in self.replsets: - secondary = self.replsets[shard].find_secondary() - mongo_uri = secondary['uri'] - self.states[shard] = OplogState(self.manager, mongo_uri) - thread = MongodumpThread( - self.states[shard], - mongo_uri, - self.timer, - self.config, - self.backup_dir, - self.version, - self.threads(), - self.do_gzip() - ) - self.dump_threads.append(thread) + try: + secondary = self.replsets[shard].find_secondary() + mongo_uri = secondary['uri'] + self.states[shard] = OplogState(self.manager, mongo_uri) + thread = MongodumpThread( + self.states[shard], + mongo_uri, + self.timer, + self.config, + self.backup_dir, + self.version, + self.threads(), + self.do_gzip() + ) + self.dump_threads.append(thread) + except Exception, e: + logging.error("Failed to get secondary for shard %s: %s" % (shard, e)) + raise e if not len(self.dump_threads) > 0: raise OperationError('No backup threads started!') diff --git a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py index 29e0f76d..145554ac 100644 --- a/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py +++ b/mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py @@ -138,12 +138,24 @@ def wait(self): def mongodump_cmd(self): mongodump_uri = self.uri.get() mongodump_cmd = [self.binary] - mongodump_flags = [ - "--host=%s" % mongodump_uri.host, - "--port=%s" % str(mongodump_uri.port), + mongodump_flags = [] + + # --host/--port (suport mongodb+srv:// too) + if self.uri.srv: + if not self.is_version_gte("3.6.0"): + logging.fatal("Mongodump must be >= 3.6.0 to use mongodb+srv:// URIs") + sys.exit(1) + mongodump_flags.append("--host=%s" % self.uri.url) + else: + mongodump_flags.extend([ + "--host=%s" % mongodump_uri.host, + "--port=%s" % str(mongodump_uri.port) + ]) + + mongodump_flags.extend([ "--oplog", "--out=%s/dump" % self.backup_dir - ] + ]) # --numParallelCollections if self.threads > 0: diff --git a/mongodb_consistent_backup/Common/DB.py b/mongodb_consistent_backup/Common/DB.py index 44c42326..e8d2aa9c 100644 --- a/mongodb_consistent_backup/Common/DB.py +++ b/mongodb_consistent_backup/Common/DB.py @@ -3,7 +3,7 @@ from bson.codec_options import CodecOptions from inspect import currentframe, getframeinfo from pymongo import DESCENDING, CursorType, MongoClient -from pymongo.errors import ConnectionFailure, OperationFailure, ServerSelectionTimeoutError +from pymongo.errors import ConfigurationError, ConnectionFailure, OperationFailure, ServerSelectionTimeoutError from ssl import CERT_REQUIRED, CERT_NONE from time import sleep @@ -113,7 +113,7 @@ def connect(self): conn = MongoClient(**self.client_opts()) if self.do_connect: conn['admin'].command({"ping": 1}) - except (ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e: + except (ConfigurationError, ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e: logging.error("Unable to connect to %s! Error: %s" % (self.uri, e)) raise DBConnectionError(e) if conn is not None: diff --git a/mongodb_consistent_backup/Common/MongoUri.py b/mongodb_consistent_backup/Common/MongoUri.py index c9b65a6f..6f625efb 100644 --- a/mongodb_consistent_backup/Common/MongoUri.py +++ b/mongodb_consistent_backup/Common/MongoUri.py @@ -1,4 +1,7 @@ +import re + from Util import validate_hostname +from mongodb_consistent_backup.Errors import OperationError class MongoAddr: @@ -22,12 +25,15 @@ def __init__(self, url, default_port=27017, replset=None): self.default_port = default_port self.replset = replset + self.srv = False self.addrs = [] self.addr_idx = 0 self.parse() def hosts(self): + if self.srv: + return self.url if len(self.addrs) > 0: hosts = [] for addr in self.addrs: @@ -35,6 +41,8 @@ def hosts(self): return ",".join(hosts) def str(self): + if self.srv: + return self.url string = self.hosts() if self.replset: string = "%s/%s" % (self.replset, string) @@ -44,6 +52,15 @@ def __str__(self): return self.str() def parse(self): + # allow mongodb+srv:// URI + if self.url.startswith("mongodb+srv://"): + rsSearch = re.search(r'replicaSet=(\S+)(&.+)?$', self.url) + if not rsSearch: + raise OperationError("replicaSet=X flag required when using mongodb+srv:// URI") + self.replset = rsSearch.group(1) + self.srv = True + return True + if "/" in self.url: self.replset, self.url = self.url.split("/") for url in self.url.split(","): diff --git a/mongodb_consistent_backup/Pipeline/Stage.py b/mongodb_consistent_backup/Pipeline/Stage.py index d8e86155..58f4e01c 100644 --- a/mongodb_consistent_backup/Pipeline/Stage.py +++ b/mongodb_consistent_backup/Pipeline/Stage.py @@ -84,6 +84,7 @@ def run(self): data = self._task.run() self.stopped = True except Exception, e: + logging.error("State %s returned error: %s" % (self.stage, e)) raise OperationError(e) finally: self.running = False diff --git a/requirements.txt b/requirements.txt index 0913ce40..14507168 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ yconf==0.3.4 google_compute_engine==2.7.2 progress==1.3 py-zabbix==1.1.3 +dnspython==1.15 diff --git a/scripts/build.sh b/scripts/build.sh index e4ee53bc..50c85539 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -103,7 +103,14 @@ if [ -d ${srcdir} ]; then pip_flags="--download-cache=${pipdir}" ${venvdir}/bin/python2.7 ${venvdir}/bin/pip --help | grep -q '\-\-cache\-dir' [ $? = 0 ] && pip_flags="--cache-dir=${pipdir}" - ${venvdir}/bin/python2.7 ${venvdir}/bin/pip install ${pip_flags} pex requests + ${venvdir}/bin/python2.7 ${venvdir}/bin/pip install ${pip_flags} "requests" + if [ $? -gt 0 ]; then + echo "Failed to install 'requests'!" + exit 1 + fi + + # build fails on Pex 1.5+ + ${venvdir}/bin/python2.7 ${venvdir}/bin/pip install ${pip_flags} "pex<=1.4" if [ $? -gt 0 ]; then echo "Failed to install pex utility for building!" exit 1