From e09585920b4c5382d76943f10ea1afbb09d70965 Mon Sep 17 00:00:00 2001 From: Daniel Schneller Date: Thu, 15 Nov 2018 19:24:35 +0100 Subject: [PATCH 1/2] Add --archive.tar.binary parameter Allows specifying a custom location of the "tar" command to use. Also, the flags sent to "tar" are sent individually (`tar -cf` becomes `tar -c -f`). This allows easily customizing how the archiving is performed without having to add lots of new options. For example, you could encrypt backup data via a simple shell script and specify it for --archive.tar.binary: ``` #!/bin/bash gpg_pubkey_id=XXXXXXX new_args="" while [ "${#}" -gt 0 ]; do case "$1" in -f) shift; original_output_file="${1}" shift new_args="${new_args} --to-stdout" ;; *) new_args="${new_args} ${1}" shift ;; esac done tar ${new_args} | gpg --always-trust --encrypt --recipient ${gpg_pubkey_id} -z 0 --output ${original_output_file} ``` This has several advantages: * Backups are never written to disk unencrypted * Encryption can be done in one go, instead of causing the potentially heavy additional I/O a separate encryption step would incur. * It's transparent for the upload stages, so you can still benefit from the integrated S3 (or other) uploads. --- mongodb_consistent_backup/Archive/Tar/Tar.py | 2 +- mongodb_consistent_backup/Archive/Tar/TarThread.py | 7 ++++--- mongodb_consistent_backup/Archive/Tar/__init__.py | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/mongodb_consistent_backup/Archive/Tar/Tar.py b/mongodb_consistent_backup/Archive/Tar/Tar.py index 5f698db2..4978276b 100644 --- a/mongodb_consistent_backup/Archive/Tar/Tar.py +++ b/mongodb_consistent_backup/Archive/Tar/Tar.py @@ -26,7 +26,7 @@ class Tar(Task): def __init__(self, manager, config, timer, base_dir, backup_dir, **kwargs): super(Tar, self).__init__(self.__class__.__name__, manager, config, timer, base_dir, backup_dir, **kwargs) self.compression_method = self.config.archive.tar.compression - self.binary = "tar" + self.binary = self.config.archive.tar.binary self._pool = None self._pooled = [] diff --git a/mongodb_consistent_backup/Archive/Tar/TarThread.py b/mongodb_consistent_backup/Archive/Tar/TarThread.py index fc47f1e0..ff6d9f10 100644 --- a/mongodb_consistent_backup/Archive/Tar/TarThread.py +++ b/mongodb_consistent_backup/Archive/Tar/TarThread.py @@ -32,12 +32,13 @@ def run(self): backup_base_name = os.path.basename(self.backup_dir) log_msg = "Archiving directory: %s" % self.backup_dir - cmd_flags = ["-C", backup_base_dir, "-cf", self.output_file, "--remove-files", backup_base_name] + cmd_flags = ["-C", backup_base_dir, "-c", "-f", self.output_file, "--remove-files"] if self.do_gzip(): - log_msg = "Archiving and compressing directory: %s" % self.backup_dir - cmd_flags = ["-C", backup_base_dir, "-czf", self.output_file, "--remove-files", backup_base_name] + log_msg = "Archiving and compressing directory: %s" % self.backup_dir + cmd_flags.append("-z") + cmd_flags.append(backup_base_name) logging.info(log_msg) self.running = True self._command = LocalCommand(self.binary, cmd_flags, self.verbose) diff --git a/mongodb_consistent_backup/Archive/Tar/__init__.py b/mongodb_consistent_backup/Archive/Tar/__init__.py index 63e733f8..1c9bc69b 100644 --- a/mongodb_consistent_backup/Archive/Tar/__init__.py +++ b/mongodb_consistent_backup/Archive/Tar/__init__.py @@ -2,6 +2,8 @@ def config(parser): + parser.add_argument("--archive.tar.binary", dest="archive.tar.binary", default='tar', type=str, + help="Path to tar binary (default: tar)") parser.add_argument("--archive.tar.compression", dest="archive.tar.compression", help="Tar archiver compression method (default: gzip)", default='gzip', choices=['gzip', 'none']) parser.add_argument("--archive.tar.threads", dest="archive.tar.threads", From 70826b7940d6cffa98d19d71fc02547487d95f3e Mon Sep 17 00:00:00 2001 From: Daniel Schneller Date: Thu, 22 Nov 2018 18:48:16 +0100 Subject: [PATCH 2/2] Related: Fix flake8: Make regex a raw string --- mongodb_consistent_backup/Upload/Rsync/Rsync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongodb_consistent_backup/Upload/Rsync/Rsync.py b/mongodb_consistent_backup/Upload/Rsync/Rsync.py index 6c7d1e0d..d0580caa 100644 --- a/mongodb_consistent_backup/Upload/Rsync/Rsync.py +++ b/mongodb_consistent_backup/Upload/Rsync/Rsync.py @@ -56,7 +56,7 @@ def init(self): def rsync_info(self): if not self._rsync_info: output = check_output([self.rsync_binary, "--version"]) - search = re.search("^rsync\s+version\s([0-9.-]+)\s+protocol\sversion\s(\d+)", output) + search = re.search(r"^rsync\s+version\s([0-9.-]+)\s+protocol\sversion\s(\d+)", output) self.rsync_version = search.group(1) self._rsync_info = {"version": self.rsync_version, "protocol_version": int(search.group(2))} return self._rsync_info