import contextlib import os import platform import subprocess import sys import time import yaml from glob import glob def term(): """Get the Terminal reference to make output pretty Returns: (blessings.Terminal): Returns a `blessings `_ terminal instance. If running in windows and not cygwin it will return an `intercessions `_ terminal instance instead """ if not hasattr(term, '_handle'): if sys.platform != "cygwin" and platform.system() == 'Windows': from intercessions import Terminal else: from blessings import Terminal term._handle = Terminal() return term._handle @contextlib.contextmanager def pushd(newDir): previousDir = os.getcwd() os.chdir(newDir) try: yield finally: os.chdir(previousDir) class DependencyException(Exception): pass class BackupException(Exception): pass def deptree(sources): deps = [] deferred = [] for name in sources.keys(): source = sources[name] if "depends" in source and not source["depends"]: deferred.append(name) else: deps.append(name) while deferred: name = deferred.pop() depends = sources[name]["depends"] # todo - detect dependency loop if name in depends: raise DependencyException('Source {} depends upon itself'.format(name)) elif set(depends).issubset(set(deps)): deps.append(name) elif not set(depends).issubset(set(deps)): missing = ', '.join(set(depends).difference(set(deps))) raise DependencyException( 'Source {0} has missing dependencies: {1}'.format(name, missing)) else: deferred.append(name) return deps def status(name, value=None): if not hasattr(status, "_handle"): status._handle = {} if value is None: return status._handle[name] if name in status._handle else None status._handle[name] = value def backup(source): failed = [x for x in source["depends"] if not status(x)] path = source['path'] if failed: process = BackupException( "Unable to backup {0} due to incomplete backups: {1}".format( path, ', '.join(failed))) active = False else: args = [] process = subprocess.Popen( ['rdiff-backup'] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL, universal_newlines=True) active = True return { 'active': active, 'name': os.path.basename(path), 'path': path, 'process': process } def config(): if hasattr(config, '_handle'): return config._handle with pushd('etc/backup.d'): with open("backup.yml") as f: config._handle = yaml.load(f, Loader=yaml.SafeLoader) return config._handle def sources(): if hasattr(sources, '_handle'): return sources._handle sources._handle = {} with pushd('etc/backup.d'): for source in config()['sources']: source = os.path.realpath(source) for path in glob('{}/*.yml'.format(source)): path = os.path.realpath(path) with pushd(os.path.dirname(path)), open(path) as f: data = yaml.load(f, Loader=yaml.SafeLoader) if "active" in data and data["active"]: data['path'] = path if "depends" not in data: data["depends"] = [] for i in range(0, len(data["depends"])): data["depends"][i] = os.path.realpath( '{}.yml'.format(data["depends"][i])) sources._handle[path] = data return sources._handle def logStatus(tasks): errors = [] t = term() for task in list(tasks): print(task['name'], end=': ') code = 1 if isinstance( task['process'], BackupException) else task['process'].poll() if code is None: print(t.blue('...')) continue if code: print(t.red('fail')) else: print(t.green('ok')) if task['active']: if code: errors.append(task['name']) task['active'] = False status(task['path'], not code) return errors def main(args): engine = config()["engine"] if engine not in ("rdiff-backup"): raise BackupException('Unknown backup engine: {}'.format(engine)) tasks = [] errors = [] maxconcurrent = config()["maxconcurrent"] tree = deptree(sources()) t = term() with t.fullscreen(), t.hidden_cursor(): while len(tree): if len([x for x in tasks if x['active']]) < maxconcurrent: path = tree.pop(0) source = sources()[path] if not [x for x in source["depends"] if status(x) is None]: tasks.append(backup(source)) else: tree.append(path) print(t.clear + t.move(0, 0), end='') errors += logStatus(tasks) while len([x for x in tasks if x['active']]): print(t.clear + t.move(0, 0), end='') errors += logStatus(tasks) time.sleep(1) errors += logStatus(tasks) if errors: raise BackupException("At least one backup failed") if __name__ == '__main__': try: main(sys.argv[1:]) except DependencyException as ex: print(ex) sys.exit(1) except BackupException as ex: print(ex) sys.exit(1) except Exception: from traceback import format_exc msg = "Error encountered:\n" + format_exc().strip() print(msg) sys.exit(1)