123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234 |
- import contextlib
- import os
- import platform
- import subprocess
- import sys
- import time
- import yaml
- from glob import glob
- def term():
- """Get the Terminal reference to make output pretty
- Returns:
- (blessings.Terminal): Returns
- a `blessings <https://blessings.readthedocs.io/en/latest>`_ terminal
- instance. If running in windows and not cygwin it will return an
- `intercessions <https://pypi.org/project/intercessions>`_ terminal
- instance instead
- """
- if not hasattr(term, '_handle'):
- if sys.platform != "cygwin" and platform.system() == 'Windows':
- from intercessions import Terminal
- else:
- from blessings import Terminal
- term._handle = Terminal()
- return term._handle
- @contextlib.contextmanager
- def pushd(newDir):
- previousDir = os.getcwd()
- os.chdir(newDir)
- try:
- yield
- finally:
- os.chdir(previousDir)
- class DependencyException(Exception):
- pass
- class BackupException(Exception):
- pass
- def deptree(sources):
- deps = []
- deferred = []
- for name in sources.keys():
- source = sources[name]
- if "depends" in source and not source["depends"]:
- deferred.append(name)
- else:
- deps.append(name)
- while deferred:
- name = deferred.pop()
- depends = sources[name]["depends"]
- # todo - detect dependency loop
- if name in depends:
- raise DependencyException('Source {} depends upon itself'.format(name))
- elif set(depends).issubset(set(deps)):
- deps.append(name)
- elif not set(depends).issubset(set(deps)):
- missing = ', '.join(set(depends).difference(set(deps)))
- raise DependencyException(
- 'Source {0} has missing dependencies: {1}'.format(name, missing))
- else:
- deferred.append(name)
- return deps
- def status(name, value=None):
- if not hasattr(status, "_handle"):
- status._handle = {}
- if value is None:
- return status._handle[name] if name in status._handle else None
- status._handle[name] = value
- def backup(source):
- failed = [x for x in source["depends"] if not status(x)]
- path = source['path']
- if failed:
- process = BackupException(
- "Unable to backup {0} due to incomplete backups: {1}".format(
- path, ', '.join(failed)))
- active = False
- else:
- args = []
- process = subprocess.Popen(
- ['rdiff-backup'] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
- stdin=subprocess.DEVNULL, universal_newlines=True)
- active = True
- return {
- 'active': active,
- 'name': os.path.basename(path),
- 'path': path,
- 'process': process
- }
- def config():
- if hasattr(config, '_handle'):
- return config._handle
- with pushd('etc/backup.d'):
- with open("backup.yml") as f:
- config._handle = yaml.load(f)
- return config._handle
- def sources():
- if hasattr(sources, '_handle'):
- return sources._handle
- sources._handle = {}
- with pushd('etc/backup.d'):
- for source in config()['sources']:
- source = os.path.realpath(source)
- for path in glob('{}/*.yml'.format(source)):
- path = os.path.realpath(path)
- with pushd(os.path.dirname(path)), open(path) as f:
- data = yaml.load(f)
- if "active" in data and data["active"]:
- data['path'] = path
- if "depends" not in data:
- data["depends"] = []
- for i in range(0, len(data["depends"])):
- data["depends"][i] = os.path.realpath(
- '{}.yml'.format(data["depends"][i]))
- sources._handle[path] = data
- return sources._handle
- def logStatus(tasks):
- errors = []
- t = term()
- for task in list(tasks):
- print(task['name'], end=': ')
- code = 1 if isinstance(
- task['process'], BackupException) else task['process'].poll()
- if code is None:
- print(t.blue('...'))
- continue
- if code:
- print(t.red('fail'))
- else:
- print(t.green('ok'))
- if task['active']:
- if code:
- errors.append(task['name'])
- task['active'] = False
- status(task['path'], not code)
- return errors
- def main(args):
- engine = config()["engine"]
- if engine not in ("rdiff-backup"):
- raise BackupException('Unknown backup engine: {}'.format(engine))
- tasks = []
- errors = []
- maxconcurrent = config()["maxconcurrent"]
- tree = deptree(sources())
- t = term()
- with t.fullscreen(), t.hidden_cursor():
- while len(tree):
- if len([x for x in tasks if x['active']]) < maxconcurrent:
- path = tree.pop(0)
- source = sources()[path]
- if not [x for x in source["depends"] if status(x) is None]:
- tasks.append(backup(source))
- else:
- tree.append(path)
- print(t.clear + t.move(0, 0), end='')
- errors += logStatus(tasks)
- while len([x for x in tasks if x['active']]):
- print(t.clear + t.move(0, 0), end='')
- errors += logStatus(tasks)
- time.sleep(1)
- errors += logStatus(tasks)
- if errors:
- raise BackupException("At least one backup failed")
- if __name__ == '__main__':
- try:
- main(sys.argv[1:])
- except DependencyException as ex:
- print(ex)
- sys.exit(1)
- except BackupException as ex:
- print(ex)
- sys.exit(1)
- except Exception:
- from traceback import format_exc
- msg = "Error encountered:\n" + format_exc().strip()
- print(msg)
- sys.exit(1)
|