backup.py 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234
  1. import contextlib
  2. import os
  3. import platform
  4. import subprocess
  5. import sys
  6. import time
  7. import yaml
  8. from glob import glob
  9. def term():
  10. """Get the Terminal reference to make output pretty
  11. Returns:
  12. (blessings.Terminal): Returns
  13. a `blessings <https://blessings.readthedocs.io/en/latest>`_ terminal
  14. instance. If running in windows and not cygwin it will return an
  15. `intercessions <https://pypi.org/project/intercessions>`_ terminal
  16. instance instead
  17. """
  18. if not hasattr(term, '_handle'):
  19. if sys.platform != "cygwin" and platform.system() == 'Windows':
  20. from intercessions import Terminal
  21. else:
  22. from blessings import Terminal
  23. term._handle = Terminal()
  24. return term._handle
  25. @contextlib.contextmanager
  26. def pushd(newDir):
  27. previousDir = os.getcwd()
  28. os.chdir(newDir)
  29. try:
  30. yield
  31. finally:
  32. os.chdir(previousDir)
  33. class DependencyException(Exception):
  34. pass
  35. class BackupException(Exception):
  36. pass
  37. def deptree(sources):
  38. deps = []
  39. deferred = []
  40. for name in sources.keys():
  41. source = sources[name]
  42. if "depends" in source and not source["depends"]:
  43. deferred.append(name)
  44. else:
  45. deps.append(name)
  46. while deferred:
  47. name = deferred.pop()
  48. depends = sources[name]["depends"]
  49. # todo - detect dependency loop
  50. if name in depends:
  51. raise DependencyException('Source {} depends upon itself'.format(name))
  52. elif set(depends).issubset(set(deps)):
  53. deps.append(name)
  54. elif not set(depends).issubset(set(deps)):
  55. missing = ', '.join(set(depends).difference(set(deps)))
  56. raise DependencyException(
  57. 'Source {0} has missing dependencies: {1}'.format(name, missing))
  58. else:
  59. deferred.append(name)
  60. return deps
  61. def status(name, value=None):
  62. if not hasattr(status, "_handle"):
  63. status._handle = {}
  64. if value is None:
  65. return status._handle[name] if name in status._handle else None
  66. status._handle[name] = value
  67. def backup(source):
  68. failed = [x for x in source["depends"] if not status(x)]
  69. path = source['path']
  70. if failed:
  71. process = BackupException(
  72. "Unable to backup {0} due to incomplete backups: {1}".format(
  73. path, ', '.join(failed)))
  74. active = False
  75. else:
  76. args = []
  77. process = subprocess.Popen(
  78. ['rdiff-backup'] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  79. stdin=subprocess.DEVNULL, universal_newlines=True)
  80. active = True
  81. return {
  82. 'active': active,
  83. 'name': os.path.basename(path),
  84. 'path': path,
  85. 'process': process
  86. }
  87. def config():
  88. if hasattr(config, '_handle'):
  89. return config._handle
  90. with pushd('etc/backup.d'):
  91. with open("backup.yml") as f:
  92. config._handle = yaml.load(f)
  93. return config._handle
  94. def sources():
  95. if hasattr(sources, '_handle'):
  96. return sources._handle
  97. sources._handle = {}
  98. with pushd('etc/backup.d'):
  99. for source in config()['sources']:
  100. source = os.path.realpath(source)
  101. for path in glob('{}/*.yml'.format(source)):
  102. path = os.path.realpath(path)
  103. with pushd(os.path.dirname(path)), open(path) as f:
  104. data = yaml.load(f)
  105. if "active" in data and data["active"]:
  106. data['path'] = path
  107. if "depends" not in data:
  108. data["depends"] = []
  109. for i in range(0, len(data["depends"])):
  110. data["depends"][i] = os.path.realpath(
  111. '{}.yml'.format(data["depends"][i]))
  112. sources._handle[path] = data
  113. return sources._handle
  114. def logStatus(tasks):
  115. errors = []
  116. t = term()
  117. for task in list(tasks):
  118. print(task['name'], end=': ')
  119. code = 1 if isinstance(
  120. task['process'], BackupException) else task['process'].poll()
  121. if code is None:
  122. print(t.blue('...'))
  123. continue
  124. if code:
  125. print(t.red('fail'))
  126. else:
  127. print(t.green('ok'))
  128. if task['active']:
  129. if code:
  130. errors.append(task['name'])
  131. task['active'] = False
  132. status(task['path'], not code)
  133. return errors
  134. def main(args):
  135. engine = config()["engine"]
  136. if engine not in ("rdiff-backup"):
  137. raise BackupException('Unknown backup engine: {}'.format(engine))
  138. tasks = []
  139. errors = []
  140. maxconcurrent = config()["maxconcurrent"]
  141. tree = deptree(sources())
  142. t = term()
  143. with t.fullscreen(), t.hidden_cursor():
  144. while len(tree):
  145. if len([x for x in tasks if x['active']]) < maxconcurrent:
  146. path = tree.pop(0)
  147. source = sources()[path]
  148. if not [x for x in source["depends"] if status(x) is None]:
  149. tasks.append(backup(source))
  150. else:
  151. tree.append(path)
  152. print(t.clear + t.move(0, 0), end='')
  153. errors += logStatus(tasks)
  154. while len([x for x in tasks if x['active']]):
  155. print(t.clear + t.move(0, 0), end='')
  156. errors += logStatus(tasks)
  157. time.sleep(1)
  158. errors += logStatus(tasks)
  159. if errors:
  160. raise BackupException("At least one backup failed")
  161. if __name__ == '__main__':
  162. try:
  163. main(sys.argv[1:])
  164. except DependencyException as ex:
  165. print(ex)
  166. sys.exit(1)
  167. except BackupException as ex:
  168. print(ex)
  169. sys.exit(1)
  170. except Exception:
  171. from traceback import format_exc
  172. msg = "Error encountered:\n" + format_exc().strip()
  173. print(msg)
  174. sys.exit(1)