From 45e0d10932f9f2818b6984814cdd12fc56b1d9fa Mon Sep 17 00:00:00 2001 From: Calum Lind Date: Fri, 9 Jun 2017 13:02:45 +0100 Subject: [PATCH] Update archiving of state file to use tarfile --- deluge/common.py | 42 +++++++++++++++++++++++++++++++++++ deluge/core/torrentmanager.py | 35 +++++++++-------------------- deluge/tests/test_common.py | 19 ++++++++++++++-- 3 files changed, 69 insertions(+), 27 deletions(-) diff --git a/deluge/common.py b/deluge/common.py index 35cab102d..a5c47706a 100644 --- a/deluge/common.py +++ b/deluge/common.py @@ -11,7 +11,9 @@ from __future__ import division, print_function, unicode_literals import base64 +import datetime import functools +import glob import locale import logging import numbers @@ -20,6 +22,7 @@ import platform import re import subprocess import sys +import tarfile import time import chardet @@ -138,6 +141,45 @@ def get_default_download_dir(): return download_dir +def archive_files(arc_name, filepaths): + """Compress a list of filepaths into timestamped tarball in config dir. + + The archiving config directory is 'archive'. + + Args: + arc_name (str): The archive output filename (appended with timestamp). + filepaths (list): A list of the files to be archived into tarball. + + Returns: + str: The full archive filepath. + + """ + + from deluge.configmanager import get_config_dir + + # Set archive compression to lzma with bz2 fallback. + arc_comp = 'xz' if not PY2 else 'bz2' + + archive_dir = os.path.join(get_config_dir(), 'archive') + timestamp = datetime.datetime.now().replace(microsecond=0).isoformat().replace(':', '-') + arc_filepath = os.path.join(archive_dir, arc_name + '-' + timestamp + '.tar.' + arc_comp) + max_num_arcs = 20 + + if not os.path.exists(archive_dir): + os.makedirs(archive_dir) + else: + old_arcs = glob.glob(os.path.join(archive_dir, arc_name) + '*') + if len(old_arcs) > max_num_arcs: + # TODO: Remove oldest timestamped archives. + log.warning('More than %s tarballs in config archive', max_num_arcs) + + with tarfile.open(arc_filepath, 'w:' + arc_comp) as tf: + for filepath in filepaths: + tf.add(filepath, arcname=os.path.basename(filepath)) + + return arc_filepath + + def windows_check(): """ Checks if the current platform is Windows diff --git a/deluge/core/torrentmanager.py b/deluge/core/torrentmanager.py index 57ef22d5c..d143ff0a8 100644 --- a/deluge/core/torrentmanager.py +++ b/deluge/core/torrentmanager.py @@ -15,7 +15,6 @@ import datetime import logging import operator import os -import shutil import time from twisted.internet import defer, reactor, threads @@ -24,7 +23,7 @@ from twisted.internet.task import LoopingCall import deluge.component as component from deluge._libtorrent import lt -from deluge.common import decode_bytes, get_magnet_info +from deluge.common import archive_files, decode_bytes, get_magnet_info from deluge.configmanager import ConfigManager, get_config_dir from deluge.core.authmanager import AUTH_LEVEL_ADMIN from deluge.core.torrent import Torrent, TorrentOptions, sanitize_filepath @@ -171,30 +170,16 @@ class TorrentManager(component.Component): def start(self): # Check for old temp file to verify safe shutdown if os.path.isfile(self.temp_file): - def archive_file(filename): - """Archives the file in 'archive' sub-directory with timestamp appended""" - filepath = os.path.join(self.state_dir, filename) - filepath_bak = filepath + '.bak' - archive_dir = os.path.join(get_config_dir(), 'archive') - if not os.path.exists(archive_dir): - os.makedirs(archive_dir) - - for _filepath in (filepath, filepath_bak): - timestamp = datetime.datetime.now().replace(microsecond=0).isoformat().replace(':', '-') - archive_filepath = os.path.join(archive_dir, filename + '-' + timestamp) - try: - shutil.copy2(_filepath, archive_filepath) - except IOError: - log.error('Unable to archive: %s', filename) - else: - log.info('Archive of %s successful: %s', filename, archive_filepath) - log.warning('Potential bad shutdown of Deluge detected, archiving torrent state files...') - archive_file('torrents.state') - archive_file('torrents.fastresume') - else: - with open(self.temp_file, 'a'): - os.utime(self.temp_file, None) + arc_filepaths = [] + for filename in ('torrents.fastresume', 'torrents.state'): + filepath = os.path.join(self.state_dir, filename) + arc_filepaths.extend([filepath, filepath + '.bak']) + archive_files('torrents_state', arc_filepaths) + os.remove(self.temp_file) + + with open(self.temp_file, 'a'): + os.utime(self.temp_file, None) # Try to load the state from file self.load_state() diff --git a/deluge/tests/test_common.py b/deluge/tests/test_common.py index 12e6b8883..404a6c405 100644 --- a/deluge/tests/test_common.py +++ b/deluge/tests/test_common.py @@ -8,16 +8,20 @@ from __future__ import unicode_literals import os +import tarfile from twisted.trial import unittest -from deluge.common import (VersionSplit, fdate, fpcnt, fpeer, fsize, fspeed, ftime, get_path_size, is_infohash, is_ip, - is_ipv4, is_ipv6, is_magnet, is_url) +from deluge.common import (VersionSplit, archive_files, fdate, fpcnt, fpeer, fsize, fspeed, ftime, get_path_size, + is_infohash, is_ip, is_ipv4, is_ipv6, is_magnet, is_url) from deluge.ui.translations_util import setup_translations +from .common import get_test_data_file, set_tmp_config_dir + class CommonTestCase(unittest.TestCase): def setUp(self): # NOQA + self.config_dir = set_tmp_config_dir() setup_translations() def tearDown(self): # NOQA @@ -127,3 +131,14 @@ class CommonTestCase(unittest.TestCase): for human_size, byte_size in sizes: parsed = parse_human_size(human_size) self.assertEqual(parsed, byte_size, 'Mismatch when converting: %s' % human_size) + + def test_archive_files(self): + arc_filelist = [ + get_test_data_file('test.torrent'), + get_test_data_file('deluge.png')] + arc_filepath = archive_files('test-arc', arc_filelist) + + with tarfile.open(arc_filepath, 'r') as tar: + for tar_info in tar: + self.assertTrue(tar_info.isfile()) + self.assertTrue(tar_info.name in [os.path.basename(arcf) for arcf in arc_filelist])