From b02b71e97849923f00edd681777b4254fca332ec Mon Sep 17 00:00:00 2001 From: Marcos Pinto Date: Mon, 15 Oct 2007 06:11:37 +0000 Subject: [PATCH] lt sync 1677 --- libtorrent/bindings/python/src/session.cpp | 17 +- libtorrent/include/libtorrent/alert_types.hpp | 10 + .../include/libtorrent/aux_/session_impl.hpp | 20 +- .../include/libtorrent/disk_io_thread.hpp | 1 + .../include/libtorrent/peer_connection.hpp | 2 +- libtorrent/include/libtorrent/policy.hpp | 2 + libtorrent/include/libtorrent/session.hpp | 16 +- .../include/libtorrent/session_settings.hpp | 2 +- libtorrent/include/libtorrent/storage.hpp | 69 +- libtorrent/include/libtorrent/torrent.hpp | 9 +- .../include/libtorrent/torrent_handle.hpp | 5 +- libtorrent/src/bt_peer_connection.cpp | 5 +- libtorrent/src/disk_io_thread.cpp | 8 + libtorrent/src/peer_connection.cpp | 65 +- libtorrent/src/piece_picker.cpp | 2 - libtorrent/src/policy.cpp | 10 +- libtorrent/src/session.cpp | 16 +- libtorrent/src/session_impl.cpp | 47 +- libtorrent/src/storage.cpp | 748 ++++++++++-------- libtorrent/src/torrent.cpp | 66 +- libtorrent/src/torrent_handle.cpp | 17 +- 21 files changed, 670 insertions(+), 467 deletions(-) diff --git a/libtorrent/bindings/python/src/session.cpp b/libtorrent/bindings/python/src/session.cpp index 6d4855c10..7332da2ea 100755 --- a/libtorrent/bindings/python/src/session.cpp +++ b/libtorrent/bindings/python/src/session.cpp @@ -87,10 +87,10 @@ namespace torrent_handle add_torrent(session& s, torrent_info const& ti , boost::filesystem::path const& save, entry const& resume - , bool compact, bool paused) + , storage_mode_t storage_mode, bool paused) { allow_threading_guard guard; - return s.add_torrent(ti, save, resume, compact, paused, default_storage_constructor); + return s.add_torrent(ti, save, resume, storage_mode, paused, default_storage_constructor); } } // namespace unnamed @@ -154,6 +154,17 @@ void bind_session() #endif ; + enum_("storage_mode_t") + .value("storage_mode_allocate", storage_mode_allocate) + .value("storage_mode_compact", storage_mode_compact) + .value("storage_mode_sparse", storage_mode_sparse) + ; + + enum_("options_t") + .value("none", session::none) + .value("delete_files", session::delete_files) + ; + class_("session", session_doc, no_init) .def( init(arg("fingerprint")=fingerprint("LT",0,1,0,0), session_init_doc) @@ -235,8 +246,6 @@ void bind_session() .def("stop_natpmp", allow_threads(&session::stop_natpmp), session_stop_natpmp_doc) ; - def("supports_sparse_files", &supports_sparse_files); - register_ptr_to_python >(); } diff --git a/libtorrent/include/libtorrent/alert_types.hpp b/libtorrent/include/libtorrent/alert_types.hpp index a46eb7817..6f41758fa 100755 --- a/libtorrent/include/libtorrent/alert_types.hpp +++ b/libtorrent/include/libtorrent/alert_types.hpp @@ -252,6 +252,16 @@ namespace libtorrent { return std::auto_ptr(new storage_moved_alert(*this)); } }; + struct TORRENT_EXPORT torrent_deleted_alert: torrent_alert + { + torrent_deleted_alert(torrent_handle const& h, std::string const& msg) + : torrent_alert(h, alert::warning, msg) + {} + + virtual std::auto_ptr clone() const + { return std::auto_ptr(new torrent_deleted_alert(*this)); } + }; + struct TORRENT_EXPORT torrent_paused_alert: torrent_alert { torrent_paused_alert(torrent_handle const& h, std::string const& msg) diff --git a/libtorrent/include/libtorrent/aux_/session_impl.hpp b/libtorrent/include/libtorrent/aux_/session_impl.hpp index d069d73e3..bff8e3387 100644 --- a/libtorrent/include/libtorrent/aux_/session_impl.hpp +++ b/libtorrent/include/libtorrent/aux_/session_impl.hpp @@ -140,7 +140,7 @@ namespace libtorrent checker_impl(session_impl& s): m_ses(s), m_abort(false) {} void operator()(); piece_checker_data* find_torrent(const sha1_hash& info_hash); - void remove_torrent(sha1_hash const& info_hash); + void remove_torrent(sha1_hash const& info_hash, int options); #ifndef NDEBUG void check_invariant() const; @@ -254,7 +254,7 @@ namespace libtorrent boost::intrusive_ptr ti , fs::path const& save_path , entry const& resume_data - , bool compact_mode + , storage_mode_t storage_mode , storage_constructor_type sc , bool paused , void* userdata); @@ -265,12 +265,12 @@ namespace libtorrent , char const* name , fs::path const& save_path , entry const& resume_data - , bool compact_mode + , storage_mode_t storage_mode , storage_constructor_type sc , bool paused , void* userdata); - void remove_torrent(torrent_handle const& h); + void remove_torrent(torrent_handle const& h, int options); std::vector get_torrents(); @@ -371,12 +371,6 @@ namespace libtorrent void on_lsd_peer(tcp::endpoint peer, sha1_hash const& ih); - // handles disk io requests asynchronously - // peers have pointers into the disk buffer - // pool, and must be destructed before this - // object. - disk_io_thread m_disk_thread; - // this pool is used to allocate and recycle send // buffers from. boost::pool<> m_send_buffers; @@ -395,6 +389,12 @@ namespace libtorrent // when they are destructed. file_pool m_files; + // handles disk io requests asynchronously + // peers have pointers into the disk buffer + // pool, and must be destructed before this + // object. + disk_io_thread m_disk_thread; + // this is a list of half-open tcp connections // (only outgoing connections) // this has to be one of the last diff --git a/libtorrent/include/libtorrent/disk_io_thread.hpp b/libtorrent/include/libtorrent/disk_io_thread.hpp index b893aaf60..bd6d5e1ba 100644 --- a/libtorrent/include/libtorrent/disk_io_thread.hpp +++ b/libtorrent/include/libtorrent/disk_io_thread.hpp @@ -64,6 +64,7 @@ namespace libtorrent , hash , move_storage , release_files + , delete_files }; action_t action; diff --git a/libtorrent/include/libtorrent/peer_connection.hpp b/libtorrent/include/libtorrent/peer_connection.hpp index 2ea3c7d34..e1581affe 100755 --- a/libtorrent/include/libtorrent/peer_connection.hpp +++ b/libtorrent/include/libtorrent/peer_connection.hpp @@ -176,7 +176,7 @@ namespace libtorrent void set_non_prioritized(bool b) { m_non_prioritized = b; } - void fast_reconnect(bool r) { m_fast_reconnect = r; } + void fast_reconnect(bool r); bool fast_reconnect() const { return m_fast_reconnect; } // this adds an announcement in the announcement queue diff --git a/libtorrent/include/libtorrent/policy.hpp b/libtorrent/include/libtorrent/policy.hpp index 551cb4f2c..c38bb426c 100755 --- a/libtorrent/include/libtorrent/policy.hpp +++ b/libtorrent/include/libtorrent/policy.hpp @@ -156,6 +156,8 @@ namespace libtorrent // this is true if the peer is a seed bool seed; + int fast_reconnects; + // true if this peer currently is unchoked // because of an optimistic unchoke. // when the optimistic unchoke is moved to diff --git a/libtorrent/include/libtorrent/session.hpp b/libtorrent/include/libtorrent/session.hpp index 3a9eb563b..5093e2336 100755 --- a/libtorrent/include/libtorrent/session.hpp +++ b/libtorrent/include/libtorrent/session.hpp @@ -115,7 +115,7 @@ namespace libtorrent : m_impl(impl) {} boost::shared_ptr m_impl; }; - + class TORRENT_EXPORT session: public boost::noncopyable, aux::eh_initializer { public: @@ -140,7 +140,7 @@ namespace libtorrent torrent_info const& ti , fs::path const& save_path , entry const& resume_data = entry() - , bool compact_mode = true + , storage_mode_t storage_mode = storage_mode_sparse , bool paused = false , storage_constructor_type sc = default_storage_constructor) TORRENT_DEPRECATED; @@ -148,7 +148,7 @@ namespace libtorrent boost::intrusive_ptr ti , fs::path const& save_path , entry const& resume_data = entry() - , bool compact_mode = true + , storage_mode_t storage_mode = storage_mode_sparse , bool paused = false , storage_constructor_type sc = default_storage_constructor , void* userdata = 0); @@ -159,7 +159,7 @@ namespace libtorrent , char const* name , fs::path const& save_path , entry const& resume_data = entry() - , bool compact_mode = true + , storage_mode_t storage_mode = storage_mode_sparse , bool paused = false , storage_constructor_type sc = default_storage_constructor , void* userdata = 0); @@ -219,7 +219,13 @@ namespace libtorrent // number of half open connections. int num_connections() const; - void remove_torrent(const torrent_handle& h); + enum options_t + { + none = 0, + delete_files = 1 + }; + + void remove_torrent(const torrent_handle& h, int options = none); void set_settings(session_settings const& s); session_settings const& settings(); diff --git a/libtorrent/include/libtorrent/session_settings.hpp b/libtorrent/include/libtorrent/session_settings.hpp index fbf2c8a03..a792e296a 100644 --- a/libtorrent/include/libtorrent/session_settings.hpp +++ b/libtorrent/include/libtorrent/session_settings.hpp @@ -99,7 +99,7 @@ namespace libtorrent , allow_multiple_connections_per_ip(false) , max_failcount(3) , min_reconnect_time(60) - , peer_connect_timeout(10) + , peer_connect_timeout(7) , ignore_limits_on_local_network(true) , connection_speed(20) , send_redundant_have(false) diff --git a/libtorrent/include/libtorrent/storage.hpp b/libtorrent/include/libtorrent/storage.hpp index 67d74153d..68a81c75b 100755 --- a/libtorrent/include/libtorrent/storage.hpp +++ b/libtorrent/include/libtorrent/storage.hpp @@ -71,6 +71,13 @@ namespace libtorrent struct file_pool; struct disk_io_job; + enum storage_mode_t + { + storage_mode_allocate = 0, + storage_mode_sparse, + storage_mode_compact + }; + #if defined(_WIN32) && defined(UNICODE) TORRENT_EXPORT std::wstring safe_convert(std::string const& s); @@ -144,6 +151,10 @@ namespace libtorrent // writing. This is called when a torrent has finished // downloading. virtual void release_files() = 0; + + // this will close all open files and delete them + virtual void delete_files() = 0; + virtual ~storage_interface() {} }; @@ -155,10 +166,6 @@ namespace libtorrent boost::intrusive_ptr ti , fs::path const& path, file_pool& fp); - // returns true if the filesystem the path relies on supports - // sparse files or automatic zero filling of files. - TORRENT_EXPORT bool supports_sparse_files(fs::path const& p); - struct disk_io_thread; class TORRENT_EXPORT piece_manager @@ -180,7 +187,8 @@ namespace libtorrent ~piece_manager(); bool check_fastresume(aux::piece_checker_data& d - , std::vector& pieces, int& num_pieces, bool compact_mode); + , std::vector& pieces, int& num_pieces, storage_mode_t storage_mode + , std::string& error_msg); std::pair check_files(std::vector& pieces , int& num_pieces, boost::recursive_mutex& mutex); @@ -191,8 +199,8 @@ namespace libtorrent bool verify_resume_data(entry& rd, std::string& error); bool is_allocating() const - { return m_state == state_allocating; } - + { return m_state == state_expand_pieces; } + void mark_failed(int index); unsigned long piece_crc( @@ -200,8 +208,9 @@ namespace libtorrent , int block_size , piece_picker::block_info const* bi); - int slot_for_piece(int piece_index) const; - + int slot_for(int piece) const; + int piece_for(int slot) const; + void async_read( peer_request const& r , boost::function const& handler @@ -221,6 +230,10 @@ namespace libtorrent boost::function const& handler = boost::function()); + void async_delete_files( + boost::function const& handler + = boost::function()); + void async_move_storage(fs::path const& p , boost::function const& handler); @@ -228,10 +241,11 @@ namespace libtorrent // slots to the piece that is stored (or // partially stored) there. -2 is the index // of unassigned pieces and -1 is unallocated - void export_piece_map(std::vector& pieces) const; + void export_piece_map(std::vector& pieces + , std::vector const& have) const; bool compact_allocation() const - { return m_compact_mode; } + { return m_storage_mode == storage_mode_compact; } #ifndef NDEBUG std::string name() const { return m_info->name(); } @@ -261,9 +275,11 @@ namespace libtorrent , int offset , int size); + void switch_to_full_mode(); sha1_hash hash_for_piece_impl(int piece); - void release_files_impl(); + void release_files_impl() { m_storage->release_files(); } + void delete_files_impl() { m_storage->delete_files(); } bool move_storage_impl(fs::path const& save_path); @@ -276,19 +292,7 @@ namespace libtorrent #endif boost::scoped_ptr m_storage; - // if this is true, pieces are always allocated at the - // lowest possible slot index. If it is false, pieces - // are always written to their final place immediately - bool m_compact_mode; - - // if this is true, pieces that haven't been downloaded - // will be filled with zeroes. Not filling with zeroes - // will not work in some cases (where a seek cannot pass - // the end of the file). - bool m_fill_mode; - - // a bitmask representing the pieces we have - std::vector m_have_piece; + storage_mode_t m_storage_mode; boost::intrusive_ptr m_info; @@ -329,10 +333,21 @@ namespace libtorrent state_create_files, // checking the files state_full_check, - // allocating files (in non-compact mode) - state_allocating + // move pieces to their final position + state_expand_pieces } m_state; int m_current_slot; + // used during check. If any piece is found + // that is not in its final position, this + // is set to true + bool m_out_of_place; + // used to move pieces while expanding + // the storage from compact allocation + // to full allocation + std::vector m_scratch_buffer; + std::vector m_scratch_buffer2; + // the piece that is in the scratch buffer + int m_scratch_piece; // this is saved in case we need to instantiate a new // storage (osed when remapping files) diff --git a/libtorrent/include/libtorrent/torrent.hpp b/libtorrent/include/libtorrent/torrent.hpp index 4bb68436c..60140f2c2 100755 --- a/libtorrent/include/libtorrent/torrent.hpp +++ b/libtorrent/include/libtorrent/torrent.hpp @@ -101,7 +101,7 @@ namespace libtorrent , boost::intrusive_ptr tf , fs::path const& save_path , tcp::endpoint const& net_interface - , bool compact_mode + , storage_mode_t m_storage_mode , int block_size , storage_constructor_type sc , bool paused); @@ -116,7 +116,7 @@ namespace libtorrent , char const* name , fs::path const& save_path , tcp::endpoint const& net_interface - , bool compact_mode + , storage_mode_t m_storage_mode , int block_size , storage_constructor_type sc , bool paused); @@ -177,6 +177,8 @@ namespace libtorrent void resume(); bool is_paused() const { return m_paused; } + void delete_files(); + // ============ start deprecation ============= void filter_piece(int index, bool filter); void filter_pieces(std::vector const& bitmask); @@ -550,6 +552,7 @@ namespace libtorrent private: + void on_files_deleted(int ret, disk_io_job const& j); void on_files_released(int ret, disk_io_job const& j); void on_torrent_paused(int ret, disk_io_job const& j); void on_storage_moved(int ret, disk_io_job const& j); @@ -751,7 +754,7 @@ namespace libtorrent fs::path m_save_path; // determines the storage state for this torrent. - const bool m_compact_mode; + storage_mode_t m_storage_mode; // defaults to 16 kiB, but can be set by the user // when creating the torrent diff --git a/libtorrent/include/libtorrent/torrent_handle.hpp b/libtorrent/include/libtorrent/torrent_handle.hpp index ec2d60ea5..8ae2f7f41 100755 --- a/libtorrent/include/libtorrent/torrent_handle.hpp +++ b/libtorrent/include/libtorrent/torrent_handle.hpp @@ -52,6 +52,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/torrent_info.hpp" #include "libtorrent/time.hpp" #include "libtorrent/config.hpp" +#include "libtorrent/storage.hpp" namespace libtorrent { @@ -106,7 +107,7 @@ namespace libtorrent , num_connections(0) , uploads_limit(0) , connections_limit(0) - , compact_mode(false) + , storage_mode(storage_mode_sparse) {} enum state_t @@ -216,7 +217,7 @@ namespace libtorrent // true if the torrent is saved in compact mode // false if it is saved in full allocation mode - bool compact_mode; + storage_mode_t storage_mode; }; struct TORRENT_EXPORT block_info diff --git a/libtorrent/src/bt_peer_connection.cpp b/libtorrent/src/bt_peer_connection.cpp index 21deec1d4..0559aff95 100755 --- a/libtorrent/src/bt_peer_connection.cpp +++ b/libtorrent/src/bt_peer_connection.cpp @@ -204,7 +204,6 @@ namespace libtorrent // if this fails, we need to reconnect // fast. - pi->connected = time_now() - seconds(m_ses.settings().min_reconnect_time); fast_reconnect(true); write_pe1_2_dhkey(); @@ -802,9 +801,9 @@ namespace libtorrent { boost::shared_ptr t = associated_torrent().lock(); TORRENT_ASSERT(t); - while (!request_queue().empty()) + while (!download_queue().empty()) { - piece_block const& b = request_queue().front(); + piece_block const& b = download_queue().front(); peer_request r; r.piece = b.piece_index; r.start = b.block_index * t->block_size(); diff --git a/libtorrent/src/disk_io_thread.cpp b/libtorrent/src/disk_io_thread.cpp index b6b2e20a7..22ee12179 100644 --- a/libtorrent/src/disk_io_thread.cpp +++ b/libtorrent/src/disk_io_thread.cpp @@ -125,6 +125,7 @@ namespace libtorrent , boost::function const& f) { TORRENT_ASSERT(!j.callback); + TORRENT_ASSERT(j.storage); boost::mutex::scoped_lock l(m_mutex); std::deque::reverse_iterator i = m_jobs.rbegin(); @@ -220,6 +221,7 @@ namespace libtorrent bool free_buffer = true; try { + TORRENT_ASSERT(j.storage); #ifdef TORRENT_DISK_STATS ptime start = time_now(); #endif @@ -288,6 +290,12 @@ namespace libtorrent #endif j.storage->release_files_impl(); break; + case disk_io_job::delete_files: +#ifdef TORRENT_DISK_STATS + m_log << log_time() << " delete" << std::endl; +#endif + j.storage->delete_files_impl(); + break; } } catch (std::exception& e) diff --git a/libtorrent/src/peer_connection.cpp b/libtorrent/src/peer_connection.cpp index 9b975ab11..625b8eacd 100755 --- a/libtorrent/src/peer_connection.cpp +++ b/libtorrent/src/peer_connection.cpp @@ -394,6 +394,16 @@ namespace libtorrent #endif } + void peer_connection::fast_reconnect(bool r) + { + if (peer_info_struct() && peer_info_struct()->fast_reconnects > 1) return; + m_fast_reconnect = r; + peer_info_struct()->connected = time_now() + - seconds(m_ses.settings().min_reconnect_time + * m_ses.settings().max_failcount); + if (peer_info_struct()) ++peer_info_struct()->fast_reconnects; + } + void peer_connection::announce_piece(int index) { // dont announce during handshake @@ -643,27 +653,23 @@ namespace libtorrent m_peer_choked = true; t->get_policy().choked(*this); - if (!t->is_seed()) + if (peer_info_struct() == 0 || !peer_info_struct()->on_parole) { - piece_picker& p = t->picker(); - // remove all pieces from this peers download queue and - // remove the 'downloading' flag from piece_picker. - for (std::deque::iterator i = m_download_queue.begin(); - i != m_download_queue.end(); ++i) + // if the peer is not in parole mode, clear the queued + // up block requests + if (!t->is_seed()) { - p.abort_download(*i); - } - for (std::deque::const_iterator i = m_request_queue.begin() - , end(m_request_queue.end()); i != end; ++i) - { - // since this piece was skipped, clear it and allow it to - // be requested from other peers - p.abort_download(*i); + piece_picker& p = t->picker(); + for (std::deque::const_iterator i = m_request_queue.begin() + , end(m_request_queue.end()); i != end; ++i) + { + // since this piece was skipped, clear it and allow it to + // be requested from other peers + p.abort_download(*i); + } } + m_request_queue.clear(); } - - m_download_queue.clear(); - m_request_queue.clear(); } bool match_request(peer_request const& r, piece_block const& b, int block_size) @@ -707,23 +713,17 @@ namespace libtorrent { b = *i; m_download_queue.erase(i); - } - else - { - i = std::find_if(m_request_queue.begin(), m_request_queue.end() - , bind(match_request, boost::cref(r), _1, t->block_size())); - - if (i != m_request_queue.end()) + + // if the peer is in parole mode, keep the request + if (peer_info_struct() && peer_info_struct()->on_parole) { - b = *i; - m_request_queue.erase(i); + m_request_queue.push_front(b); + } + else if (!t->is_seed()) + { + piece_picker& p = t->picker(); + p.abort_download(b); } - } - - if (b.piece_index != -1 && !t->is_seed()) - { - piece_picker& p = t->picker(); - p.abort_download(b); } #ifdef TORRENT_VERBOSE_LOGGING else @@ -1932,7 +1932,6 @@ namespace libtorrent void peer_connection::timed_out() { - if (m_peer_info) ++m_peer_info->failcount; #if defined(TORRENT_VERBOSE_LOGGING) || defined(TORRENT_LOGGING) (*m_ses.m_logger) << "CONNECTION TIMED OUT: " << m_remote.address().to_string() << "\n"; diff --git a/libtorrent/src/piece_picker.cpp b/libtorrent/src/piece_picker.cpp index 652426806..ebfe07637 100755 --- a/libtorrent/src/piece_picker.cpp +++ b/libtorrent/src/piece_picker.cpp @@ -1732,7 +1732,6 @@ namespace libtorrent ++i->writing; info.state = block_info::state_writing; if (info.num_peers > 0) --info.num_peers; - TORRENT_ASSERT(info.num_peers >= 0); if (i->requested == 0) { @@ -1855,7 +1854,6 @@ namespace libtorrent block_info& info = i->info[block.block_index]; --info.num_peers; - TORRENT_ASSERT(info.num_peers >= 0); if (info.num_peers > 0) return; if (i->info[block.block_index].state == block_info::state_finished diff --git a/libtorrent/src/policy.cpp b/libtorrent/src/policy.cpp index dff860bd2..4de01d055 100755 --- a/libtorrent/src/policy.cpp +++ b/libtorrent/src/policy.cpp @@ -986,7 +986,8 @@ namespace libtorrent i->second.prev_amount_upload = 0; i->second.connection = &c; TORRENT_ASSERT(i->second.connection); - i->second.connected = time_now(); + if (!c.fast_reconnect()) + i->second.connected = time_now(); // m_last_optimistic_disconnect = time_now(); } @@ -1045,10 +1046,10 @@ namespace libtorrent // we don't have any info about this peer. // add a new entry - peer p(remote, peer::connectable, src); - i = m_peers.insert(std::make_pair(remote.address(), p)); + i = m_peers.insert(std::make_pair(remote.address() + , peer(remote, peer::connectable, src))); #ifndef TORRENT_DISABLE_ENCRYPTION - if (flags & 0x01) p.pe_support = true; + if (flags & 0x01) i->second.pe_support = true; #endif if (flags & 0x02) i->second.seed = true; @@ -1503,6 +1504,7 @@ namespace libtorrent , failcount(0) , hashfails(0) , seed(false) + , fast_reconnects(0) , optimistically_unchoked(false) , last_optimistically_unchoked(min_time()) , connected(min_time()) diff --git a/libtorrent/src/session.cpp b/libtorrent/src/session.cpp index 68cbd619f..5bdb6b07e 100755 --- a/libtorrent/src/session.cpp +++ b/libtorrent/src/session.cpp @@ -186,28 +186,28 @@ namespace libtorrent torrent_info const& ti , fs::path const& save_path , entry const& resume_data - , bool compact_mode + , storage_mode_t storage_mode , bool paused , storage_constructor_type sc) { TORRENT_ASSERT(!ti.m_half_metadata); boost::intrusive_ptr tip(new torrent_info(ti)); return m_impl->add_torrent(tip, save_path, resume_data - , compact_mode, sc, paused, 0); + , storage_mode, sc, paused, 0); } torrent_handle session::add_torrent( boost::intrusive_ptr ti , fs::path const& save_path , entry const& resume_data - , bool compact_mode + , storage_mode_t storage_mode , bool paused , storage_constructor_type sc , void* userdata) { TORRENT_ASSERT(!ti->m_half_metadata); return m_impl->add_torrent(ti, save_path, resume_data - , compact_mode, sc, paused, userdata); + , storage_mode, sc, paused, userdata); } torrent_handle session::add_torrent( @@ -216,18 +216,18 @@ namespace libtorrent , char const* name , fs::path const& save_path , entry const& e - , bool compact_mode + , storage_mode_t storage_mode , bool paused , storage_constructor_type sc , void* userdata) { return m_impl->add_torrent(tracker_url, info_hash, name, save_path, e - , compact_mode, sc, paused, userdata); + , storage_mode, sc, paused, userdata); } - void session::remove_torrent(const torrent_handle& h) + void session::remove_torrent(const torrent_handle& h, int options) { - m_impl->remove_torrent(h); + m_impl->remove_torrent(h, options); } bool session::listen_on( diff --git a/libtorrent/src/session_impl.cpp b/libtorrent/src/session_impl.cpp index 83498ac14..63c039010 100755 --- a/libtorrent/src/session_impl.cpp +++ b/libtorrent/src/session_impl.cpp @@ -189,9 +189,11 @@ namespace detail t->parse_resume_data(t->resume_data, t->torrent_ptr->torrent_file() , error_msg); + // lock the session to add the new torrent + session_impl::mutex_t::scoped_lock l(m_ses.m_mutex); + if (!error_msg.empty() && m_ses.m_alerts.should_post(alert::warning)) { - session_impl::mutex_t::scoped_lock l(m_ses.m_mutex); m_ses.m_alerts.post_alert(fastresume_rejected_alert( t->torrent_ptr->get_handle() , error_msg)); @@ -202,8 +204,6 @@ namespace detail #endif } - // lock the session to add the new torrent - session_impl::mutex_t::scoped_lock l(m_ses.m_mutex); mutex::scoped_lock l2(m_mutex); if (m_torrents.empty() || m_torrents.front() != t) @@ -328,7 +328,7 @@ namespace detail boost::tie(finished, progress) = processing->torrent_ptr->check_files(); { - mutex::scoped_lock l(m_mutex); + mutex::scoped_lock l2(m_mutex); INVARIANT_CHECK; @@ -340,9 +340,9 @@ namespace detail m_processing.pop_front(); // make sure the lock order is correct - l.unlock(); + l2.unlock(); session_impl::mutex_t::scoped_lock l(m_ses.m_mutex); - l.lock(); + l2.lock(); processing->torrent_ptr->abort(); processing.reset(); @@ -481,7 +481,7 @@ namespace detail return 0; } - void checker_impl::remove_torrent(sha1_hash const& info_hash) + void checker_impl::remove_torrent(sha1_hash const& info_hash, int options) { INVARIANT_CHECK; for (std::deque >::iterator i @@ -490,6 +490,8 @@ namespace detail if ((*i)->info_hash == info_hash) { TORRENT_ASSERT((*i)->processing == false); + if (options & session::delete_files) + (*i)->torrent_ptr->delete_files(); m_torrents.erase(i); return; } @@ -500,6 +502,8 @@ namespace detail if ((*i)->info_hash == info_hash) { TORRENT_ASSERT((*i)->processing == false); + if (options & session::delete_files) + (*i)->torrent_ptr->delete_files(); m_processing.erase(i); return; } @@ -565,8 +569,19 @@ namespace detail , m_checker_impl(*this) { #ifdef WIN32 - // windows XP has a limit of 10 simultaneous connections - m_half_open.limit(8); + // windows XP has a limit on the number of + // simultaneous half-open TCP connections + DWORD windows_version = ::GetVersion(); + if ((windows_version & 0xff) >= 6) + { + // on vista the limit is 5 (in home edition) + m_half_open.limit(4); + } + else + { + // on XP SP2 it's 10 + m_half_open.limit(8); + } #endif m_bandwidth_manager[peer_connection::download_channel] = &m_download_channel; @@ -1623,7 +1638,7 @@ namespace detail boost::intrusive_ptr ti , fs::path const& save_path , entry const& resume_data - , bool compact_mode + , storage_mode_t storage_mode , storage_constructor_type sc , bool paused , void* userdata) @@ -1655,7 +1670,7 @@ namespace detail // the thread boost::shared_ptr torrent_ptr( new torrent(*this, m_checker_impl, ti, save_path - , m_listen_interface, compact_mode, 16 * 1024 + , m_listen_interface, storage_mode, 16 * 1024 , sc, paused)); torrent_ptr->start(); @@ -1701,7 +1716,7 @@ namespace detail , char const* name , fs::path const& save_path , entry const& - , bool compact_mode + , storage_mode_t storage_mode , storage_constructor_type sc , bool paused , void* userdata) @@ -1735,7 +1750,7 @@ namespace detail // the thread boost::shared_ptr torrent_ptr( new torrent(*this, m_checker_impl, tracker_url, info_hash, name - , save_path, m_listen_interface, compact_mode, 16 * 1024 + , save_path, m_listen_interface, storage_mode, 16 * 1024 , sc, paused)); torrent_ptr->start(); @@ -1754,7 +1769,7 @@ namespace detail return torrent_handle(this, &m_checker_impl, info_hash); } - void session_impl::remove_torrent(const torrent_handle& h) + void session_impl::remove_torrent(const torrent_handle& h, int options) { if (h.m_ses != this) return; TORRENT_ASSERT(h.m_chk == &m_checker_impl || h.m_chk == 0); @@ -1769,6 +1784,8 @@ namespace detail if (i != m_torrents.end()) { torrent& t = *i->second; + if (options & session::delete_files) + t.delete_files(); t.abort(); if ((!t.is_paused() || t.should_request()) @@ -1815,7 +1832,7 @@ namespace detail if (d != 0) { if (d->processing) d->abort = true; - else m_checker_impl.remove_torrent(h.m_info_hash); + else m_checker_impl.remove_torrent(h.m_info_hash, options); return; } } diff --git a/libtorrent/src/storage.cpp b/libtorrent/src/storage.cpp index 1cac99e44..6671e38e9 100755 --- a/libtorrent/src/storage.cpp +++ b/libtorrent/src/storage.cpp @@ -361,6 +361,7 @@ namespace libtorrent } void release_files(); + void delete_files(); void initialize(bool allocate_files); bool move_storage(fs::path save_path); size_type read(char* buf, int slot, int offset, int size); @@ -447,7 +448,7 @@ namespace libtorrent } // if the file is empty, just create it. But also make sure - // the directory exits. + // the directory exists. if (file_iter->size == 0) { file(m_save_path / file_iter->path, file::out); @@ -470,6 +471,38 @@ namespace libtorrent std::vector().swap(m_scratch_buffer); } + void storage::delete_files() + { + // make sure we don't have the files open + m_files.release(this); + std::vector().swap(m_scratch_buffer); + + // delete the files from disk + std::set directories; + typedef std::set::iterator iter_t; + for (torrent_info::file_iterator i = m_info->begin_files(true) + , end(m_info->end_files(true)); i != end; ++i) + { + std::string p = (m_save_path / i->path).string(); + fs::path bp = i->path.branch_path(); + std::pair ret = directories.insert(bp.string()); + while (ret.second && !bp.empty()) + { + bp = bp.branch_path(); + std::pair ret = directories.insert(bp.string()); + } + std::remove(p.c_str()); + } + + // remove the directories. Reverse order to delete + // subdirectories first + std::for_each(directories.rbegin(), directories.rend() + , bind((int(*)(char const*))&std::remove, bind(&std::string::c_str, _1))); + + std::string p = (m_save_path / m_info->name()).string(); + std::remove(p.c_str()); + } + void storage::write_resume_data(entry& rd) const { std::vector > file_sizes @@ -931,107 +964,6 @@ namespace libtorrent return new storage(ti, path, fp); } - bool supports_sparse_files(fs::path const& p) - { - TORRENT_ASSERT(p.is_complete()); -#if defined(_WIN32) - // assume windows API is available - DWORD max_component_len = 0; - DWORD volume_flags = 0; - std::string root_device = p.root_name() + "\\"; -#if defined(UNICODE) - std::wstring wph(safe_convert(root_device)); - bool ret = ::GetVolumeInformation(wph.c_str(), 0 - , 0, 0, &max_component_len, &volume_flags, 0, 0); -#else - bool ret = ::GetVolumeInformation(root_device.c_str(), 0 - , 0, 0, &max_component_len, &volume_flags, 0, 0); -#endif - - if (!ret) return false; - if (volume_flags & FILE_SUPPORTS_SPARSE_FILES) - return true; -#endif - -#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) - // find the last existing directory of the save path - fs::path query_path = p; - while (!query_path.empty() && !exists(query_path)) - query_path = query_path.branch_path(); -#endif - -#if defined(__APPLE__) - - struct statfs fsinfo; - int ret = statfs(query_path.native_directory_string().c_str(), &fsinfo); - if (ret != 0) return false; - - attrlist request; - request.bitmapcount = ATTR_BIT_MAP_COUNT; - request.reserved = 0; - request.commonattr = 0; - request.volattr = ATTR_VOL_CAPABILITIES; - request.dirattr = 0; - request.fileattr = 0; - request.forkattr = 0; - - struct vol_capabilities_attr_buf - { - unsigned long length; - vol_capabilities_attr_t info; - } vol_cap; - - ret = getattrlist(fsinfo.f_mntonname, &request, &vol_cap - , sizeof(vol_cap), 0); - if (ret != 0) return false; - - if (vol_cap.info.capabilities[VOL_CAPABILITIES_FORMAT] - & (VOL_CAP_FMT_SPARSE_FILES | VOL_CAP_FMT_ZERO_RUNS)) - { - return true; - } - - // workaround for bugs in Mac OS X where zero run is not reported - if (!strcmp(fsinfo.f_fstypename, "hfs") - || !strcmp(fsinfo.f_fstypename, "ufs")) - return true; - - return false; -#endif - -#if defined(__linux__) || defined(__FreeBSD__) - struct statfs buf; - int err = statfs(query_path.native_directory_string().c_str(), &buf); - if (err == 0) - { - switch (buf.f_type) - { - case 0x5346544e: // NTFS - case 0xEF51: // EXT2 OLD - case 0xEF53: // EXT2 and EXT3 - case 0x00011954: // UFS - case 0x52654973: // ReiserFS - case 0x52345362: // Reiser4 - case 0x58465342: // XFS - case 0x65735546: // NTFS-3G - case 0x19540119: // UFS2 - return true; - } - } -#ifndef NDEBUG - else - { - std::cerr << "statfs returned " << err << std::endl; - std::cerr << "errno: " << errno << std::endl; - std::cerr << "path: " << query_path.native_directory_string() << std::endl; - } -#endif -#endif - - // TODO: POSIX implementation - return false; - } - // -- piece_manager ----------------------------------------------------- piece_manager::piece_manager( @@ -1042,15 +974,16 @@ namespace libtorrent , disk_io_thread& io , storage_constructor_type sc) : m_storage(sc(ti, save_path, fp)) - , m_compact_mode(false) - , m_fill_mode(true) + , m_storage_mode(storage_mode_sparse) , m_info(ti) , m_save_path(complete(save_path)) + , m_current_slot(0) + , m_out_of_place(false) + , m_scratch_piece(-1) , m_storage_constructor(sc) , m_io_thread(io) , m_torrent(torrent) { - m_fill_mode = !supports_sparse_files(save_path); } piece_manager::~piece_manager() @@ -1081,6 +1014,15 @@ namespace libtorrent m_io_thread.add_job(j, handler); } + void piece_manager::async_delete_files( + boost::function const& handler) + { + disk_io_job j; + j.storage = this; + j.action = disk_io_job::delete_files; + m_io_thread.add_job(j, handler); + } + void piece_manager::async_move_storage(fs::path const& p , boost::function const& handler) { @@ -1158,16 +1100,11 @@ namespace libtorrent m_piece_hasher.erase(i); } - int slot = m_piece_to_slot[piece]; + int slot = slot_for(piece); TORRENT_ASSERT(slot != has_no_slot); return m_storage->hash_for_slot(slot, ph, m_info->piece_size(piece)); } - void piece_manager::release_files_impl() - { - m_storage->release_files(); - } - bool piece_manager::move_storage_impl(fs::path const& save_path) { if (m_storage->move_storage(save_path)) @@ -1177,26 +1114,39 @@ namespace libtorrent } return false; } + void piece_manager::export_piece_map( - std::vector& p) const + std::vector& p, std::vector const& have) const { boost::recursive_mutex::scoped_lock lock(m_mutex); INVARIANT_CHECK; - p.clear(); - std::vector::const_reverse_iterator last; - for (last = m_slot_to_piece.rbegin(); - last != m_slot_to_piece.rend(); ++last) + if (m_storage_mode == storage_mode_compact) { - if (*last != unallocated) break; - } + p.clear(); + p.reserve(m_info->num_pieces()); + std::vector::const_reverse_iterator last; + for (last = m_slot_to_piece.rbegin(); + last != m_slot_to_piece.rend(); ++last) + { + if (*last != unallocated && have[*last]) break; + } - for (std::vector::const_iterator i = - m_slot_to_piece.begin(); - i != last.base(); ++i) + for (std::vector::const_iterator i = + m_slot_to_piece.begin(); + i != last.base(); ++i) + { + p.push_back(have[*i] ? *i : unassigned); + } + } + else { - p.push_back(*i); + p.reserve(m_info->num_pieces()); + for (int i = 0; i < m_info->num_pieces(); ++i) + { + p.push_back(have[i] ? i : unassigned); + } } } @@ -1206,11 +1156,10 @@ namespace libtorrent INVARIANT_CHECK; + if (m_storage_mode != storage_mode_compact) return; + TORRENT_ASSERT(piece_index >= 0 && piece_index < (int)m_piece_to_slot.size()); - TORRENT_ASSERT(m_piece_to_slot[piece_index] >= 0); - int slot_index = m_piece_to_slot[piece_index]; - TORRENT_ASSERT(slot_index >= 0); m_slot_to_piece[slot_index] = unassigned; @@ -1218,12 +1167,6 @@ namespace libtorrent m_free_slots.push_back(slot_index); } - int piece_manager::slot_for_piece(int piece_index) const - { - TORRENT_ASSERT(piece_index >= 0 && piece_index < m_info->num_pieces()); - return m_piece_to_slot[piece_index]; - } - unsigned long piece_manager::piece_crc( int slot_index , int block_size @@ -1275,11 +1218,7 @@ namespace libtorrent TORRENT_ASSERT(buf); TORRENT_ASSERT(offset >= 0); TORRENT_ASSERT(size > 0); - TORRENT_ASSERT(piece_index >= 0 && piece_index < (int)m_piece_to_slot.size()); - TORRENT_ASSERT(m_piece_to_slot[piece_index] >= 0 - && m_piece_to_slot[piece_index] < (int)m_slot_to_piece.size()); - int slot = m_piece_to_slot[piece_index]; - TORRENT_ASSERT(slot >= 0 && slot < (int)m_slot_to_piece.size()); + int slot = slot_for(piece_index); return m_storage->read(buf, slot, offset, size); } @@ -1292,7 +1231,7 @@ namespace libtorrent TORRENT_ASSERT(buf); TORRENT_ASSERT(offset >= 0); TORRENT_ASSERT(size > 0); - TORRENT_ASSERT(piece_index >= 0 && piece_index < (int)m_piece_to_slot.size()); + TORRENT_ASSERT(piece_index >= 0 && piece_index < m_info->num_pieces()); if (offset == 0) { @@ -1317,7 +1256,6 @@ namespace libtorrent } int slot = allocate_slot_for_piece(piece_index); - TORRENT_ASSERT(slot >= 0 && slot < (int)m_slot_to_piece.size()); m_storage->write(buf, slot, offset, size); } @@ -1426,7 +1364,8 @@ namespace libtorrent // that piece as unassigned, since this slot // is the correct place for the piece. m_slot_to_piece[other_slot] = unassigned; - m_free_slots.push_back(other_slot); + if (m_storage_mode == storage_mode_compact) + m_free_slots.push_back(other_slot); } TORRENT_ASSERT(m_piece_to_slot[piece_index] != current_slot); TORRENT_ASSERT(m_piece_to_slot[piece_index] >= 0); @@ -1485,7 +1424,8 @@ namespace libtorrent bool piece_manager::check_fastresume( aux::piece_checker_data& data , std::vector& pieces - , int& num_pieces, bool compact_mode) + , int& num_pieces, storage_mode_t storage_mode + , std::string& error_msg) { boost::recursive_mutex::scoped_lock lock(m_mutex); @@ -1493,7 +1433,7 @@ namespace libtorrent TORRENT_ASSERT(m_info->piece_length() > 0); - m_compact_mode = compact_mode; + m_storage_mode = storage_mode; // This will corrupt the storage // use while debugging to find @@ -1503,9 +1443,13 @@ namespace libtorrent m_piece_to_slot.resize(m_info->num_pieces(), has_no_slot); m_slot_to_piece.resize(m_info->num_pieces(), unallocated); - m_free_slots.clear(); - m_unallocated_slots.clear(); + TORRENT_ASSERT(m_free_slots.empty()); + TORRENT_ASSERT(m_unallocated_slots.empty()); + // assume no piece is out of place (i.e. in a slot + // other than the one it should be in) + bool out_of_place = false; + pieces.clear(); pieces.resize(m_info->num_pieces(), false); num_pieces = 0; @@ -1513,13 +1457,14 @@ namespace libtorrent // if we have fast-resume info // use it instead of doing the actual checking if (!data.piece_map.empty() - && data.piece_map.size() <= m_slot_to_piece.size()) + && int(data.piece_map.size()) <= m_info->num_pieces()) { for (int i = 0; i < (int)data.piece_map.size(); ++i) { m_slot_to_piece[i] = data.piece_map[i]; if (data.piece_map[i] >= 0) { + if (data.piece_map[i] != i) out_of_place = true; m_piece_to_slot[data.piece_map[i]] = i; int found_piece = data.piece_map[i]; @@ -1537,27 +1482,54 @@ namespace libtorrent } else if (data.piece_map[i] == unassigned) { - m_free_slots.push_back(i); + if (m_storage_mode == storage_mode_compact) + m_free_slots.push_back(i); } else { TORRENT_ASSERT(data.piece_map[i] == unallocated); - m_unallocated_slots.push_back(i); + if (m_storage_mode == storage_mode_compact) + m_unallocated_slots.push_back(i); } } - m_unallocated_slots.reserve(int(pieces.size() - data.piece_map.size())); - for (int i = (int)data.piece_map.size(); i < (int)pieces.size(); ++i) + if (m_storage_mode == storage_mode_compact) { - m_unallocated_slots.push_back(i); + m_unallocated_slots.reserve(int(m_info->num_pieces() - data.piece_map.size())); + for (int i = (int)data.piece_map.size(); i < (int)m_info->num_pieces(); ++i) + { + m_unallocated_slots.push_back(i); + } + if (m_unallocated_slots.empty()) + { + switch_to_full_mode(); + } + } + else + { + if (!out_of_place) + { + // if no piece is out of place + // since we're in full allocation mode, we can + // forget the piece allocation tables + + std::vector().swap(m_piece_to_slot); + std::vector().swap(m_slot_to_piece); + m_state = state_create_files; + return false; + } + else + { + // in this case we're in full allocation mode, but + // we're resuming a compact allocated storage + m_state = state_expand_pieces; + m_current_slot = 0; + error_msg = "pieces needs to be reordered"; + return false; + } } - if (m_unallocated_slots.empty()) - m_state = state_create_files; - else if (m_compact_mode) - m_state = state_create_files; - else - m_state = state_allocating; + m_state = state_create_files; return false; } @@ -1572,18 +1544,13 @@ namespace libtorrent | | | v - | +------------+ - | | full_check | - | +------------+ - | | - | v - | +------------+ - |->| allocating | - | +------------+ - | | - | v - | +--------------+ - |->| create_files | + | +------------+ +---------------+ + | | full_check |-->| expand_pieses | + | +------------+ +---------------+ + | | | + | v | + | +--------------+ | + +->| create_files | <------+ +--------------+ | v @@ -1602,67 +1569,97 @@ namespace libtorrent std::pair piece_manager::check_files( std::vector& pieces, int& num_pieces, boost::recursive_mutex& mutex) { +#ifndef NDEBUG + boost::recursive_mutex::scoped_lock l_(mutex); TORRENT_ASSERT(num_pieces == std::count(pieces.begin(), pieces.end(), true)); - - if (m_state == state_allocating) - { - if (m_compact_mode || m_unallocated_slots.empty()) - { - m_state = state_create_files; - return std::make_pair(false, 1.f); - } - - if (int(m_unallocated_slots.size()) == m_info->num_pieces() - && !m_fill_mode) - { - // if there is not a single file on disk, just - // create the files - m_state = state_create_files; - return std::make_pair(false, 1.f); - } - - // if we're not in compact mode, make sure the - // pieces are spread out and placed at their - // final position. - TORRENT_ASSERT(!m_unallocated_slots.empty()); - - if (!m_fill_mode) - { - // if we're not filling the allocation - // just make sure we move the current pieces - // into place, and just skip all other - // allocation - // allocate_slots returns true if it had to - // move any data - allocate_slots(m_unallocated_slots.size(), true); - } - else - { - allocate_slots(1); - } - - return std::make_pair(false, 1.f - (float)m_unallocated_slots.size() - / (float)m_slot_to_piece.size()); - } + l_.unlock(); +#endif if (m_state == state_create_files) { - m_storage->initialize(!m_fill_mode && !m_compact_mode); - - if (!m_unallocated_slots.empty() && !m_compact_mode) - { - TORRENT_ASSERT(!m_fill_mode); - std::vector().swap(m_unallocated_slots); - std::fill(m_slot_to_piece.begin(), m_slot_to_piece.end(), int(unassigned)); - m_free_slots.resize(m_info->num_pieces()); - for (int i = 0; i < m_info->num_pieces(); ++i) - m_free_slots[i] = i; - } - + m_storage->initialize(m_storage_mode == storage_mode_allocate); m_state = state_finished; return std::make_pair(true, 1.f); } + if (m_state == state_expand_pieces) + { + INVARIANT_CHECK; + + if (m_scratch_piece >= 0) + { + int piece = m_scratch_piece; + int other_piece = m_slot_to_piece[piece]; + m_scratch_piece = -1; + + if (other_piece >= 0) + { + if (m_scratch_buffer2.empty()) + m_scratch_buffer2.resize(m_info->piece_length()); + + m_storage->read(&m_scratch_buffer2[0], piece, 0, m_info->piece_size(other_piece)); + m_scratch_piece = other_piece; + m_piece_to_slot[other_piece] = unassigned; + } + + // the slot where this piece belongs is + // free. Just move the piece there. + m_storage->write(&m_scratch_buffer[0], piece, 0, m_info->piece_size(piece)); + m_piece_to_slot[piece] = piece; + m_slot_to_piece[piece] = piece; + + if (other_piece >= 0) + m_scratch_buffer.swap(m_scratch_buffer2); + + return std::make_pair(false, (float)m_current_slot / m_info->num_pieces()); + } + + while (m_current_slot < m_info->num_pieces() + && (m_slot_to_piece[m_current_slot] == m_current_slot + || m_slot_to_piece[m_current_slot] < 0)) + { + ++m_current_slot; + } + + if (m_current_slot == m_info->num_pieces()) + { + m_state = state_create_files; + std::vector().swap(m_scratch_buffer); + std::vector().swap(m_scratch_buffer2); + if (m_storage_mode != storage_mode_compact) + { + std::vector().swap(m_piece_to_slot); + std::vector().swap(m_slot_to_piece); + } + return std::make_pair(false, 1.f); + } + + int piece = m_slot_to_piece[m_current_slot]; + TORRENT_ASSERT(piece >= 0); + int other_piece = m_slot_to_piece[piece]; + if (other_piece >= 0) + { + // there is another piece in the slot + // where this one goes. Store it in the scratch + // buffer until next iteration. + if (m_scratch_buffer.empty()) + m_scratch_buffer.resize(m_info->piece_length()); + + m_storage->read(&m_scratch_buffer[0], piece, 0, m_info->piece_size(other_piece)); + m_scratch_piece = other_piece; + m_piece_to_slot[other_piece] = unassigned; + } + + // the slot where this piece belongs is + // free. Just move the piece there. + m_storage->move_slot(m_current_slot, piece); + m_piece_to_slot[piece] = piece; + m_slot_to_piece[m_current_slot] = unassigned; + m_slot_to_piece[piece] = piece; + + return std::make_pair(false, (float)m_current_slot / m_info->num_pieces()); + } + TORRENT_ASSERT(m_state == state_full_check); // ------------------------ @@ -1674,12 +1671,13 @@ namespace libtorrent // initialization for the full check if (m_hash_to_piece.empty()) { - m_current_slot = 0; for (int i = 0; i < m_info->num_pieces(); ++i) { m_hash_to_piece.insert(std::make_pair(m_info->hash_for_piece(i), i)); } + boost::recursive_mutex::scoped_lock l(mutex); std::fill(pieces.begin(), pieces.end(), false); + num_pieces = 0; } m_piece_data.resize(int(m_info->piece_length())); @@ -1694,6 +1692,10 @@ namespace libtorrent int piece_index = identify_data(m_piece_data, m_current_slot , pieces, num_pieces, m_hash_to_piece, mutex); + if (piece_index != m_current_slot + && piece_index >= 0) + m_out_of_place = true; + TORRENT_ASSERT(num_pieces == std::count(pieces.begin(), pieces.end(), true)); TORRENT_ASSERT(piece_index == unassigned || piece_index >= 0); @@ -1745,8 +1747,11 @@ namespace libtorrent std::vector::iterator i = std::find(m_free_slots.begin(), m_free_slots.end(), other_slot); TORRENT_ASSERT(i != m_free_slots.end()); - m_free_slots.erase(i); - m_free_slots.push_back(m_current_slot); + if (m_storage_mode == storage_mode_compact) + { + m_free_slots.erase(i); + m_free_slots.push_back(m_current_slot); + } } if (other_piece >= 0) @@ -1770,7 +1775,8 @@ namespace libtorrent m_slot_to_piece[other_slot] = piece_index; m_piece_to_slot[other_piece] = m_current_slot; - if (piece_index == unassigned) + if (piece_index == unassigned + && m_storage_mode == storage_mode_compact) m_free_slots.push_back(other_slot); if (piece_index >= 0) @@ -1845,8 +1851,11 @@ namespace libtorrent std::vector::iterator i = std::find(m_free_slots.begin(), m_free_slots.end(), slot1); TORRENT_ASSERT(i != m_free_slots.end()); - m_free_slots.erase(i); - m_free_slots.push_back(slot2); + if (m_storage_mode == storage_mode_compact) + { + m_free_slots.erase(i); + m_free_slots.push_back(slot2); + } } if (piece1 >= 0) @@ -1873,7 +1882,7 @@ namespace libtorrent // the slot was identified as piece 'piece_index' if (piece_index != unassigned) m_piece_to_slot[piece_index] = m_current_slot; - else + else if (m_storage_mode == storage_mode_compact) m_free_slots.push_back(m_current_slot); m_slot_to_piece[m_current_slot] = piece_index; @@ -1899,10 +1908,13 @@ namespace libtorrent (file_offset - current_offset + m_info->piece_length() - 1) / m_info->piece_length()); - for (int i = m_current_slot; i < m_current_slot + skip_blocks; ++i) + if (m_storage_mode == storage_mode_compact) { - TORRENT_ASSERT(m_slot_to_piece[i] == unallocated); - m_unallocated_slots.push_back(i); + for (int i = m_current_slot; i < m_current_slot + skip_blocks; ++i) + { + TORRENT_ASSERT(m_slot_to_piece[i] == unallocated); + m_unallocated_slots.push_back(i); + } } // current slot will increase by one at the end of the for-loop too @@ -1910,15 +1922,46 @@ namespace libtorrent } ++m_current_slot; - if (m_current_slot >= m_info->num_pieces()) + if (m_current_slot >= m_info->num_pieces()) { TORRENT_ASSERT(m_current_slot == m_info->num_pieces()); // clear the memory we've been using std::vector().swap(m_piece_data); std::multimap().swap(m_hash_to_piece); - m_state = state_allocating; + + if (m_storage_mode != storage_mode_compact) + { + if (!m_out_of_place) + { + // if no piece is out of place + // since we're in full allocation mode, we can + // forget the piece allocation tables + + std::vector().swap(m_piece_to_slot); + std::vector().swap(m_slot_to_piece); + m_state = state_create_files; + return std::make_pair(false, 1.f); + } + else + { + // in this case we're in full allocation mode, but + // we're resuming a compact allocated storage + m_state = state_expand_pieces; + m_current_slot = 0; + return std::make_pair(false, 0.f); + } + } + else if (m_unallocated_slots.empty()) + { + switch_to_full_mode(); + } + m_state = state_create_files; + +#ifndef NDEBUG + boost::recursive_mutex::scoped_lock l(mutex); TORRENT_ASSERT(num_pieces == std::count(pieces.begin(), pieces.end(), true)); +#endif return std::make_pair(false, 1.f); } @@ -1927,10 +1970,26 @@ namespace libtorrent return std::make_pair(false, (float)m_current_slot / m_info->num_pieces()); } + void piece_manager::switch_to_full_mode() + { + TORRENT_ASSERT(m_storage_mode == storage_mode_compact); + TORRENT_ASSERT(m_unallocated_slots.empty()); + // we have allocated all slots, switch to + // full allocation mode in order to free + // some unnecessary memory. + m_storage_mode = storage_mode_sparse; + std::vector().swap(m_unallocated_slots); + std::vector().swap(m_free_slots); + std::vector().swap(m_piece_to_slot); + std::vector().swap(m_slot_to_piece); + } + int piece_manager::allocate_slot_for_piece(int piece_index) { boost::recursive_mutex::scoped_lock lock(m_mutex); + if (m_storage_mode != storage_mode_compact) return piece_index; + // INVARIANT_CHECK; TORRENT_ASSERT(piece_index >= 0); @@ -2030,26 +2089,27 @@ namespace libtorrent debug_log(); #endif } - TORRENT_ASSERT(slot_index >= 0); TORRENT_ASSERT(slot_index < (int)m_slot_to_piece.size()); + + if (m_unallocated_slots.empty()) + { + switch_to_full_mode(); + } + return slot_index; } bool piece_manager::allocate_slots(int num_slots, bool abort_on_disk) { - TORRENT_ASSERT(num_slots > 0); - boost::recursive_mutex::scoped_lock lock(m_mutex); + TORRENT_ASSERT(num_slots > 0); // INVARIANT_CHECK; TORRENT_ASSERT(!m_unallocated_slots.empty()); + TORRENT_ASSERT(m_storage_mode == storage_mode_compact); - const int stack_buffer_size = 16*1024; - char zeroes[stack_buffer_size]; - memset(zeroes, 0, stack_buffer_size); - bool written = false; for (int i = 0; i < num_slots && !m_unallocated_slots.empty(); ++i) @@ -2069,134 +2129,160 @@ namespace libtorrent m_piece_to_slot[pos] = pos; written = true; } - else if (m_fill_mode) - { - int piece_size = int(m_info->piece_size(pos)); - int offset = 0; - for (; piece_size > 0; piece_size -= stack_buffer_size - , offset += stack_buffer_size) - { - m_storage->write(zeroes, pos, offset - , (std::min)(piece_size, stack_buffer_size)); - } - written = true; - } m_unallocated_slots.erase(m_unallocated_slots.begin()); m_slot_to_piece[new_free_slot] = unassigned; m_free_slots.push_back(new_free_slot); - if (abort_on_disk && written) return true; + if (abort_on_disk && written) break; } TORRENT_ASSERT(m_free_slots.size() > 0); return written; } + int piece_manager::slot_for(int piece) const + { + if (m_storage_mode != storage_mode_compact) return piece; + TORRENT_ASSERT(piece < int(m_piece_to_slot.size())); + TORRENT_ASSERT(piece >= 0); + return m_piece_to_slot[piece]; + } + + int piece_manager::piece_for(int slot) const + { + if (m_storage_mode != storage_mode_compact) return slot; + TORRENT_ASSERT(slot < int(m_slot_to_piece.size())); + TORRENT_ASSERT(slot >= 0); + return m_slot_to_piece[slot]; + } + #ifndef NDEBUG void piece_manager::check_invariant() const { boost::recursive_mutex::scoped_lock lock(m_mutex); - if (m_piece_to_slot.empty()) return; - TORRENT_ASSERT((int)m_piece_to_slot.size() == m_info->num_pieces()); - TORRENT_ASSERT((int)m_slot_to_piece.size() == m_info->num_pieces()); - - for (std::vector::const_iterator i = m_free_slots.begin(); - i != m_free_slots.end(); ++i) + if (m_unallocated_slots.empty() && m_state == state_finished) { - TORRENT_ASSERT(*i < (int)m_slot_to_piece.size()); - TORRENT_ASSERT(*i >= 0); - TORRENT_ASSERT(m_slot_to_piece[*i] == unassigned); - TORRENT_ASSERT(std::find(i+1, m_free_slots.end(), *i) - == m_free_slots.end()); + TORRENT_ASSERT(m_storage_mode != storage_mode_compact); } - - for (std::vector::const_iterator i = m_unallocated_slots.begin(); - i != m_unallocated_slots.end(); ++i) + + if (m_storage_mode != storage_mode_compact) { - TORRENT_ASSERT(*i < (int)m_slot_to_piece.size()); - TORRENT_ASSERT(*i >= 0); - TORRENT_ASSERT(m_slot_to_piece[*i] == unallocated); - TORRENT_ASSERT(std::find(i+1, m_unallocated_slots.end(), *i) - == m_unallocated_slots.end()); + TORRENT_ASSERT(m_unallocated_slots.empty()); + TORRENT_ASSERT(m_free_slots.empty()); } - - for (int i = 0; i < m_info->num_pieces(); ++i) + + if (m_storage_mode != storage_mode_compact + && m_state != state_expand_pieces + && m_state != state_full_check) { - // Check domain of piece_to_slot's elements - if (m_piece_to_slot[i] != has_no_slot) + TORRENT_ASSERT(m_piece_to_slot.empty()); + TORRENT_ASSERT(m_slot_to_piece.empty()); + } + else + { + if (m_piece_to_slot.empty()) return; + + TORRENT_ASSERT((int)m_piece_to_slot.size() == m_info->num_pieces()); + TORRENT_ASSERT((int)m_slot_to_piece.size() == m_info->num_pieces()); + + for (std::vector::const_iterator i = m_free_slots.begin(); + i != m_free_slots.end(); ++i) { - TORRENT_ASSERT(m_piece_to_slot[i] >= 0); - TORRENT_ASSERT(m_piece_to_slot[i] < (int)m_slot_to_piece.size()); + TORRENT_ASSERT(*i < (int)m_slot_to_piece.size()); + TORRENT_ASSERT(*i >= 0); + TORRENT_ASSERT(m_slot_to_piece[*i] == unassigned); + TORRENT_ASSERT(std::find(i+1, m_free_slots.end(), *i) + == m_free_slots.end()); } - // Check domain of slot_to_piece's elements - if (m_slot_to_piece[i] != unallocated - && m_slot_to_piece[i] != unassigned) + for (std::vector::const_iterator i = m_unallocated_slots.begin(); + i != m_unallocated_slots.end(); ++i) { - TORRENT_ASSERT(m_slot_to_piece[i] >= 0); - TORRENT_ASSERT(m_slot_to_piece[i] < (int)m_piece_to_slot.size()); + TORRENT_ASSERT(*i < (int)m_slot_to_piece.size()); + TORRENT_ASSERT(*i >= 0); + TORRENT_ASSERT(m_slot_to_piece[*i] == unallocated); + TORRENT_ASSERT(std::find(i+1, m_unallocated_slots.end(), *i) + == m_unallocated_slots.end()); } - // do more detailed checks on piece_to_slot - if (m_piece_to_slot[i] >= 0) + for (int i = 0; i < m_info->num_pieces(); ++i) { - TORRENT_ASSERT(m_slot_to_piece[m_piece_to_slot[i]] == i); - if (m_piece_to_slot[i] != i) + // Check domain of piece_to_slot's elements + if (m_piece_to_slot[i] != has_no_slot) { - TORRENT_ASSERT(m_slot_to_piece[i] == unallocated); + TORRENT_ASSERT(m_piece_to_slot[i] >= 0); + TORRENT_ASSERT(m_piece_to_slot[i] < (int)m_slot_to_piece.size()); } - } - else - { - TORRENT_ASSERT(m_piece_to_slot[i] == has_no_slot); - } - // do more detailed checks on slot_to_piece + // Check domain of slot_to_piece's elements + if (m_slot_to_piece[i] != unallocated + && m_slot_to_piece[i] != unassigned) + { + TORRENT_ASSERT(m_slot_to_piece[i] >= 0); + TORRENT_ASSERT(m_slot_to_piece[i] < (int)m_piece_to_slot.size()); + } - if (m_slot_to_piece[i] >= 0) - { - TORRENT_ASSERT(m_slot_to_piece[i] < (int)m_piece_to_slot.size()); - TORRENT_ASSERT(m_piece_to_slot[m_slot_to_piece[i]] == i); + // do more detailed checks on piece_to_slot + if (m_piece_to_slot[i] >= 0) + { + TORRENT_ASSERT(m_slot_to_piece[m_piece_to_slot[i]] == i); + if (m_piece_to_slot[i] != i) + { + TORRENT_ASSERT(m_slot_to_piece[i] == unallocated); + } + } + else + { + TORRENT_ASSERT(m_piece_to_slot[i] == has_no_slot); + } + + // do more detailed checks on slot_to_piece + + if (m_slot_to_piece[i] >= 0) + { + TORRENT_ASSERT(m_slot_to_piece[i] < (int)m_piece_to_slot.size()); + TORRENT_ASSERT(m_piece_to_slot[m_slot_to_piece[i]] == i); #ifdef TORRENT_STORAGE_DEBUG - TORRENT_ASSERT( - std::find( - m_unallocated_slots.begin() - , m_unallocated_slots.end() - , i) == m_unallocated_slots.end() - ); - TORRENT_ASSERT( - std::find( - m_free_slots.begin() - , m_free_slots.end() - , i) == m_free_slots.end() - ); + TORRENT_ASSERT( + std::find( + m_unallocated_slots.begin() + , m_unallocated_slots.end() + , i) == m_unallocated_slots.end() + ); + TORRENT_ASSERT( + std::find( + m_free_slots.begin() + , m_free_slots.end() + , i) == m_free_slots.end() + ); #endif - } - else if (m_slot_to_piece[i] == unallocated) - { + } + else if (m_slot_to_piece[i] == unallocated) + { #ifdef TORRENT_STORAGE_DEBUG - TORRENT_ASSERT(m_unallocated_slots.empty() - || (std::find( - m_unallocated_slots.begin() - , m_unallocated_slots.end() - , i) != m_unallocated_slots.end()) - ); + TORRENT_ASSERT(m_unallocated_slots.empty() + || (std::find( + m_unallocated_slots.begin() + , m_unallocated_slots.end() + , i) != m_unallocated_slots.end()) + ); #endif - } - else if (m_slot_to_piece[i] == unassigned) - { + } + else if (m_slot_to_piece[i] == unassigned) + { #ifdef TORRENT_STORAGE_DEBUG - TORRENT_ASSERT( - std::find( - m_free_slots.begin() - , m_free_slots.end() - , i) != m_free_slots.end() - ); + TORRENT_ASSERT( + std::find( + m_free_slots.begin() + , m_free_slots.end() + , i) != m_free_slots.end() + ); #endif - } - else - { - TORRENT_ASSERT(false && "m_slot_to_piece[i] is invalid"); + } + else + { + TORRENT_ASSERT(false && "m_slot_to_piece[i] is invalid"); + } } } } diff --git a/libtorrent/src/torrent.cpp b/libtorrent/src/torrent.cpp index a094fa749..840e488ba 100755 --- a/libtorrent/src/torrent.cpp +++ b/libtorrent/src/torrent.cpp @@ -154,7 +154,7 @@ namespace libtorrent , boost::intrusive_ptr tf , fs::path const& save_path , tcp::endpoint const& net_interface - , bool compact_mode + , storage_mode_t storage_mode , int block_size , storage_constructor_type sc , bool paused) @@ -195,7 +195,7 @@ namespace libtorrent , m_total_redundant_bytes(0) , m_net_interface(net_interface.address(), 0) , m_save_path(complete(save_path)) - , m_compact_mode(compact_mode) + , m_storage_mode(storage_mode) , m_default_block_size(block_size) , m_connections_initialized(true) , m_settings(ses.settings()) @@ -215,7 +215,7 @@ namespace libtorrent , char const* name , fs::path const& save_path , tcp::endpoint const& net_interface - , bool compact_mode + , storage_mode_t storage_mode , int block_size , storage_constructor_type sc , bool paused) @@ -255,7 +255,7 @@ namespace libtorrent , m_total_redundant_bytes(0) , m_net_interface(net_interface.address(), 0) , m_save_path(complete(save_path)) - , m_compact_mode(compact_mode) + , m_storage_mode(storage_mode) , m_default_block_size(block_size) , m_connections_initialized(false) , m_settings(ses.settings()) @@ -1032,6 +1032,16 @@ namespace libtorrent m_announce_timer.cancel(); } + void torrent::on_files_deleted(int ret, disk_io_job const& j) + { + session_impl::mutex_t::scoped_lock l(m_ses.m_mutex); + + if (alerts().should_post(alert::warning)) + { + alerts().post_alert(torrent_deleted_alert(get_handle(), "files deleted")); + } + } + void torrent::on_files_released(int ret, disk_io_job const& j) { /* @@ -1668,8 +1678,6 @@ namespace libtorrent try { - TORRENT_ASSERT(m_connections.find(a) == m_connections.end()); - // add the newly connected peer to this torrent's peer list TORRENT_ASSERT(m_connections.find(a) == m_connections.end()); m_connections.insert( @@ -1883,10 +1891,13 @@ namespace libtorrent std::make_pair(a, boost::get_pointer(c))); m_ses.m_connections.insert(std::make_pair(s, c)); + int timeout = settings().peer_connect_timeout; + if (peerinfo) timeout += 3 * peerinfo->failcount; + m_ses.m_half_open.enqueue( bind(&peer_connection::connect, c, _1) , bind(&peer_connection::timed_out, c) - , seconds(settings().peer_connect_timeout)); + , seconds(timeout)); } catch (std::exception& e) { @@ -2215,10 +2226,22 @@ namespace libtorrent bool done = true; try { + std::string error_msg; TORRENT_ASSERT(m_storage); TORRENT_ASSERT(m_owning_storage.get()); done = m_storage->check_fastresume(data, m_have_pieces, m_num_pieces - , m_compact_mode); + , m_storage_mode, error_msg); + + if (!error_msg.empty() && m_ses.m_alerts.should_post(alert::warning)) + { + m_ses.m_alerts.post_alert(fastresume_rejected_alert( + get_handle(), error_msg)); +#if defined(TORRENT_VERBOSE_LOGGING) || defined(TORRENT_LOGGING) + (*m_ses.m_logger) << "fastresume data for " + << torrent_file().name() << " rejected: " + << error_msg << "\n"; +#endif + } } catch (std::exception& e) { @@ -2378,8 +2401,6 @@ namespace libtorrent piece_manager& torrent::filesystem() { - INVARIANT_CHECK; - TORRENT_ASSERT(m_owning_storage.get()); return *m_owning_storage; } @@ -2537,6 +2558,29 @@ namespace libtorrent return limit; } + void torrent::delete_files() + { +#if defined(TORRENT_VERBOSE_LOGGING) + for (peer_iterator i = m_connections.begin(); + i != m_connections.end(); ++i) + { + (*i->second->m_logger) << "*** DELETING FILES IN TORRENT\n"; + } +#endif + + disconnect_all(); + m_paused = true; + // tell the tracker that we stopped + m_event = tracker_request::stopped; + + if (m_owning_storage.get()) + { + TORRENT_ASSERT(m_storage); + m_storage->async_delete_files( + bind(&torrent::on_files_deleted, shared_from_this(), _1, _2)); + } + } + void torrent::pause() { INVARIANT_CHECK; @@ -2768,7 +2812,7 @@ namespace libtorrent !boost::bind(&peer_connection::is_connecting , boost::bind(&std::map::value_type::second, _1))); - st.compact_mode = m_compact_mode; + st.storage_mode = m_storage_mode; st.num_complete = m_complete; st.num_incomplete = m_incomplete; diff --git a/libtorrent/src/torrent_handle.cpp b/libtorrent/src/torrent_handle.cpp index f6b5d47de..9418d50e8 100755 --- a/libtorrent/src/torrent_handle.cpp +++ b/libtorrent/src/torrent_handle.cpp @@ -661,7 +661,7 @@ namespace libtorrent if (!t->valid_metadata()) return entry(); - t->filesystem().export_piece_map(piece_index); + std::vector have_pieces = t->pieces(); entry ret(entry::dictionary_t); @@ -673,10 +673,6 @@ namespace libtorrent const sha1_hash& info_hash = t->torrent_file().info_hash(); ret["info-hash"] = std::string((char*)info_hash.begin(), (char*)info_hash.end()); - ret["slots"] = entry(entry::list_t); - entry::list_type& slots = ret["slots"].list(); - std::copy(piece_index.begin(), piece_index.end(), std::back_inserter(slots)); - // blocks per piece int num_blocks_per_piece = static_cast(t->torrent_file().piece_length()) / t->block_size(); @@ -706,6 +702,8 @@ namespace libtorrent // the unfinished piece's index piece_struct["piece"] = i->index; + have_pieces[i->index] = true; + std::string bitmask; const int num_bitmask_bytes = (std::max)(num_blocks_per_piece / 8, 1); @@ -722,10 +720,10 @@ namespace libtorrent } piece_struct["bitmask"] = bitmask; - TORRENT_ASSERT(t->filesystem().slot_for_piece(i->index) >= 0); + TORRENT_ASSERT(t->filesystem().slot_for(i->index) >= 0); unsigned long adler = t->filesystem().piece_crc( - t->filesystem().slot_for_piece(i->index) + t->filesystem().slot_for(i->index) , t->block_size() , i->info); @@ -735,6 +733,11 @@ namespace libtorrent up.push_back(piece_struct); } } + + t->filesystem().export_piece_map(piece_index, have_pieces); + entry::list_type& slots = ret["slots"].list(); + std::copy(piece_index.begin(), piece_index.end(), std::back_inserter(slots)); + // write local peers entry::list_type& peer_list = ret["peers"].list();