From bd02f837bb412695273df153e90afae502a1b519 Mon Sep 17 00:00:00 2001 From: Marcos Pinto Date: Tue, 9 Oct 2007 21:22:42 +0000 Subject: [PATCH] first attempt at switching over to new storage allocation --- .../include/libtorrent/allocate_resources.hpp | 78 ++ .../aux_/allocate_resources_impl.hpp | 328 ++++++++ .../include/libtorrent/aux_/session_impl.hpp | 4 +- .../include/libtorrent/resource_request.hpp | 99 +++ libtorrent/include/libtorrent/session.hpp | 8 +- libtorrent/include/libtorrent/storage.hpp | 51 +- libtorrent/include/libtorrent/torrent.hpp | 6 +- .../include/libtorrent/torrent_handle.hpp | 5 +- libtorrent/src/allocate_resources.cpp | 225 ++++++ libtorrent/src/piece_picker.cpp | 2 - libtorrent/src/session.cpp | 12 +- libtorrent/src/session_impl.cpp | 14 +- libtorrent/src/storage.cpp | 701 ++++++++++-------- libtorrent/src/torrent.cpp | 24 +- libtorrent/src/torrent_handle.cpp | 17 +- src/deluge_core.cpp | 11 +- 16 files changed, 1198 insertions(+), 387 deletions(-) create mode 100644 libtorrent/include/libtorrent/allocate_resources.hpp create mode 100644 libtorrent/include/libtorrent/aux_/allocate_resources_impl.hpp create mode 100755 libtorrent/include/libtorrent/resource_request.hpp create mode 100644 libtorrent/src/allocate_resources.cpp diff --git a/libtorrent/include/libtorrent/allocate_resources.hpp b/libtorrent/include/libtorrent/allocate_resources.hpp new file mode 100644 index 000000000..3d8237914 --- /dev/null +++ b/libtorrent/include/libtorrent/allocate_resources.hpp @@ -0,0 +1,78 @@ +/* + +Copyright (c) 2003, Magnus Jonsson +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef TORRENT_ALLOCATE_RESOURCES_HPP_INCLUDED +#define TORRENT_ALLOCATE_RESOURCES_HPP_INCLUDED + +#include +#include + +#include + +#include "libtorrent/resource_request.hpp" +#include "libtorrent/peer_id.hpp" +#include "libtorrent/socket.hpp" +#include "libtorrent/session.hpp" + +namespace libtorrent +{ + class peer_connection; + class torrent; + + int saturated_add(int a, int b); + + // Function to allocate a limited resource fairly among many consumers. + // It takes into account the current use, and the consumer's desired use. + // Should be invoked periodically to allow it adjust to the situation (make + // sure "used" is updated between calls!). + // If resources = std::numeric_limits::max() it means there is an infinite + // supply of resources (so everyone can get what they want). + + void allocate_resources( + int resources + , std::map >& torrents + , resource_request torrent::* res); + + void allocate_resources( + int resources + , std::map& connections + , resource_request peer_connection::* res); + + // Used for global limits. + void allocate_resources( + int resources + , std::vector& _sessions + , resource_request session::* res); +} + + +#endif diff --git a/libtorrent/include/libtorrent/aux_/allocate_resources_impl.hpp b/libtorrent/include/libtorrent/aux_/allocate_resources_impl.hpp new file mode 100644 index 000000000..31865d40a --- /dev/null +++ b/libtorrent/include/libtorrent/aux_/allocate_resources_impl.hpp @@ -0,0 +1,328 @@ +/* + +Copyright (c) 2003, Magnus Jonsson +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef TORRENT_ALLOCATE_RESOURCES_IMPL_HPP_INCLUDED +#define TORRENT_ALLOCATE_RESOURCES_IMPL_HPP_INCLUDED + +#include +#include + +#include + +#include "libtorrent/resource_request.hpp" +#include "libtorrent/peer_id.hpp" +#include "libtorrent/socket.hpp" +#include "libtorrent/size_type.hpp" + +#ifdef min +#undef min +#endif + +#ifdef max +#undef max +#endif + +namespace libtorrent +{ + + int saturated_add(int a, int b); + + namespace aux + { + // give num_resources to r, + // return how how many were actually accepted. + inline int give(resource_request& r, int num_resources) + { + assert(num_resources >= 0); + assert(r.given <= r.max); + + int accepted = (std::min)(num_resources, r.max - r.given); + assert(accepted >= 0); + + r.given += accepted; + assert(r.given <= r.max); + + return accepted; + } + + inline int div_round_up(int numerator, int denominator) + { + return (numerator + denominator - 1) / denominator; + } + +#ifndef NDEBUG + + template + class allocate_resources_contract_check + { + int m_resources; + It m_start; + It m_end; + resource_request T::* m_res; + + public: + allocate_resources_contract_check( + int resources + , It start + , It end + , resource_request T::* res) + : m_resources(resources) + , m_start(start) + , m_end(end) + , m_res(res) + { + assert(m_resources >= 0); + for (It i = m_start, end(m_end); i != end; ++i) + { + assert(((*i).*m_res).max >= 0); + assert(((*i).*m_res).given >= 0); + } + } + + ~allocate_resources_contract_check() + { + int sum_given = 0; + int sum_max = 0; + int sum_min = 0; + for (It i = m_start, end(m_end); i != end; ++i) + { + assert(((*i).*m_res).max >= 0); + assert(((*i).*m_res).min >= 0); + assert(((*i).*m_res).max >= ((*i).*m_res).min); + assert(((*i).*m_res).given >= 0); + assert(((*i).*m_res).given <= ((*i).*m_res).max); + + sum_given = saturated_add(sum_given, ((*i).*m_res).given); + sum_max = saturated_add(sum_max, ((*i).*m_res).max); + sum_min = saturated_add(sum_min, ((*i).*m_res).min); + } + if (sum_given != (std::min)(std::max(m_resources, sum_min), sum_max)) + { + std::cerr << sum_given << " " << m_resources << " " << sum_min << " " << sum_max << std::endl; + assert(false); + } + } + }; + +#endif + + template + void allocate_resources_impl( + int resources + , It start + , It end + , resource_request T::* res) + { + assert(resources >= 0); + #ifndef NDEBUG + allocate_resources_contract_check contract_check( + resources + , start + , end + , res); + #endif + + for (It i = start; i != end; ++i) + { + resource_request& r = (*i).*res; + r.leftovers = (std::max)(r.used - r.given, 0); + } + + if (resources == resource_request::inf) + { + // No competition for resources. + // Just give everyone what they want. + for (It i = start; i != end; ++i) + { + ((*i).*res).given = ((*i).*res).max; + } + return; + } + + // Resources are scarce + + int sum_max = 0; + int sum_min = 0; + // the number of consumer that saturated their + // quota last time slice + int num_saturated = 0; + // the total resources that those saturated their + // quota used. This is used to calculate the mean + // of the saturating consumers, in order to + // balance their quotas for the next time slice. + size_type saturated_sum = 0; + for (It i = start; i != end; ++i) + { + resource_request& r = (*i).*res; + sum_max = saturated_add(sum_max, r.max); + assert(r.min < resource_request::inf); + assert(r.min >= 0); + assert(r.min <= r.max); + sum_min += r.min; + + // a consumer that uses 95% or more of its assigned + // quota is considered saturating + size_type used = r.used; + if (r.given == 0) continue; + if (used * 20 / r.given >= 19) + { + ++num_saturated; + saturated_sum += r.given; + } + } + + if (sum_max <= resources) + { + // it turns out that there's no competition for resources + // after all. + for (It i = start; i != end; ++i) + { + ((*i).*res).given = ((*i).*res).max; + } + return; + } + + if (sum_min >= resources) + { + // the amount of resources is smaller than + // the minimum resources to distribute, so + // give everyone the minimum + for (It i = start; i != end; ++i) + { + ((*i).*res).given = ((*i).*res).min; + } + return; + } + + // now, the "used" field will be used as a target value. + // the algorithm following this loop will then scale the + // used values to fit the available resources and store + // the scaled values as given. So, the ratios of the + // used values will be maintained. + for (It i = start; i != end; ++i) + { + resource_request& r = (*i).*res; + + int target; + size_type used = r.used; + if (r.given > 0 && used * 20 / r.given >= 19) + { + assert(num_saturated > 0); + target = div_round_up(saturated_sum, num_saturated); + target += div_round_up(target, 10); + } + else + { + target = r.used; + } + if (target > r.max) target = r.max; + else if (target < r.min) target = r.min; + + // move 12.5% towards the the target value + r.used = r.given + div_round_up(target - r.given, 8); + r.given = r.min; + } + + + resources = (std::max)(resources, sum_min); + int resources_to_distribute = (std::min)(resources, sum_max) - sum_min; + assert(resources_to_distribute >= 0); +#ifndef NDEBUG + int prev_resources_to_distribute = resources_to_distribute; +#endif + while (resources_to_distribute > 0) + { + // in order to scale, we need to calculate the sum of + // all the used values. + size_type total_used = 0; + size_type max_used = 0; + for (It i = start; i != end; ++i) + { + resource_request& r = (*i).*res; + if (r.given == r.max) continue; + + assert(r.given < r.max); + + max_used = (std::max)(max_used, (size_type)r.used + 1); + total_used += (size_type)r.used + 1; + } + + + size_type kNumer = resources_to_distribute; + size_type kDenom = total_used; + assert(kNumer >= 0); + assert(kDenom >= 0); + assert(kNumer <= (std::numeric_limits::max)()); + + if (kNumer * max_used <= kDenom) + { + kNumer = 1; + kDenom = max_used; + assert(kDenom >= 0); + } + + for (It i = start; i != end && resources_to_distribute > 0; ++i) + { + resource_request& r = (*i).*res; + if (r.given == r.max) continue; + + assert(r.given < r.max); + + size_type used = (size_type)r.used + 1; + if (used < 1) used = 1; + size_type to_give = used * kNumer / kDenom; + if (to_give > resources_to_distribute) + to_give = resources_to_distribute; + assert(to_give >= 0); + assert(to_give <= resources_to_distribute); +#ifndef NDEBUG + int tmp = resources_to_distribute; +#endif + resources_to_distribute -= give(r, (int)to_give); + assert(resources_to_distribute <= tmp); + assert(resources_to_distribute >= 0); + } + + assert(resources_to_distribute >= 0); + assert(resources_to_distribute < prev_resources_to_distribute); +#ifndef NDEBUG + prev_resources_to_distribute = resources_to_distribute; +#endif + } + assert(resources_to_distribute == 0); + } + + } // namespace libtorrent::aux +} + + +#endif diff --git a/libtorrent/include/libtorrent/aux_/session_impl.hpp b/libtorrent/include/libtorrent/aux_/session_impl.hpp index d069d73e3..80ade9dff 100644 --- a/libtorrent/include/libtorrent/aux_/session_impl.hpp +++ b/libtorrent/include/libtorrent/aux_/session_impl.hpp @@ -254,7 +254,7 @@ namespace libtorrent boost::intrusive_ptr ti , fs::path const& save_path , entry const& resume_data - , bool compact_mode + , storage_mode_t storage_mode , storage_constructor_type sc , bool paused , void* userdata); @@ -265,7 +265,7 @@ namespace libtorrent , char const* name , fs::path const& save_path , entry const& resume_data - , bool compact_mode + , storage_mode_t storage_mode , storage_constructor_type sc , bool paused , void* userdata); diff --git a/libtorrent/include/libtorrent/resource_request.hpp b/libtorrent/include/libtorrent/resource_request.hpp new file mode 100755 index 000000000..1e41b9cbb --- /dev/null +++ b/libtorrent/include/libtorrent/resource_request.hpp @@ -0,0 +1,99 @@ +/* + +Copyright (c) 2003, Magnus Jonsson, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +#ifndef TORRENT_RESOURCE_REQUEST_HPP_INCLUDED +#define TORRENT_RESOURCE_REQUEST_HPP_INCLUDED + +#include + +#ifdef min +#undef min +#endif + +#ifdef max +#undef max +#endif + +#include "libtorrent/config.hpp" + +namespace libtorrent +{ + struct TORRENT_EXPORT resource_request + { + resource_request() + : used(0) + , min(0) + , max(0) + , given(0) + , leftovers(0) + {} + + resource_request(int used_, int min_, int max_, int given_) + : used(used_) + , min(min_) + , max(max_) + , given(given_) + , leftovers(0) + {} + + int left() const + { + assert(given <= max); + assert(given >= min); + assert(used >= 0); + return (std::max)(given - used, 0); + } + + void reset() { used = leftovers; leftovers = 0; } + + static const int inf = boost::integer_traits::const_max; + + // right now I'm actively using this amount + int used; + + // given cannot be smaller than min + // and not greater than max. + int min; + int max; + + // Reply: Okay, you're allowed to use this amount (a compromise): + int given; + + // this is the amount of resources that exceeded the + // given limit. When the used field is reset (after resources + // have been distributed), it is reset to this number. + int leftovers; + }; +} + + +#endif diff --git a/libtorrent/include/libtorrent/session.hpp b/libtorrent/include/libtorrent/session.hpp index 3a9eb563b..446fe74c8 100755 --- a/libtorrent/include/libtorrent/session.hpp +++ b/libtorrent/include/libtorrent/session.hpp @@ -115,7 +115,7 @@ namespace libtorrent : m_impl(impl) {} boost::shared_ptr m_impl; }; - + class TORRENT_EXPORT session: public boost::noncopyable, aux::eh_initializer { public: @@ -140,7 +140,7 @@ namespace libtorrent torrent_info const& ti , fs::path const& save_path , entry const& resume_data = entry() - , bool compact_mode = true + , storage_mode_t storage_mode = storage_mode_sparse , bool paused = false , storage_constructor_type sc = default_storage_constructor) TORRENT_DEPRECATED; @@ -148,7 +148,7 @@ namespace libtorrent boost::intrusive_ptr ti , fs::path const& save_path , entry const& resume_data = entry() - , bool compact_mode = true + , storage_mode_t storage_mode = storage_mode_sparse , bool paused = false , storage_constructor_type sc = default_storage_constructor , void* userdata = 0); @@ -159,7 +159,7 @@ namespace libtorrent , char const* name , fs::path const& save_path , entry const& resume_data = entry() - , bool compact_mode = true + , storage_mode_t storage_mode = storage_mode_sparse , bool paused = false , storage_constructor_type sc = default_storage_constructor , void* userdata = 0); diff --git a/libtorrent/include/libtorrent/storage.hpp b/libtorrent/include/libtorrent/storage.hpp index 67d74153d..bad4daca6 100755 --- a/libtorrent/include/libtorrent/storage.hpp +++ b/libtorrent/include/libtorrent/storage.hpp @@ -71,6 +71,13 @@ namespace libtorrent struct file_pool; struct disk_io_job; + enum storage_mode_t + { + storage_mode_allocate = 0, + storage_mode_sparse, + storage_mode_compact + }; + #if defined(_WIN32) && defined(UNICODE) TORRENT_EXPORT std::wstring safe_convert(std::string const& s); @@ -180,7 +187,8 @@ namespace libtorrent ~piece_manager(); bool check_fastresume(aux::piece_checker_data& d - , std::vector& pieces, int& num_pieces, bool compact_mode); + , std::vector& pieces, int& num_pieces, storage_mode_t storage_mode + , std::string& error_msg); std::pair check_files(std::vector& pieces , int& num_pieces, boost::recursive_mutex& mutex); @@ -191,8 +199,8 @@ namespace libtorrent bool verify_resume_data(entry& rd, std::string& error); bool is_allocating() const - { return m_state == state_allocating; } - + { return m_state == state_expand_pieces; } + void mark_failed(int index); unsigned long piece_crc( @@ -200,8 +208,9 @@ namespace libtorrent , int block_size , piece_picker::block_info const* bi); - int slot_for_piece(int piece_index) const; - + int slot_for(int piece) const; + int piece_for(int slot) const; + void async_read( peer_request const& r , boost::function const& handler @@ -228,10 +237,11 @@ namespace libtorrent // slots to the piece that is stored (or // partially stored) there. -2 is the index // of unassigned pieces and -1 is unallocated - void export_piece_map(std::vector& pieces) const; + void export_piece_map(std::vector& pieces + , std::vector const& have) const; bool compact_allocation() const - { return m_compact_mode; } + { return m_storage_mode == storage_mode_compact; } #ifndef NDEBUG std::string name() const { return m_info->name(); } @@ -261,6 +271,7 @@ namespace libtorrent , int offset , int size); + void switch_to_full_mode(); sha1_hash hash_for_piece_impl(int piece); void release_files_impl(); @@ -276,16 +287,7 @@ namespace libtorrent #endif boost::scoped_ptr m_storage; - // if this is true, pieces are always allocated at the - // lowest possible slot index. If it is false, pieces - // are always written to their final place immediately - bool m_compact_mode; - - // if this is true, pieces that haven't been downloaded - // will be filled with zeroes. Not filling with zeroes - // will not work in some cases (where a seek cannot pass - // the end of the file). - bool m_fill_mode; + storage_mode_t m_storage_mode; // a bitmask representing the pieces we have std::vector m_have_piece; @@ -329,10 +331,21 @@ namespace libtorrent state_create_files, // checking the files state_full_check, - // allocating files (in non-compact mode) - state_allocating + // move pieces to their final position + state_expand_pieces } m_state; int m_current_slot; + // used during check. If any piece is found + // that is not in its final position, this + // is set to true + bool m_out_of_place; + // used to move pieces while expanding + // the storage from compact allocation + // to full allocation + std::vector m_scratch_buffer; + std::vector m_scratch_buffer2; + // the piece that is in the scratch buffer + int m_scratch_piece; // this is saved in case we need to instantiate a new // storage (osed when remapping files) diff --git a/libtorrent/include/libtorrent/torrent.hpp b/libtorrent/include/libtorrent/torrent.hpp index 4bb68436c..7d5520b18 100755 --- a/libtorrent/include/libtorrent/torrent.hpp +++ b/libtorrent/include/libtorrent/torrent.hpp @@ -101,7 +101,7 @@ namespace libtorrent , boost::intrusive_ptr tf , fs::path const& save_path , tcp::endpoint const& net_interface - , bool compact_mode + , storage_mode_t m_storage_mode , int block_size , storage_constructor_type sc , bool paused); @@ -116,7 +116,7 @@ namespace libtorrent , char const* name , fs::path const& save_path , tcp::endpoint const& net_interface - , bool compact_mode + , storage_mode_t m_storage_mode , int block_size , storage_constructor_type sc , bool paused); @@ -751,7 +751,7 @@ namespace libtorrent fs::path m_save_path; // determines the storage state for this torrent. - const bool m_compact_mode; + storage_mode_t m_storage_mode; // defaults to 16 kiB, but can be set by the user // when creating the torrent diff --git a/libtorrent/include/libtorrent/torrent_handle.hpp b/libtorrent/include/libtorrent/torrent_handle.hpp index ec2d60ea5..8ae2f7f41 100755 --- a/libtorrent/include/libtorrent/torrent_handle.hpp +++ b/libtorrent/include/libtorrent/torrent_handle.hpp @@ -52,6 +52,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "libtorrent/torrent_info.hpp" #include "libtorrent/time.hpp" #include "libtorrent/config.hpp" +#include "libtorrent/storage.hpp" namespace libtorrent { @@ -106,7 +107,7 @@ namespace libtorrent , num_connections(0) , uploads_limit(0) , connections_limit(0) - , compact_mode(false) + , storage_mode(storage_mode_sparse) {} enum state_t @@ -216,7 +217,7 @@ namespace libtorrent // true if the torrent is saved in compact mode // false if it is saved in full allocation mode - bool compact_mode; + storage_mode_t storage_mode; }; struct TORRENT_EXPORT block_info diff --git a/libtorrent/src/allocate_resources.cpp b/libtorrent/src/allocate_resources.cpp new file mode 100644 index 000000000..deef06dc4 --- /dev/null +++ b/libtorrent/src/allocate_resources.cpp @@ -0,0 +1,225 @@ +/* + +Copyright (c) 2006, Magnus Jonsson, Arvid Norberg +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +*/ + +//The Standard Library defines the two template functions std::min() +//and std::max() in the header. In general, you should +//use these template functions for calculating the min and max values +//of a pair. Unfortunately, Visual C++ does not define these function +// templates. This is because the names min and max clash with +//the traditional min and max macros defined in . +//As a workaround, Visual C++ defines two alternative templates with +//identical functionality called _cpp_min() and _cpp_max(). You can +//use them instead of std::min() and std::max().To disable the +//generation of the min and max macros in Visual C++, #define +//NOMINMAX before #including . + +#include "libtorrent/pch.hpp" + +#ifdef _WIN32 + //support boost1.32.0(2004-11-19 18:47) + //now all libs can be compiled and linked with static module + #define NOMINMAX +#endif + +#include "libtorrent/allocate_resources.hpp" +#include "libtorrent/size_type.hpp" +#include "libtorrent/peer_connection.hpp" +#include "libtorrent/torrent.hpp" +#include "libtorrent/aux_/allocate_resources_impl.hpp" + +#include +#include +#include + +#if defined(_MSC_VER) && _MSC_VER < 1310 +#define for if (false) {} else for +#else +#include +#endif + +namespace libtorrent +{ + int saturated_add(int a, int b) + { + assert(a >= 0); + assert(b >= 0); + assert(a <= resource_request::inf); + assert(b <= resource_request::inf); + assert(resource_request::inf + resource_request::inf < 0); + + unsigned int sum = unsigned(a) + unsigned(b); + if (sum > unsigned(resource_request::inf)) + sum = resource_request::inf; + + assert(sum >= unsigned(a) && sum >= unsigned(b)); + return int(sum); + } + +#if defined(_MSC_VER) && _MSC_VER < 1310 + + namespace detail + { + struct iterator_wrapper + { + typedef std::map >::iterator orig_iter; + + orig_iter iter; + + iterator_wrapper(orig_iter i): iter(i) {} + void operator++() { ++iter; } + torrent& operator*() { return *(iter->second); } + bool operator==(const iterator_wrapper& i) const + { return iter == i.iter; } + bool operator!=(const iterator_wrapper& i) const + { return iter != i.iter; } + }; + + struct iterator_wrapper2 + { + typedef std::map::iterator orig_iter; + + orig_iter iter; + + iterator_wrapper2(orig_iter i): iter(i) {} + void operator++() { ++iter; } + peer_connection& operator*() { return *(iter->second); } + bool operator==(const iterator_wrapper2& i) const + { return iter == i.iter; } + bool operator!=(const iterator_wrapper2& i) const + { return iter != i.iter; } + }; + + } + + void allocate_resources( + int resources + , std::map >& c + , resource_request torrent::* res) + { + aux::allocate_resources_impl( + resources + , detail::iterator_wrapper(c.begin()) + , detail::iterator_wrapper(c.end()) + , res); + } + + void allocate_resources( + int resources + , std::map& c + , resource_request peer_connection::* res) + { + aux::allocate_resources_impl( + resources + , detail::iterator_wrapper2(c.begin()) + , detail::iterator_wrapper2(c.end()) + , res); + } + +#else + + namespace aux + { + peer_connection& pick_peer( + std::pair + , boost::intrusive_ptr > const& p) + { + return *p.second; + } + + peer_connection& pick_peer2( + std::pair const& p) + { + return *p.second; + } + + torrent& deref(std::pair > const& p) + { + return *p.second; + } + + session& deref(session* p) + { + return *p; + } + } + + void allocate_resources( + int resources + , std::map >& c + , resource_request torrent::* res) + { + typedef std::map >::iterator orig_iter; + typedef std::pair > in_param; + typedef boost::transform_iterator new_iter; + + aux::allocate_resources_impl( + resources + , new_iter(c.begin(), &aux::deref) + , new_iter(c.end(), &aux::deref) + , res); + } + + void allocate_resources( + int resources + , std::map& c + , resource_request peer_connection::* res) + { + typedef std::map::iterator orig_iter; + typedef std::pair in_param; + typedef boost::transform_iterator new_iter; + + aux::allocate_resources_impl( + resources + , new_iter(c.begin(), &aux::pick_peer2) + , new_iter(c.end(), &aux::pick_peer2) + , res); + } + + void allocate_resources( + int resources + , std::vector& _sessions + , resource_request session::* res) + { + typedef std::vector::iterator orig_iter; + typedef session* in_param; + typedef boost::transform_iterator new_iter; + + aux::allocate_resources_impl( + resources + , new_iter(_sessions.begin(), &aux::deref) + , new_iter(_sessions.end(), &aux::deref) + , res); + } + +#endif + +} // namespace libtorrent diff --git a/libtorrent/src/piece_picker.cpp b/libtorrent/src/piece_picker.cpp index 652426806..ebfe07637 100755 --- a/libtorrent/src/piece_picker.cpp +++ b/libtorrent/src/piece_picker.cpp @@ -1732,7 +1732,6 @@ namespace libtorrent ++i->writing; info.state = block_info::state_writing; if (info.num_peers > 0) --info.num_peers; - TORRENT_ASSERT(info.num_peers >= 0); if (i->requested == 0) { @@ -1855,7 +1854,6 @@ namespace libtorrent block_info& info = i->info[block.block_index]; --info.num_peers; - TORRENT_ASSERT(info.num_peers >= 0); if (info.num_peers > 0) return; if (i->info[block.block_index].state == block_info::state_finished diff --git a/libtorrent/src/session.cpp b/libtorrent/src/session.cpp index 68cbd619f..e0c3ec57a 100755 --- a/libtorrent/src/session.cpp +++ b/libtorrent/src/session.cpp @@ -186,28 +186,28 @@ namespace libtorrent torrent_info const& ti , fs::path const& save_path , entry const& resume_data - , bool compact_mode + , storage_mode_t storage_mode , bool paused , storage_constructor_type sc) { TORRENT_ASSERT(!ti.m_half_metadata); boost::intrusive_ptr tip(new torrent_info(ti)); return m_impl->add_torrent(tip, save_path, resume_data - , compact_mode, sc, paused, 0); + , storage_mode, sc, paused, 0); } torrent_handle session::add_torrent( boost::intrusive_ptr ti , fs::path const& save_path , entry const& resume_data - , bool compact_mode + , storage_mode_t storage_mode , bool paused , storage_constructor_type sc , void* userdata) { TORRENT_ASSERT(!ti->m_half_metadata); return m_impl->add_torrent(ti, save_path, resume_data - , compact_mode, sc, paused, userdata); + , storage_mode, sc, paused, userdata); } torrent_handle session::add_torrent( @@ -216,13 +216,13 @@ namespace libtorrent , char const* name , fs::path const& save_path , entry const& e - , bool compact_mode + , storage_mode_t storage_mode , bool paused , storage_constructor_type sc , void* userdata) { return m_impl->add_torrent(tracker_url, info_hash, name, save_path, e - , compact_mode, sc, paused, userdata); + , storage_mode, sc, paused, userdata); } void session::remove_torrent(const torrent_handle& h) diff --git a/libtorrent/src/session_impl.cpp b/libtorrent/src/session_impl.cpp index 83498ac14..e229155e8 100755 --- a/libtorrent/src/session_impl.cpp +++ b/libtorrent/src/session_impl.cpp @@ -189,9 +189,11 @@ namespace detail t->parse_resume_data(t->resume_data, t->torrent_ptr->torrent_file() , error_msg); + // lock the session to add the new torrent + session_impl::mutex_t::scoped_lock l(m_ses.m_mutex); + if (!error_msg.empty() && m_ses.m_alerts.should_post(alert::warning)) { - session_impl::mutex_t::scoped_lock l(m_ses.m_mutex); m_ses.m_alerts.post_alert(fastresume_rejected_alert( t->torrent_ptr->get_handle() , error_msg)); @@ -202,8 +204,6 @@ namespace detail #endif } - // lock the session to add the new torrent - session_impl::mutex_t::scoped_lock l(m_ses.m_mutex); mutex::scoped_lock l2(m_mutex); if (m_torrents.empty() || m_torrents.front() != t) @@ -1623,7 +1623,7 @@ namespace detail boost::intrusive_ptr ti , fs::path const& save_path , entry const& resume_data - , bool compact_mode + , storage_mode_t storage_mode , storage_constructor_type sc , bool paused , void* userdata) @@ -1655,7 +1655,7 @@ namespace detail // the thread boost::shared_ptr torrent_ptr( new torrent(*this, m_checker_impl, ti, save_path - , m_listen_interface, compact_mode, 16 * 1024 + , m_listen_interface, storage_mode, 16 * 1024 , sc, paused)); torrent_ptr->start(); @@ -1701,7 +1701,7 @@ namespace detail , char const* name , fs::path const& save_path , entry const& - , bool compact_mode + , storage_mode_t storage_mode , storage_constructor_type sc , bool paused , void* userdata) @@ -1735,7 +1735,7 @@ namespace detail // the thread boost::shared_ptr torrent_ptr( new torrent(*this, m_checker_impl, tracker_url, info_hash, name - , save_path, m_listen_interface, compact_mode, 16 * 1024 + , save_path, m_listen_interface, storage_mode, 16 * 1024 , sc, paused)); torrent_ptr->start(); diff --git a/libtorrent/src/storage.cpp b/libtorrent/src/storage.cpp index 1cac99e44..15a869a03 100755 --- a/libtorrent/src/storage.cpp +++ b/libtorrent/src/storage.cpp @@ -447,7 +447,7 @@ namespace libtorrent } // if the file is empty, just create it. But also make sure - // the directory exits. + // the directory exists. if (file_iter->size == 0) { file(m_save_path / file_iter->path, file::out); @@ -931,107 +931,6 @@ namespace libtorrent return new storage(ti, path, fp); } - bool supports_sparse_files(fs::path const& p) - { - TORRENT_ASSERT(p.is_complete()); -#if defined(_WIN32) - // assume windows API is available - DWORD max_component_len = 0; - DWORD volume_flags = 0; - std::string root_device = p.root_name() + "\\"; -#if defined(UNICODE) - std::wstring wph(safe_convert(root_device)); - bool ret = ::GetVolumeInformation(wph.c_str(), 0 - , 0, 0, &max_component_len, &volume_flags, 0, 0); -#else - bool ret = ::GetVolumeInformation(root_device.c_str(), 0 - , 0, 0, &max_component_len, &volume_flags, 0, 0); -#endif - - if (!ret) return false; - if (volume_flags & FILE_SUPPORTS_SPARSE_FILES) - return true; -#endif - -#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) - // find the last existing directory of the save path - fs::path query_path = p; - while (!query_path.empty() && !exists(query_path)) - query_path = query_path.branch_path(); -#endif - -#if defined(__APPLE__) - - struct statfs fsinfo; - int ret = statfs(query_path.native_directory_string().c_str(), &fsinfo); - if (ret != 0) return false; - - attrlist request; - request.bitmapcount = ATTR_BIT_MAP_COUNT; - request.reserved = 0; - request.commonattr = 0; - request.volattr = ATTR_VOL_CAPABILITIES; - request.dirattr = 0; - request.fileattr = 0; - request.forkattr = 0; - - struct vol_capabilities_attr_buf - { - unsigned long length; - vol_capabilities_attr_t info; - } vol_cap; - - ret = getattrlist(fsinfo.f_mntonname, &request, &vol_cap - , sizeof(vol_cap), 0); - if (ret != 0) return false; - - if (vol_cap.info.capabilities[VOL_CAPABILITIES_FORMAT] - & (VOL_CAP_FMT_SPARSE_FILES | VOL_CAP_FMT_ZERO_RUNS)) - { - return true; - } - - // workaround for bugs in Mac OS X where zero run is not reported - if (!strcmp(fsinfo.f_fstypename, "hfs") - || !strcmp(fsinfo.f_fstypename, "ufs")) - return true; - - return false; -#endif - -#if defined(__linux__) || defined(__FreeBSD__) - struct statfs buf; - int err = statfs(query_path.native_directory_string().c_str(), &buf); - if (err == 0) - { - switch (buf.f_type) - { - case 0x5346544e: // NTFS - case 0xEF51: // EXT2 OLD - case 0xEF53: // EXT2 and EXT3 - case 0x00011954: // UFS - case 0x52654973: // ReiserFS - case 0x52345362: // Reiser4 - case 0x58465342: // XFS - case 0x65735546: // NTFS-3G - case 0x19540119: // UFS2 - return true; - } - } -#ifndef NDEBUG - else - { - std::cerr << "statfs returned " << err << std::endl; - std::cerr << "errno: " << errno << std::endl; - std::cerr << "path: " << query_path.native_directory_string() << std::endl; - } -#endif -#endif - - // TODO: POSIX implementation - return false; - } - // -- piece_manager ----------------------------------------------------- piece_manager::piece_manager( @@ -1042,15 +941,16 @@ namespace libtorrent , disk_io_thread& io , storage_constructor_type sc) : m_storage(sc(ti, save_path, fp)) - , m_compact_mode(false) - , m_fill_mode(true) + , m_storage_mode(storage_mode_sparse) , m_info(ti) , m_save_path(complete(save_path)) + , m_current_slot(0) + , m_out_of_place(false) + , m_scratch_piece(-1) , m_storage_constructor(sc) , m_io_thread(io) , m_torrent(torrent) { - m_fill_mode = !supports_sparse_files(save_path); } piece_manager::~piece_manager() @@ -1158,7 +1058,7 @@ namespace libtorrent m_piece_hasher.erase(i); } - int slot = m_piece_to_slot[piece]; + int slot = slot_for(piece); TORRENT_ASSERT(slot != has_no_slot); return m_storage->hash_for_slot(slot, ph, m_info->piece_size(piece)); } @@ -1178,25 +1078,37 @@ namespace libtorrent return false; } void piece_manager::export_piece_map( - std::vector& p) const + std::vector& p, std::vector const& have) const { boost::recursive_mutex::scoped_lock lock(m_mutex); INVARIANT_CHECK; - p.clear(); - std::vector::const_reverse_iterator last; - for (last = m_slot_to_piece.rbegin(); - last != m_slot_to_piece.rend(); ++last) + if (m_storage_mode == storage_mode_compact) { - if (*last != unallocated) break; - } + p.clear(); + p.reserve(m_info->num_pieces()); + std::vector::const_reverse_iterator last; + for (last = m_slot_to_piece.rbegin(); + last != m_slot_to_piece.rend(); ++last) + { + if (*last != unallocated && have[*last]) break; + } - for (std::vector::const_iterator i = - m_slot_to_piece.begin(); - i != last.base(); ++i) + for (std::vector::const_iterator i = + m_slot_to_piece.begin(); + i != last.base(); ++i) + { + p.push_back(have[*i] ? *i : unassigned); + } + } + else { - p.push_back(*i); + p.reserve(m_info->num_pieces()); + for (int i = 0; i < m_info->num_pieces(); ++i) + { + p.push_back(have[i] ? i : unassigned); + } } } @@ -1206,11 +1118,10 @@ namespace libtorrent INVARIANT_CHECK; + if (m_storage_mode != storage_mode_compact) return; + TORRENT_ASSERT(piece_index >= 0 && piece_index < (int)m_piece_to_slot.size()); - TORRENT_ASSERT(m_piece_to_slot[piece_index] >= 0); - int slot_index = m_piece_to_slot[piece_index]; - TORRENT_ASSERT(slot_index >= 0); m_slot_to_piece[slot_index] = unassigned; @@ -1218,12 +1129,6 @@ namespace libtorrent m_free_slots.push_back(slot_index); } - int piece_manager::slot_for_piece(int piece_index) const - { - TORRENT_ASSERT(piece_index >= 0 && piece_index < m_info->num_pieces()); - return m_piece_to_slot[piece_index]; - } - unsigned long piece_manager::piece_crc( int slot_index , int block_size @@ -1275,11 +1180,7 @@ namespace libtorrent TORRENT_ASSERT(buf); TORRENT_ASSERT(offset >= 0); TORRENT_ASSERT(size > 0); - TORRENT_ASSERT(piece_index >= 0 && piece_index < (int)m_piece_to_slot.size()); - TORRENT_ASSERT(m_piece_to_slot[piece_index] >= 0 - && m_piece_to_slot[piece_index] < (int)m_slot_to_piece.size()); - int slot = m_piece_to_slot[piece_index]; - TORRENT_ASSERT(slot >= 0 && slot < (int)m_slot_to_piece.size()); + int slot = slot_for(piece_index); return m_storage->read(buf, slot, offset, size); } @@ -1292,7 +1193,7 @@ namespace libtorrent TORRENT_ASSERT(buf); TORRENT_ASSERT(offset >= 0); TORRENT_ASSERT(size > 0); - TORRENT_ASSERT(piece_index >= 0 && piece_index < (int)m_piece_to_slot.size()); + TORRENT_ASSERT(piece_index >= 0 && piece_index < m_info->num_pieces()); if (offset == 0) { @@ -1317,7 +1218,6 @@ namespace libtorrent } int slot = allocate_slot_for_piece(piece_index); - TORRENT_ASSERT(slot >= 0 && slot < (int)m_slot_to_piece.size()); m_storage->write(buf, slot, offset, size); } @@ -1426,7 +1326,8 @@ namespace libtorrent // that piece as unassigned, since this slot // is the correct place for the piece. m_slot_to_piece[other_slot] = unassigned; - m_free_slots.push_back(other_slot); + if (m_storage_mode == storage_mode_compact) + m_free_slots.push_back(other_slot); } TORRENT_ASSERT(m_piece_to_slot[piece_index] != current_slot); TORRENT_ASSERT(m_piece_to_slot[piece_index] >= 0); @@ -1485,7 +1386,8 @@ namespace libtorrent bool piece_manager::check_fastresume( aux::piece_checker_data& data , std::vector& pieces - , int& num_pieces, bool compact_mode) + , int& num_pieces, storage_mode_t storage_mode + , std::string& error_msg) { boost::recursive_mutex::scoped_lock lock(m_mutex); @@ -1493,7 +1395,7 @@ namespace libtorrent TORRENT_ASSERT(m_info->piece_length() > 0); - m_compact_mode = compact_mode; + m_storage_mode = storage_mode; // This will corrupt the storage // use while debugging to find @@ -1503,9 +1405,13 @@ namespace libtorrent m_piece_to_slot.resize(m_info->num_pieces(), has_no_slot); m_slot_to_piece.resize(m_info->num_pieces(), unallocated); - m_free_slots.clear(); - m_unallocated_slots.clear(); + TORRENT_ASSERT(m_free_slots.empty()); + TORRENT_ASSERT(m_unallocated_slots.empty()); + // assume no piece is out of place (i.e. in a slot + // other than the one it should be in) + bool out_of_place = false; + pieces.clear(); pieces.resize(m_info->num_pieces(), false); num_pieces = 0; @@ -1513,13 +1419,14 @@ namespace libtorrent // if we have fast-resume info // use it instead of doing the actual checking if (!data.piece_map.empty() - && data.piece_map.size() <= m_slot_to_piece.size()) + && int(data.piece_map.size()) <= m_info->num_pieces()) { for (int i = 0; i < (int)data.piece_map.size(); ++i) { m_slot_to_piece[i] = data.piece_map[i]; if (data.piece_map[i] >= 0) { + if (data.piece_map[i] != i) out_of_place = true; m_piece_to_slot[data.piece_map[i]] = i; int found_piece = data.piece_map[i]; @@ -1537,30 +1444,58 @@ namespace libtorrent } else if (data.piece_map[i] == unassigned) { - m_free_slots.push_back(i); + if (m_storage_mode == storage_mode_compact) + m_free_slots.push_back(i); } else { TORRENT_ASSERT(data.piece_map[i] == unallocated); - m_unallocated_slots.push_back(i); + if (m_storage_mode == storage_mode_compact) + m_unallocated_slots.push_back(i); } } - m_unallocated_slots.reserve(int(pieces.size() - data.piece_map.size())); - for (int i = (int)data.piece_map.size(); i < (int)pieces.size(); ++i) + if (m_storage_mode == storage_mode_compact) { - m_unallocated_slots.push_back(i); + m_unallocated_slots.reserve(int(m_info->num_pieces() - data.piece_map.size())); + for (int i = (int)data.piece_map.size(); i < (int)m_info->num_pieces(); ++i) + { + m_unallocated_slots.push_back(i); + } + if (m_unallocated_slots.empty()) + { + switch_to_full_mode(); + } + } + else + { + if (!out_of_place) + { + // if no piece is out of place + // since we're in full allocation mode, we can + // forget the piece allocation tables + + std::vector().swap(m_piece_to_slot); + std::vector().swap(m_slot_to_piece); + m_state = state_create_files; + return false; + } + else + { + // in this case we're in full allocation mode, but + // we're resuming a compact allocated storage + m_state = state_expand_pieces; + m_current_slot = 0; + error_msg = "pieces needs to be reordered"; + return false; + } } - if (m_unallocated_slots.empty()) - m_state = state_create_files; - else if (m_compact_mode) - m_state = state_create_files; - else - m_state = state_allocating; + m_state = state_create_files; return false; } + error_msg = "empty piece map"; m_state = state_full_check; return false; } @@ -1572,18 +1507,13 @@ namespace libtorrent | | | v - | +------------+ - | | full_check | - | +------------+ - | | - | v - | +------------+ - |->| allocating | - | +------------+ - | | - | v - | +--------------+ - |->| create_files | + | +------------+ +---------------+ + | | full_check |-->| expand_pieses | + | +------------+ +---------------+ + | | | + | v | + | +--------------+ | + +->| create_files | <------+ +--------------+ | v @@ -1602,67 +1532,97 @@ namespace libtorrent std::pair piece_manager::check_files( std::vector& pieces, int& num_pieces, boost::recursive_mutex& mutex) { +#ifndef NDEBUG + boost::recursive_mutex::scoped_lock l_(mutex); TORRENT_ASSERT(num_pieces == std::count(pieces.begin(), pieces.end(), true)); - - if (m_state == state_allocating) - { - if (m_compact_mode || m_unallocated_slots.empty()) - { - m_state = state_create_files; - return std::make_pair(false, 1.f); - } - - if (int(m_unallocated_slots.size()) == m_info->num_pieces() - && !m_fill_mode) - { - // if there is not a single file on disk, just - // create the files - m_state = state_create_files; - return std::make_pair(false, 1.f); - } - - // if we're not in compact mode, make sure the - // pieces are spread out and placed at their - // final position. - TORRENT_ASSERT(!m_unallocated_slots.empty()); - - if (!m_fill_mode) - { - // if we're not filling the allocation - // just make sure we move the current pieces - // into place, and just skip all other - // allocation - // allocate_slots returns true if it had to - // move any data - allocate_slots(m_unallocated_slots.size(), true); - } - else - { - allocate_slots(1); - } - - return std::make_pair(false, 1.f - (float)m_unallocated_slots.size() - / (float)m_slot_to_piece.size()); - } + l_.unlock(); +#endif if (m_state == state_create_files) { - m_storage->initialize(!m_fill_mode && !m_compact_mode); - - if (!m_unallocated_slots.empty() && !m_compact_mode) - { - TORRENT_ASSERT(!m_fill_mode); - std::vector().swap(m_unallocated_slots); - std::fill(m_slot_to_piece.begin(), m_slot_to_piece.end(), int(unassigned)); - m_free_slots.resize(m_info->num_pieces()); - for (int i = 0; i < m_info->num_pieces(); ++i) - m_free_slots[i] = i; - } - + m_storage->initialize(m_storage_mode == storage_mode_allocate); m_state = state_finished; return std::make_pair(true, 1.f); } + if (m_state == state_expand_pieces) + { + INVARIANT_CHECK; + + if (m_scratch_piece >= 0) + { + int piece = m_scratch_piece; + int other_piece = m_slot_to_piece[piece]; + m_scratch_piece = -1; + + if (other_piece >= 0) + { + if (m_scratch_buffer2.empty()) + m_scratch_buffer2.resize(m_info->piece_length()); + + m_storage->read(&m_scratch_buffer2[0], piece, 0, m_info->piece_size(other_piece)); + m_scratch_piece = other_piece; + m_piece_to_slot[other_piece] = unassigned; + } + + // the slot where this piece belongs is + // free. Just move the piece there. + m_storage->write(&m_scratch_buffer[0], piece, 0, m_info->piece_size(piece)); + m_piece_to_slot[piece] = piece; + m_slot_to_piece[piece] = piece; + + if (other_piece >= 0) + m_scratch_buffer.swap(m_scratch_buffer2); + + return std::make_pair(false, (float)m_current_slot / m_info->num_pieces()); + } + + while (m_current_slot < m_info->num_pieces() + && (m_slot_to_piece[m_current_slot] == m_current_slot + || m_slot_to_piece[m_current_slot] < 0)) + { + ++m_current_slot; + } + + if (m_current_slot == m_info->num_pieces()) + { + m_state = state_create_files; + std::vector().swap(m_scratch_buffer); + std::vector().swap(m_scratch_buffer2); + if (m_storage_mode != storage_mode_compact) + { + std::vector().swap(m_piece_to_slot); + std::vector().swap(m_slot_to_piece); + } + return std::make_pair(false, 1.f); + } + + int piece = m_slot_to_piece[m_current_slot]; + TORRENT_ASSERT(piece >= 0); + int other_piece = m_slot_to_piece[piece]; + if (other_piece >= 0) + { + // there is another piece in the slot + // where this one goes. Store it in the scratch + // buffer until next iteration. + if (m_scratch_buffer.empty()) + m_scratch_buffer.resize(m_info->piece_length()); + + m_storage->read(&m_scratch_buffer[0], piece, 0, m_info->piece_size(other_piece)); + m_scratch_piece = other_piece; + m_piece_to_slot[other_piece] = unassigned; + } + + // the slot where this piece belongs is + // free. Just move the piece there. + m_storage->move_slot(m_current_slot, piece); + m_piece_to_slot[piece] = piece; + m_slot_to_piece[m_current_slot] = unassigned; + m_slot_to_piece[piece] = piece; + + return std::make_pair(false, (float)m_current_slot / m_info->num_pieces()); + } + TORRENT_ASSERT(m_state == state_full_check); // ------------------------ @@ -1674,12 +1634,13 @@ namespace libtorrent // initialization for the full check if (m_hash_to_piece.empty()) { - m_current_slot = 0; for (int i = 0; i < m_info->num_pieces(); ++i) { m_hash_to_piece.insert(std::make_pair(m_info->hash_for_piece(i), i)); } + boost::recursive_mutex::scoped_lock l(mutex); std::fill(pieces.begin(), pieces.end(), false); + num_pieces = 0; } m_piece_data.resize(int(m_info->piece_length())); @@ -1694,6 +1655,10 @@ namespace libtorrent int piece_index = identify_data(m_piece_data, m_current_slot , pieces, num_pieces, m_hash_to_piece, mutex); + if (piece_index != m_current_slot + && piece_index >= 0) + m_out_of_place = true; + TORRENT_ASSERT(num_pieces == std::count(pieces.begin(), pieces.end(), true)); TORRENT_ASSERT(piece_index == unassigned || piece_index >= 0); @@ -1745,8 +1710,11 @@ namespace libtorrent std::vector::iterator i = std::find(m_free_slots.begin(), m_free_slots.end(), other_slot); TORRENT_ASSERT(i != m_free_slots.end()); - m_free_slots.erase(i); - m_free_slots.push_back(m_current_slot); + if (m_storage_mode == storage_mode_compact) + { + m_free_slots.erase(i); + m_free_slots.push_back(m_current_slot); + } } if (other_piece >= 0) @@ -1770,7 +1738,8 @@ namespace libtorrent m_slot_to_piece[other_slot] = piece_index; m_piece_to_slot[other_piece] = m_current_slot; - if (piece_index == unassigned) + if (piece_index == unassigned + && m_storage_mode == storage_mode_compact) m_free_slots.push_back(other_slot); if (piece_index >= 0) @@ -1845,8 +1814,11 @@ namespace libtorrent std::vector::iterator i = std::find(m_free_slots.begin(), m_free_slots.end(), slot1); TORRENT_ASSERT(i != m_free_slots.end()); - m_free_slots.erase(i); - m_free_slots.push_back(slot2); + if (m_storage_mode == storage_mode_compact) + { + m_free_slots.erase(i); + m_free_slots.push_back(slot2); + } } if (piece1 >= 0) @@ -1873,7 +1845,7 @@ namespace libtorrent // the slot was identified as piece 'piece_index' if (piece_index != unassigned) m_piece_to_slot[piece_index] = m_current_slot; - else + else if (m_storage_mode == storage_mode_compact) m_free_slots.push_back(m_current_slot); m_slot_to_piece[m_current_slot] = piece_index; @@ -1899,10 +1871,13 @@ namespace libtorrent (file_offset - current_offset + m_info->piece_length() - 1) / m_info->piece_length()); - for (int i = m_current_slot; i < m_current_slot + skip_blocks; ++i) + if (m_storage_mode == storage_mode_compact) { - TORRENT_ASSERT(m_slot_to_piece[i] == unallocated); - m_unallocated_slots.push_back(i); + for (int i = m_current_slot; i < m_current_slot + skip_blocks; ++i) + { + TORRENT_ASSERT(m_slot_to_piece[i] == unallocated); + m_unallocated_slots.push_back(i); + } } // current slot will increase by one at the end of the for-loop too @@ -1910,15 +1885,46 @@ namespace libtorrent } ++m_current_slot; - if (m_current_slot >= m_info->num_pieces()) + if (m_current_slot >= m_info->num_pieces()) { TORRENT_ASSERT(m_current_slot == m_info->num_pieces()); // clear the memory we've been using std::vector().swap(m_piece_data); std::multimap().swap(m_hash_to_piece); - m_state = state_allocating; + + if (m_storage_mode != storage_mode_compact) + { + if (!m_out_of_place) + { + // if no piece is out of place + // since we're in full allocation mode, we can + // forget the piece allocation tables + + std::vector().swap(m_piece_to_slot); + std::vector().swap(m_slot_to_piece); + m_state = state_create_files; + return std::make_pair(false, 1.f); + } + else + { + // in this case we're in full allocation mode, but + // we're resuming a compact allocated storage + m_state = state_expand_pieces; + m_current_slot = 0; + return std::make_pair(false, 0.f); + } + } + else if (m_unallocated_slots.empty()) + { + switch_to_full_mode(); + } + m_state = state_create_files; + +#ifndef NDEBUG + boost::recursive_mutex::scoped_lock l(mutex); TORRENT_ASSERT(num_pieces == std::count(pieces.begin(), pieces.end(), true)); +#endif return std::make_pair(false, 1.f); } @@ -1927,10 +1933,26 @@ namespace libtorrent return std::make_pair(false, (float)m_current_slot / m_info->num_pieces()); } + void piece_manager::switch_to_full_mode() + { + TORRENT_ASSERT(m_storage_mode == storage_mode_compact); + TORRENT_ASSERT(m_unallocated_slots.empty()); + // we have allocated all slots, switch to + // full allocation mode in order to free + // some unnecessary memory. + m_storage_mode = storage_mode_sparse; + std::vector().swap(m_unallocated_slots); + std::vector().swap(m_free_slots); + std::vector().swap(m_piece_to_slot); + std::vector().swap(m_slot_to_piece); + } + int piece_manager::allocate_slot_for_piece(int piece_index) { boost::recursive_mutex::scoped_lock lock(m_mutex); + if (m_storage_mode != storage_mode_compact) return piece_index; + // INVARIANT_CHECK; TORRENT_ASSERT(piece_index >= 0); @@ -2030,26 +2052,27 @@ namespace libtorrent debug_log(); #endif } - TORRENT_ASSERT(slot_index >= 0); TORRENT_ASSERT(slot_index < (int)m_slot_to_piece.size()); + + if (m_unallocated_slots.empty()) + { + switch_to_full_mode(); + } + return slot_index; } bool piece_manager::allocate_slots(int num_slots, bool abort_on_disk) { - TORRENT_ASSERT(num_slots > 0); - boost::recursive_mutex::scoped_lock lock(m_mutex); + TORRENT_ASSERT(num_slots > 0); // INVARIANT_CHECK; TORRENT_ASSERT(!m_unallocated_slots.empty()); + TORRENT_ASSERT(m_storage_mode == storage_mode_compact); - const int stack_buffer_size = 16*1024; - char zeroes[stack_buffer_size]; - memset(zeroes, 0, stack_buffer_size); - bool written = false; for (int i = 0; i < num_slots && !m_unallocated_slots.empty(); ++i) @@ -2069,134 +2092,160 @@ namespace libtorrent m_piece_to_slot[pos] = pos; written = true; } - else if (m_fill_mode) - { - int piece_size = int(m_info->piece_size(pos)); - int offset = 0; - for (; piece_size > 0; piece_size -= stack_buffer_size - , offset += stack_buffer_size) - { - m_storage->write(zeroes, pos, offset - , (std::min)(piece_size, stack_buffer_size)); - } - written = true; - } m_unallocated_slots.erase(m_unallocated_slots.begin()); m_slot_to_piece[new_free_slot] = unassigned; m_free_slots.push_back(new_free_slot); - if (abort_on_disk && written) return true; + if (abort_on_disk && written) break; } TORRENT_ASSERT(m_free_slots.size() > 0); return written; } + int piece_manager::slot_for(int piece) const + { + if (m_storage_mode != storage_mode_compact) return piece; + TORRENT_ASSERT(piece < int(m_piece_to_slot.size())); + TORRENT_ASSERT(piece >= 0); + return m_piece_to_slot[piece]; + } + + int piece_manager::piece_for(int slot) const + { + if (m_storage_mode != storage_mode_compact) return slot; + TORRENT_ASSERT(slot < int(m_slot_to_piece.size())); + TORRENT_ASSERT(slot >= 0); + return m_slot_to_piece[slot]; + } + #ifndef NDEBUG void piece_manager::check_invariant() const { boost::recursive_mutex::scoped_lock lock(m_mutex); - if (m_piece_to_slot.empty()) return; - TORRENT_ASSERT((int)m_piece_to_slot.size() == m_info->num_pieces()); - TORRENT_ASSERT((int)m_slot_to_piece.size() == m_info->num_pieces()); - - for (std::vector::const_iterator i = m_free_slots.begin(); - i != m_free_slots.end(); ++i) + if (m_unallocated_slots.empty() && m_state == state_finished) { - TORRENT_ASSERT(*i < (int)m_slot_to_piece.size()); - TORRENT_ASSERT(*i >= 0); - TORRENT_ASSERT(m_slot_to_piece[*i] == unassigned); - TORRENT_ASSERT(std::find(i+1, m_free_slots.end(), *i) - == m_free_slots.end()); + TORRENT_ASSERT(m_storage_mode != storage_mode_compact); } - - for (std::vector::const_iterator i = m_unallocated_slots.begin(); - i != m_unallocated_slots.end(); ++i) + + if (m_storage_mode != storage_mode_compact) { - TORRENT_ASSERT(*i < (int)m_slot_to_piece.size()); - TORRENT_ASSERT(*i >= 0); - TORRENT_ASSERT(m_slot_to_piece[*i] == unallocated); - TORRENT_ASSERT(std::find(i+1, m_unallocated_slots.end(), *i) - == m_unallocated_slots.end()); + TORRENT_ASSERT(m_unallocated_slots.empty()); + TORRENT_ASSERT(m_free_slots.empty()); } - - for (int i = 0; i < m_info->num_pieces(); ++i) + + if (m_storage_mode != storage_mode_compact + && m_state != state_expand_pieces + && m_state != state_full_check) { - // Check domain of piece_to_slot's elements - if (m_piece_to_slot[i] != has_no_slot) + TORRENT_ASSERT(m_piece_to_slot.empty()); + TORRENT_ASSERT(m_slot_to_piece.empty()); + } + else + { + if (m_piece_to_slot.empty()) return; + + TORRENT_ASSERT((int)m_piece_to_slot.size() == m_info->num_pieces()); + TORRENT_ASSERT((int)m_slot_to_piece.size() == m_info->num_pieces()); + + for (std::vector::const_iterator i = m_free_slots.begin(); + i != m_free_slots.end(); ++i) { - TORRENT_ASSERT(m_piece_to_slot[i] >= 0); - TORRENT_ASSERT(m_piece_to_slot[i] < (int)m_slot_to_piece.size()); + TORRENT_ASSERT(*i < (int)m_slot_to_piece.size()); + TORRENT_ASSERT(*i >= 0); + TORRENT_ASSERT(m_slot_to_piece[*i] == unassigned); + TORRENT_ASSERT(std::find(i+1, m_free_slots.end(), *i) + == m_free_slots.end()); } - // Check domain of slot_to_piece's elements - if (m_slot_to_piece[i] != unallocated - && m_slot_to_piece[i] != unassigned) + for (std::vector::const_iterator i = m_unallocated_slots.begin(); + i != m_unallocated_slots.end(); ++i) { - TORRENT_ASSERT(m_slot_to_piece[i] >= 0); - TORRENT_ASSERT(m_slot_to_piece[i] < (int)m_piece_to_slot.size()); + TORRENT_ASSERT(*i < (int)m_slot_to_piece.size()); + TORRENT_ASSERT(*i >= 0); + TORRENT_ASSERT(m_slot_to_piece[*i] == unallocated); + TORRENT_ASSERT(std::find(i+1, m_unallocated_slots.end(), *i) + == m_unallocated_slots.end()); } - // do more detailed checks on piece_to_slot - if (m_piece_to_slot[i] >= 0) + for (int i = 0; i < m_info->num_pieces(); ++i) { - TORRENT_ASSERT(m_slot_to_piece[m_piece_to_slot[i]] == i); - if (m_piece_to_slot[i] != i) + // Check domain of piece_to_slot's elements + if (m_piece_to_slot[i] != has_no_slot) { - TORRENT_ASSERT(m_slot_to_piece[i] == unallocated); + TORRENT_ASSERT(m_piece_to_slot[i] >= 0); + TORRENT_ASSERT(m_piece_to_slot[i] < (int)m_slot_to_piece.size()); } - } - else - { - TORRENT_ASSERT(m_piece_to_slot[i] == has_no_slot); - } - // do more detailed checks on slot_to_piece + // Check domain of slot_to_piece's elements + if (m_slot_to_piece[i] != unallocated + && m_slot_to_piece[i] != unassigned) + { + TORRENT_ASSERT(m_slot_to_piece[i] >= 0); + TORRENT_ASSERT(m_slot_to_piece[i] < (int)m_piece_to_slot.size()); + } - if (m_slot_to_piece[i] >= 0) - { - TORRENT_ASSERT(m_slot_to_piece[i] < (int)m_piece_to_slot.size()); - TORRENT_ASSERT(m_piece_to_slot[m_slot_to_piece[i]] == i); + // do more detailed checks on piece_to_slot + if (m_piece_to_slot[i] >= 0) + { + TORRENT_ASSERT(m_slot_to_piece[m_piece_to_slot[i]] == i); + if (m_piece_to_slot[i] != i) + { + TORRENT_ASSERT(m_slot_to_piece[i] == unallocated); + } + } + else + { + TORRENT_ASSERT(m_piece_to_slot[i] == has_no_slot); + } + + // do more detailed checks on slot_to_piece + + if (m_slot_to_piece[i] >= 0) + { + TORRENT_ASSERT(m_slot_to_piece[i] < (int)m_piece_to_slot.size()); + TORRENT_ASSERT(m_piece_to_slot[m_slot_to_piece[i]] == i); #ifdef TORRENT_STORAGE_DEBUG - TORRENT_ASSERT( - std::find( - m_unallocated_slots.begin() - , m_unallocated_slots.end() - , i) == m_unallocated_slots.end() - ); - TORRENT_ASSERT( - std::find( - m_free_slots.begin() - , m_free_slots.end() - , i) == m_free_slots.end() - ); + TORRENT_ASSERT( + std::find( + m_unallocated_slots.begin() + , m_unallocated_slots.end() + , i) == m_unallocated_slots.end() + ); + TORRENT_ASSERT( + std::find( + m_free_slots.begin() + , m_free_slots.end() + , i) == m_free_slots.end() + ); #endif - } - else if (m_slot_to_piece[i] == unallocated) - { + } + else if (m_slot_to_piece[i] == unallocated) + { #ifdef TORRENT_STORAGE_DEBUG - TORRENT_ASSERT(m_unallocated_slots.empty() - || (std::find( - m_unallocated_slots.begin() - , m_unallocated_slots.end() - , i) != m_unallocated_slots.end()) - ); + TORRENT_ASSERT(m_unallocated_slots.empty() + || (std::find( + m_unallocated_slots.begin() + , m_unallocated_slots.end() + , i) != m_unallocated_slots.end()) + ); #endif - } - else if (m_slot_to_piece[i] == unassigned) - { + } + else if (m_slot_to_piece[i] == unassigned) + { #ifdef TORRENT_STORAGE_DEBUG - TORRENT_ASSERT( - std::find( - m_free_slots.begin() - , m_free_slots.end() - , i) != m_free_slots.end() - ); + TORRENT_ASSERT( + std::find( + m_free_slots.begin() + , m_free_slots.end() + , i) != m_free_slots.end() + ); #endif - } - else - { - TORRENT_ASSERT(false && "m_slot_to_piece[i] is invalid"); + } + else + { + TORRENT_ASSERT(false && "m_slot_to_piece[i] is invalid"); + } } } } diff --git a/libtorrent/src/torrent.cpp b/libtorrent/src/torrent.cpp index a094fa749..a81f6ba73 100755 --- a/libtorrent/src/torrent.cpp +++ b/libtorrent/src/torrent.cpp @@ -154,7 +154,7 @@ namespace libtorrent , boost::intrusive_ptr tf , fs::path const& save_path , tcp::endpoint const& net_interface - , bool compact_mode + , storage_mode_t storage_mode , int block_size , storage_constructor_type sc , bool paused) @@ -195,7 +195,7 @@ namespace libtorrent , m_total_redundant_bytes(0) , m_net_interface(net_interface.address(), 0) , m_save_path(complete(save_path)) - , m_compact_mode(compact_mode) + , m_storage_mode(storage_mode) , m_default_block_size(block_size) , m_connections_initialized(true) , m_settings(ses.settings()) @@ -215,7 +215,7 @@ namespace libtorrent , char const* name , fs::path const& save_path , tcp::endpoint const& net_interface - , bool compact_mode + , storage_mode_t storage_mode , int block_size , storage_constructor_type sc , bool paused) @@ -255,7 +255,7 @@ namespace libtorrent , m_total_redundant_bytes(0) , m_net_interface(net_interface.address(), 0) , m_save_path(complete(save_path)) - , m_compact_mode(compact_mode) + , m_storage_mode(storage_mode) , m_default_block_size(block_size) , m_connections_initialized(false) , m_settings(ses.settings()) @@ -2215,10 +2215,22 @@ namespace libtorrent bool done = true; try { + std::string error_msg; TORRENT_ASSERT(m_storage); TORRENT_ASSERT(m_owning_storage.get()); done = m_storage->check_fastresume(data, m_have_pieces, m_num_pieces - , m_compact_mode); + , m_storage_mode, error_msg); + + if (!error_msg.empty() && m_ses.m_alerts.should_post(alert::warning)) + { + m_ses.m_alerts.post_alert(fastresume_rejected_alert( + get_handle(), error_msg)); +#if defined(TORRENT_VERBOSE_LOGGING) || defined(TORRENT_LOGGING) + (*m_ses.m_logger) << "fastresume data for " + << torrent_file().name() << " rejected: " + << error_msg << "\n"; +#endif + } } catch (std::exception& e) { @@ -2768,7 +2780,7 @@ namespace libtorrent !boost::bind(&peer_connection::is_connecting , boost::bind(&std::map::value_type::second, _1))); - st.compact_mode = m_compact_mode; + st.storage_mode = m_storage_mode; st.num_complete = m_complete; st.num_incomplete = m_incomplete; diff --git a/libtorrent/src/torrent_handle.cpp b/libtorrent/src/torrent_handle.cpp index f6b5d47de..9418d50e8 100755 --- a/libtorrent/src/torrent_handle.cpp +++ b/libtorrent/src/torrent_handle.cpp @@ -661,7 +661,7 @@ namespace libtorrent if (!t->valid_metadata()) return entry(); - t->filesystem().export_piece_map(piece_index); + std::vector have_pieces = t->pieces(); entry ret(entry::dictionary_t); @@ -673,10 +673,6 @@ namespace libtorrent const sha1_hash& info_hash = t->torrent_file().info_hash(); ret["info-hash"] = std::string((char*)info_hash.begin(), (char*)info_hash.end()); - ret["slots"] = entry(entry::list_t); - entry::list_type& slots = ret["slots"].list(); - std::copy(piece_index.begin(), piece_index.end(), std::back_inserter(slots)); - // blocks per piece int num_blocks_per_piece = static_cast(t->torrent_file().piece_length()) / t->block_size(); @@ -706,6 +702,8 @@ namespace libtorrent // the unfinished piece's index piece_struct["piece"] = i->index; + have_pieces[i->index] = true; + std::string bitmask; const int num_bitmask_bytes = (std::max)(num_blocks_per_piece / 8, 1); @@ -722,10 +720,10 @@ namespace libtorrent } piece_struct["bitmask"] = bitmask; - TORRENT_ASSERT(t->filesystem().slot_for_piece(i->index) >= 0); + TORRENT_ASSERT(t->filesystem().slot_for(i->index) >= 0); unsigned long adler = t->filesystem().piece_crc( - t->filesystem().slot_for_piece(i->index) + t->filesystem().slot_for(i->index) , t->block_size() , i->info); @@ -735,6 +733,11 @@ namespace libtorrent up.push_back(piece_struct); } } + + t->filesystem().export_piece_map(piece_index, have_pieces); + entry::list_type& slots = ret["slots"].list(); + std::copy(piece_index.begin(), piece_index.end(), std::back_inserter(slots)); + // write local peers entry::list_type& peer_list = ret["peers"].list(); diff --git a/src/deluge_core.cpp b/src/deluge_core.cpp index eb0d542f7..93350a0c8 100644 --- a/src/deluge_core.cpp +++ b/src/deluge_core.cpp @@ -242,8 +242,13 @@ boost::filesystem::path const& save_path) // Create new torrent object torrent_t new_torrent; - - torrent_handle h = M_ses->add_torrent(t, save_path, resume_data, compact_mode); + libtorrent::storage_mode_t storage_mode; + if (compact_mode){ + storage_mode = storage_mode_compact; + } else { + storage_mode = storage_mode_sparse; + } + torrent_handle h = M_ses->add_torrent(t, save_path, resume_data, storage_mode); // h.set_max_connections(60); // at some point we should use this h.set_max_uploads(-1); h.set_ratio(preferred_ratio); @@ -893,7 +898,7 @@ static PyObject *torrent_get_torrent_state(PyObject *self, PyObject *args) "num_seeds", connected_seeds, "distributed_copies", s.distributed_copies == -1.0 ? 0.0 : s.distributed_copies, "download_rate", s.download_payload_rate, - "compact_mode", s.compact_mode, + "storage_mode", s.storage_mode, "upload_rate", s.upload_payload_rate, "total_download", s.total_download, "total_upload", s.total_upload,