first attempt at switching over to new storage allocation

libtorrent/include/libtorrent/allocate_resources.hpp

@@ -0,0 +1,78 @@
+/*
+
+Copyright (c) 2003, Magnus Jonsson
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#ifndef TORRENT_ALLOCATE_RESOURCES_HPP_INCLUDED
+#define TORRENT_ALLOCATE_RESOURCES_HPP_INCLUDED
+
+#include <map>
+#include <utility>
+
+#include <boost/shared_ptr.hpp>
+
+#include "libtorrent/resource_request.hpp"
+#include "libtorrent/peer_id.hpp"
+#include "libtorrent/socket.hpp"
+#include "libtorrent/session.hpp"
+
+namespace libtorrent
+{
+	class peer_connection;
+	class torrent;
+
+	int saturated_add(int a, int b);
+
+	// Function to allocate a limited resource fairly among many consumers.
+	// It takes into account the current use, and the consumer's desired use.
+	// Should be invoked periodically to allow it adjust to the situation (make
+	// sure "used" is updated between calls!).
+	// If resources = std::numeric_limits<int>::max() it means there is an infinite
+	// supply of resources (so everyone can get what they want).
+
+	void allocate_resources(
+		int resources
+		, std::map<sha1_hash, boost::shared_ptr<torrent> >& torrents
+		, resource_request torrent::* res);
+
+	void allocate_resources(
+		int resources
+		, std::map<tcp::endpoint, peer_connection*>& connections
+		, resource_request peer_connection::* res);
+
+	// Used for global limits.
+	void allocate_resources(
+		int resources
+		, std::vector<session*>& _sessions
+		, resource_request session::* res);
+}
+
+
+#endif

libtorrent/include/libtorrent/aux_/allocate_resources_impl.hpp

@@ -0,0 +1,328 @@
+/*
+
+Copyright (c) 2003, Magnus Jonsson
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#ifndef TORRENT_ALLOCATE_RESOURCES_IMPL_HPP_INCLUDED
+#define TORRENT_ALLOCATE_RESOURCES_IMPL_HPP_INCLUDED
+
+#include <map>
+#include <utility>
+
+#include <boost/shared_ptr.hpp>
+
+#include "libtorrent/resource_request.hpp"
+#include "libtorrent/peer_id.hpp"
+#include "libtorrent/socket.hpp"
+#include "libtorrent/size_type.hpp"
+
+#ifdef min
+#undef min
+#endif
+
+#ifdef max
+#undef max
+#endif
+
+namespace libtorrent
+{
+
+	int saturated_add(int a, int b);
+
+	namespace aux
+	{
+		// give num_resources to r,
+		// return how how many were actually accepted.
+		inline int give(resource_request& r, int num_resources)
+		{
+			assert(num_resources >= 0);
+			assert(r.given <= r.max);
+			
+			int accepted = (std::min)(num_resources, r.max - r.given);
+			assert(accepted >= 0);
+
+			r.given += accepted;
+			assert(r.given <= r.max);
+
+			return accepted;
+		}
+
+		inline int div_round_up(int numerator, int denominator)
+		{
+			return (numerator + denominator - 1) / denominator;
+		}
+
+#ifndef NDEBUG
+
+		template<class It, class T>
+		class allocate_resources_contract_check
+		{
+			int m_resources;
+			It m_start;
+			It m_end;
+			resource_request T::* m_res;
+
+		public:
+			allocate_resources_contract_check(
+				int resources
+				, It start
+				, It end
+				, resource_request T::* res)
+				: m_resources(resources)
+				, m_start(start)
+				, m_end(end)
+				, m_res(res)
+			{
+				assert(m_resources >= 0);
+				for (It i = m_start, end(m_end); i != end; ++i)
+				{
+					assert(((*i).*m_res).max >= 0);
+					assert(((*i).*m_res).given >= 0);
+				}
+			}
+
+			~allocate_resources_contract_check()
+			{
+				int sum_given = 0;
+				int sum_max = 0;
+				int sum_min = 0;
+				for (It i = m_start, end(m_end); i != end; ++i)
+				{
+					assert(((*i).*m_res).max >= 0);
+					assert(((*i).*m_res).min >= 0);
+					assert(((*i).*m_res).max >= ((*i).*m_res).min);
+					assert(((*i).*m_res).given >= 0);
+					assert(((*i).*m_res).given <= ((*i).*m_res).max);
+
+					sum_given = saturated_add(sum_given, ((*i).*m_res).given);
+					sum_max = saturated_add(sum_max, ((*i).*m_res).max);
+					sum_min = saturated_add(sum_min, ((*i).*m_res).min);
+				}
+				if (sum_given != (std::min)(std::max(m_resources, sum_min), sum_max))
+				{
+					std::cerr << sum_given << " " << m_resources << " " << sum_min << " " << sum_max << std::endl;
+					assert(false);
+				}
+			}
+		};
+
+#endif
+
+		template<class It, class T>
+		void allocate_resources_impl(
+			int resources
+			, It start
+			, It end
+			, resource_request T::* res)
+		{
+			assert(resources >= 0);
+	#ifndef NDEBUG
+			allocate_resources_contract_check<It, T> contract_check(
+				resources
+				, start
+				, end
+				, res);
+	#endif
+
+			for (It i = start; i != end; ++i)
+			{
+				resource_request& r = (*i).*res;
+				r.leftovers = (std::max)(r.used - r.given, 0);
+			}
+
+			if (resources == resource_request::inf)
+			{
+				// No competition for resources.
+				// Just give everyone what they want.
+				for (It i = start; i != end; ++i)
+				{
+					((*i).*res).given = ((*i).*res).max;
+				}
+				return;
+			}
+
+			// Resources are scarce
+
+			int sum_max = 0;
+			int sum_min = 0;
+			// the number of consumer that saturated their
+			// quota last time slice
+			int num_saturated = 0;
+			// the total resources that those saturated their
+			// quota used. This is used to calculate the mean
+			// of the saturating consumers, in order to
+			// balance their quotas for the next time slice.
+			size_type saturated_sum = 0;
+			for (It i = start; i != end; ++i)
+			{
+				resource_request& r = (*i).*res;
+				sum_max = saturated_add(sum_max, r.max);
+				assert(r.min < resource_request::inf);
+				assert(r.min >= 0);
+				assert(r.min <= r.max);
+				sum_min += r.min;
+				
+				// a consumer that uses 95% or more of its assigned
+				// quota is considered saturating
+				size_type used = r.used;
+				if (r.given == 0) continue;
+				if (used * 20 / r.given >= 19)
+				{
+					++num_saturated;
+					saturated_sum += r.given;
+				}
+			}
+
+			if (sum_max <= resources)
+			{
+				// it turns out that there's no competition for resources
+				// after all.
+				for (It i = start; i != end; ++i)
+				{
+					((*i).*res).given = ((*i).*res).max;
+				}
+				return;
+			}
+
+			if (sum_min >= resources)
+			{
+				// the amount of resources is smaller than
+				// the minimum resources to distribute, so
+				// give everyone the minimum
+				for (It i = start; i != end; ++i)
+				{
+					((*i).*res).given = ((*i).*res).min;
+				}
+				return;
+			}
+
+			// now, the "used" field will be used as a target value.
+			// the algorithm following this loop will then scale the
+			// used values to fit the available resources and store
+			// the scaled values as given. So, the ratios of the
+			// used values will be maintained.
+			for (It i = start; i != end; ++i)
+			{
+				resource_request& r = (*i).*res;
+				
+				int target;
+				size_type used = r.used;
+				if (r.given > 0 && used * 20 / r.given >= 19)
+				{
+					assert(num_saturated > 0);
+					target = div_round_up(saturated_sum, num_saturated);
+					target += div_round_up(target, 10);
+				}
+				else
+				{
+					target = r.used;
+				}
+				if (target > r.max) target = r.max;
+				else if (target < r.min) target = r.min;
+
+				// move 12.5% towards the the target value
+				r.used = r.given + div_round_up(target - r.given, 8);
+				r.given = r.min;
+			}
+
+
+			resources = (std::max)(resources, sum_min);
+			int resources_to_distribute = (std::min)(resources, sum_max) - sum_min;
+			assert(resources_to_distribute >= 0);
+#ifndef NDEBUG
+			int prev_resources_to_distribute = resources_to_distribute;
+#endif
+			while (resources_to_distribute > 0)
+			{
+				// in order to scale, we need to calculate the sum of
+				// all the used values.
+				size_type total_used = 0;
+				size_type max_used = 0;
+				for (It i = start; i != end; ++i)
+				{
+					resource_request& r = (*i).*res;
+					if (r.given == r.max) continue;
+
+					assert(r.given < r.max);
+
+					max_used = (std::max)(max_used, (size_type)r.used + 1);
+					total_used += (size_type)r.used + 1;
+				}
+
+
+				size_type kNumer = resources_to_distribute;
+				size_type kDenom = total_used;
+				assert(kNumer >= 0);
+				assert(kDenom >= 0);
+				assert(kNumer <= (std::numeric_limits<int>::max)());
+
+				if (kNumer * max_used <= kDenom)
+				{
+					kNumer = 1;
+					kDenom = max_used;
+					assert(kDenom >= 0);
+				}
+
+				for (It i = start; i != end && resources_to_distribute > 0; ++i)
+				{
+					resource_request& r = (*i).*res;
+					if (r.given == r.max) continue;
+
+					assert(r.given < r.max);
+
+					size_type used = (size_type)r.used + 1;
+					if (used < 1) used = 1;
+					size_type to_give = used * kNumer / kDenom;
+					if (to_give > resources_to_distribute)
+						to_give = resources_to_distribute;
+					assert(to_give >= 0);
+					assert(to_give <= resources_to_distribute);
+#ifndef NDEBUG
+					int tmp = resources_to_distribute;
+#endif
+					resources_to_distribute -= give(r, (int)to_give);
+					assert(resources_to_distribute <= tmp);
+					assert(resources_to_distribute >= 0);
+				}
+
+				assert(resources_to_distribute >= 0);
+				assert(resources_to_distribute < prev_resources_to_distribute);
+#ifndef NDEBUG
+				prev_resources_to_distribute = resources_to_distribute;
+#endif
+			}
+			assert(resources_to_distribute == 0);
+		}
+
+	} // namespace libtorrent::aux
+}
+
+
+#endif

libtorrent/include/libtorrent/aux_/session_impl.hpp

@@ -254,7 +254,7 @@ namespace libtorrent
 				boost::intrusive_ptr<torrent_info> ti
 				, fs::path const& save_path
 				, entry const& resume_data
-				, bool compact_mode
+				, storage_mode_t storage_mode
 				, storage_constructor_type sc
 				, bool paused
 				, void* userdata);
@@ -265,7 +265,7 @@ namespace libtorrent
 				, char const* name
 				, fs::path const& save_path
 				, entry const& resume_data
-				, bool compact_mode
+				, storage_mode_t storage_mode
 				, storage_constructor_type sc
 				, bool paused
 				, void* userdata);

libtorrent/include/libtorrent/resource_request.hpp

@@ -0,0 +1,99 @@
+/*
+
+Copyright (c) 2003, Magnus Jonsson, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/
+
+#ifndef TORRENT_RESOURCE_REQUEST_HPP_INCLUDED
+#define TORRENT_RESOURCE_REQUEST_HPP_INCLUDED
+
+#include <boost/integer_traits.hpp>
+
+#ifdef min
+#undef min
+#endif
+
+#ifdef max
+#undef max
+#endif
+
+#include "libtorrent/config.hpp"
+
+namespace libtorrent
+{
+	struct TORRENT_EXPORT resource_request
+	{
+		resource_request()
+			: used(0)
+			, min(0)
+			, max(0)
+			, given(0)
+			, leftovers(0)
+		{}
+
+		resource_request(int used_, int min_, int max_, int given_)
+			: used(used_)
+			, min(min_)
+			, max(max_)
+			, given(given_)
+			, leftovers(0)
+		{}
+
+		int left() const
+		{
+			assert(given <= max);
+			assert(given >= min);
+			assert(used >= 0);
+			return (std::max)(given - used, 0);
+		}
+		
+		void reset() { used = leftovers; leftovers = 0; }
+
+		static const int inf = boost::integer_traits<int>::const_max;
+
+		// right now I'm actively using this amount
+		int used;
+
+		// given cannot be smaller than min
+		// and not greater than max.
+		int min;
+		int max;
+
+		// Reply: Okay, you're allowed to use this amount (a compromise):
+		int given;
+
+		// this is the amount of resources that exceeded the
+		// given limit. When the used field is reset (after resources
+		// have been distributed), it is reset to this number.
+		int leftovers;
+	};
+}
+
+
+#endif

libtorrent/include/libtorrent/session.hpp

@@ -140,7 +140,7 @@ namespace libtorrent
 			torrent_info const& ti
 			, fs::path const& save_path
 			, entry const& resume_data = entry()
-			, bool compact_mode = true
+			, storage_mode_t storage_mode = storage_mode_sparse
 			, bool paused = false
 			, storage_constructor_type sc = default_storage_constructor) TORRENT_DEPRECATED;
 
@@ -148,7 +148,7 @@ namespace libtorrent
 			boost::intrusive_ptr<torrent_info> ti
 			, fs::path const& save_path
 			, entry const& resume_data = entry()
-			, bool compact_mode = true
+			, storage_mode_t storage_mode = storage_mode_sparse
 			, bool paused = false
 			, storage_constructor_type sc = default_storage_constructor
 			, void* userdata = 0);
@@ -159,7 +159,7 @@ namespace libtorrent
 			, char const* name
 			, fs::path const& save_path
 			, entry const& resume_data = entry()
-			, bool compact_mode = true
+			, storage_mode_t storage_mode = storage_mode_sparse
 			, bool paused = false
 			, storage_constructor_type sc = default_storage_constructor
 			, void* userdata = 0);

libtorrent/include/libtorrent/storage.hpp

@@ -71,6 +71,13 @@ namespace libtorrent
 	struct file_pool;
 	struct disk_io_job;
 
+	enum storage_mode_t
+	{
+		storage_mode_allocate = 0,
+		storage_mode_sparse,
+		storage_mode_compact
+	};
+	
 #if defined(_WIN32) && defined(UNICODE)
 
 	TORRENT_EXPORT std::wstring safe_convert(std::string const& s);
@@ -180,7 +187,8 @@ namespace libtorrent
 		~piece_manager();
 
 		bool check_fastresume(aux::piece_checker_data& d
-			, std::vector<bool>& pieces, int& num_pieces, bool compact_mode);
+			, std::vector<bool>& pieces, int& num_pieces, storage_mode_t storage_mode
+			, std::string& error_msg);
 		std::pair<bool, float> check_files(std::vector<bool>& pieces
 			, int& num_pieces, boost::recursive_mutex& mutex);
 
@@ -191,7 +199,7 @@ namespace libtorrent
 		bool verify_resume_data(entry& rd, std::string& error);
 
 		bool is_allocating() const
-		{ return m_state == state_allocating; }
+		{ return m_state == state_expand_pieces; }
 
 		void mark_failed(int index);
 
@@ -200,7 +208,8 @@ namespace libtorrent
 			, int block_size
 			, piece_picker::block_info const* bi);
 
-		int slot_for_piece(int piece_index) const;
+		int slot_for(int piece) const;
+		int piece_for(int slot) const;
 		
 		void async_read(
 			peer_request const& r
@@ -228,10 +237,11 @@ namespace libtorrent
 		// slots to the piece that is stored (or
 		// partially stored) there. -2 is the index
 		// of unassigned pieces and -1 is unallocated
-		void export_piece_map(std::vector<int>& pieces) const;
+		void export_piece_map(std::vector<int>& pieces
+			, std::vector<bool> const& have) const;
 
 		bool compact_allocation() const
-		{ return m_compact_mode; }
+		{ return m_storage_mode == storage_mode_compact; }
 
 #ifndef NDEBUG
 		std::string name() const { return m_info->name(); }
@@ -261,6 +271,7 @@ namespace libtorrent
 			, int offset
 			, int size);
 
+		void switch_to_full_mode();
 		sha1_hash hash_for_piece_impl(int piece);
 
 		void release_files_impl();
@@ -276,16 +287,7 @@ namespace libtorrent
 #endif
 		boost::scoped_ptr<storage_interface> m_storage;
 
-		// if this is true, pieces are always allocated at the
+		storage_mode_t m_storage_mode;
-		// lowest possible slot index. If it is false, pieces
-		// are always written to their final place immediately
-		bool m_compact_mode;
-
-		// if this is true, pieces that haven't been downloaded
-		// will be filled with zeroes. Not filling with zeroes
-		// will not work in some cases (where a seek cannot pass
-		// the end of the file).
-		bool m_fill_mode;
 
 		// a bitmask representing the pieces we have
 		std::vector<bool> m_have_piece;
@@ -329,10 +331,21 @@ namespace libtorrent
 			state_create_files,
 			// checking the files
 			state_full_check,
-			// allocating files (in non-compact mode)
+			// move pieces to their final position
-			state_allocating
+			state_expand_pieces
 		} m_state;
 		int m_current_slot;
+		// used during check. If any piece is found
+		// that is not in its final position, this
+		// is set to true
+		bool m_out_of_place;
+		// used to move pieces while expanding
+		// the storage from compact allocation
+		// to full allocation
+		std::vector<char> m_scratch_buffer;
+		std::vector<char> m_scratch_buffer2;
+		// the piece that is in the scratch buffer
+		int m_scratch_piece;
 		
 		// this is saved in case we need to instantiate a new
 		// storage (osed when remapping files)

libtorrent/include/libtorrent/torrent.hpp

@@ -101,7 +101,7 @@ namespace libtorrent
 			, boost::intrusive_ptr<torrent_info> tf
 			, fs::path const& save_path
 			, tcp::endpoint const& net_interface
-			, bool compact_mode
+			, storage_mode_t m_storage_mode
 			, int block_size
 			, storage_constructor_type sc
 			, bool paused);
@@ -116,7 +116,7 @@ namespace libtorrent
 			, char const* name
 			, fs::path const& save_path
 			, tcp::endpoint const& net_interface
-			, bool compact_mode
+			, storage_mode_t m_storage_mode
 			, int block_size
 			, storage_constructor_type sc
 			, bool paused);
@@ -751,7 +751,7 @@ namespace libtorrent
 		fs::path m_save_path;
 
 		// determines the storage state for this torrent.
-		const bool m_compact_mode;
+		storage_mode_t m_storage_mode;
 
 		// defaults to 16 kiB, but can be set by the user
 		// when creating the torrent

libtorrent/include/libtorrent/torrent_handle.hpp

@@ -52,6 +52,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "libtorrent/torrent_info.hpp"
 #include "libtorrent/time.hpp"
 #include "libtorrent/config.hpp"
+#include "libtorrent/storage.hpp"
 
 namespace libtorrent
 {
@@ -106,7 +107,7 @@ namespace libtorrent
 			, num_connections(0)
 			, uploads_limit(0)
 			, connections_limit(0)
-			, compact_mode(false)
+			, storage_mode(storage_mode_sparse)
 		{}
 
 		enum state_t
@@ -216,7 +217,7 @@ namespace libtorrent
 
 		// true if the torrent is saved in compact mode
 		// false if it is saved in full allocation mode
-		bool compact_mode;
+		storage_mode_t storage_mode;
 	};
 
 	struct TORRENT_EXPORT block_info

libtorrent/src/allocate_resources.cpp

@@ -0,0 +1,225 @@
+/*
+
+Copyright (c) 2006, Magnus Jonsson, Arvid Norberg
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the distribution.
+    * Neither the name of the author nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+*/ 
+
+//The Standard Library defines the two template functions std::min() 
+//and std::max() in the <algorithm> header. In general, you should 
+//use these template functions for calculating the min and max values 
+//of a pair. Unfortunately, Visual C++ does not define these function
+// templates. This is because the names min and max clash with 
+//the traditional min and max macros defined in <windows.h>. 
+//As a workaround, Visual C++ defines two alternative templates with 
+//identical functionality called _cpp_min() and _cpp_max(). You can 
+//use them instead of std::min() and std::max().To disable the 
+//generation of the min and max macros in Visual C++, #define 
+//NOMINMAX before #including <windows.h>.
+
+#include "libtorrent/pch.hpp"
+
+#ifdef _WIN32
+    //support boost1.32.0(2004-11-19 18:47)
+    //now all libs can be compiled and linked with static module
+	#define NOMINMAX
+#endif
+
+#include "libtorrent/allocate_resources.hpp"
+#include "libtorrent/size_type.hpp"
+#include "libtorrent/peer_connection.hpp"
+#include "libtorrent/torrent.hpp"
+#include "libtorrent/aux_/allocate_resources_impl.hpp"
+
+#include <cassert>
+#include <algorithm>
+#include <boost/limits.hpp>
+
+#if defined(_MSC_VER) && _MSC_VER < 1310
+#define for if (false) {} else for
+#else
+#include <boost/iterator/transform_iterator.hpp>
+#endif
+
+namespace libtorrent
+{
+	int saturated_add(int a, int b)
+	{
+		assert(a >= 0);
+		assert(b >= 0);
+		assert(a <= resource_request::inf);
+		assert(b <= resource_request::inf);
+		assert(resource_request::inf + resource_request::inf < 0);
+
+		unsigned int sum = unsigned(a) + unsigned(b);
+		if (sum > unsigned(resource_request::inf))
+			sum = resource_request::inf;
+
+		assert(sum >= unsigned(a) && sum >= unsigned(b));
+		return int(sum);
+	}
+
+#if defined(_MSC_VER) && _MSC_VER < 1310
+
+	namespace detail
+	{
+		struct iterator_wrapper
+		{
+			typedef std::map<sha1_hash, boost::shared_ptr<torrent> >::iterator orig_iter;
+
+			orig_iter iter;
+
+			iterator_wrapper(orig_iter i): iter(i) {}
+			void operator++() { ++iter; }
+			torrent& operator*() { return *(iter->second); }
+			bool operator==(const iterator_wrapper& i) const
+			{ return iter == i.iter; }
+			bool operator!=(const iterator_wrapper& i) const
+			{ return iter != i.iter; }
+		};
+
+		struct iterator_wrapper2
+		{
+			typedef std::map<tcp::endpoint, peer_connection*>::iterator orig_iter;
+
+			orig_iter iter;
+
+			iterator_wrapper2(orig_iter i): iter(i) {}
+			void operator++() { ++iter; }
+			peer_connection& operator*() { return *(iter->second); }
+			bool operator==(const iterator_wrapper2& i) const
+			{ return iter == i.iter; }
+			bool operator!=(const iterator_wrapper2& i) const
+			{ return iter != i.iter; }
+		};
+
+	}
+
+	void allocate_resources(
+		int resources
+		, std::map<sha1_hash, boost::shared_ptr<torrent> >& c
+		, resource_request torrent::* res)
+	{
+		aux::allocate_resources_impl(
+			resources
+			, detail::iterator_wrapper(c.begin())
+			, detail::iterator_wrapper(c.end())
+			, res);
+	}
+
+	void allocate_resources(
+		int resources
+		, std::map<tcp::endpoint, peer_connection*>& c
+		, resource_request peer_connection::* res)
+	{
+		aux::allocate_resources_impl(
+			resources
+			, detail::iterator_wrapper2(c.begin())
+			, detail::iterator_wrapper2(c.end())
+			, res);
+	}
+
+#else
+
+	namespace aux
+	{
+		peer_connection& pick_peer(
+			std::pair<boost::shared_ptr<stream_socket>
+			, boost::intrusive_ptr<peer_connection> > const& p)
+		{
+			return *p.second;
+		}
+
+		peer_connection& pick_peer2(
+			std::pair<tcp::endpoint, peer_connection*> const& p)
+		{
+			return *p.second;
+		}
+
+		torrent& deref(std::pair<sha1_hash, boost::shared_ptr<torrent> > const& p)
+		{
+			return *p.second;
+		}
+
+		session& deref(session* p)
+		{
+			return *p;
+		}
+	}
+
+	void allocate_resources(
+		int resources
+		, std::map<sha1_hash, boost::shared_ptr<torrent> >& c
+		, resource_request torrent::* res)
+	{
+		typedef std::map<sha1_hash, boost::shared_ptr<torrent> >::iterator orig_iter;
+		typedef std::pair<sha1_hash, boost::shared_ptr<torrent> > in_param;
+		typedef boost::transform_iterator<torrent& (*)(in_param const&), orig_iter> new_iter;
+
+		aux::allocate_resources_impl(
+			resources
+			, new_iter(c.begin(), &aux::deref)
+			, new_iter(c.end(), &aux::deref)
+			, res);
+	}
+
+	void allocate_resources(
+		int resources
+		, std::map<tcp::endpoint, peer_connection*>& c
+		, resource_request peer_connection::* res)
+	{
+		typedef std::map<tcp::endpoint, peer_connection*>::iterator orig_iter;
+		typedef std::pair<tcp::endpoint, peer_connection*> in_param;
+		typedef boost::transform_iterator<peer_connection& (*)(in_param const&), orig_iter> new_iter;
+
+		aux::allocate_resources_impl(
+			resources
+			, new_iter(c.begin(), &aux::pick_peer2)
+			, new_iter(c.end(), &aux::pick_peer2)
+			, res);
+	}
+
+	void allocate_resources(
+		int resources
+		, std::vector<session*>& _sessions
+		, resource_request session::* res)
+	{
+		typedef std::vector<session*>::iterator orig_iter;
+		typedef session* in_param;
+		typedef boost::transform_iterator<session& (*)(in_param), orig_iter> new_iter;
+
+		aux::allocate_resources_impl(
+			resources
+			, new_iter(_sessions.begin(), &aux::deref)
+			, new_iter(_sessions.end(), &aux::deref)
+			, res);
+	}
+
+#endif
+
+} // namespace libtorrent

libtorrent/src/piece_picker.cpp

@@ -1732,7 +1732,6 @@ namespace libtorrent
 		++i->writing;
 		info.state = block_info::state_writing;
 		if (info.num_peers > 0) --info.num_peers;
-		TORRENT_ASSERT(info.num_peers >= 0);
 
 		if (i->requested == 0)
 		{
@@ -1855,7 +1854,6 @@ namespace libtorrent
 
 		block_info& info = i->info[block.block_index];
 		--info.num_peers;
-		TORRENT_ASSERT(info.num_peers >= 0);
 		if (info.num_peers > 0) return;
 
 		if (i->info[block.block_index].state == block_info::state_finished

libtorrent/src/session.cpp

@@ -186,28 +186,28 @@ namespace libtorrent
 		torrent_info const& ti
 		, fs::path const& save_path
 		, entry const& resume_data
-		, bool compact_mode
+		, storage_mode_t storage_mode
 		, bool paused
 		, storage_constructor_type sc)
 	{
 		TORRENT_ASSERT(!ti.m_half_metadata);
 		boost::intrusive_ptr<torrent_info> tip(new torrent_info(ti));
 		return m_impl->add_torrent(tip, save_path, resume_data
-			, compact_mode, sc, paused, 0);
+			, storage_mode, sc, paused, 0);
 	}
 
 	torrent_handle session::add_torrent(
 		boost::intrusive_ptr<torrent_info> ti
 		, fs::path const& save_path
 		, entry const& resume_data
-		, bool compact_mode
+		, storage_mode_t storage_mode
 		, bool paused
 		, storage_constructor_type sc
 		, void* userdata)
 	{
 		TORRENT_ASSERT(!ti->m_half_metadata);
 		return m_impl->add_torrent(ti, save_path, resume_data
-			, compact_mode, sc, paused, userdata);
+			, storage_mode, sc, paused, userdata);
 	}
 
 	torrent_handle session::add_torrent(
@@ -216,13 +216,13 @@ namespace libtorrent
 		, char const* name
 		, fs::path const& save_path
 		, entry const& e
-		, bool compact_mode
+		, storage_mode_t storage_mode
 		, bool paused
 		, storage_constructor_type sc
 		, void* userdata)
 	{
 		return m_impl->add_torrent(tracker_url, info_hash, name, save_path, e
-			, compact_mode, sc, paused, userdata);
+			, storage_mode, sc, paused, userdata);
 	}
 
 	void session::remove_torrent(const torrent_handle& h)

libtorrent/src/session_impl.cpp

@@ -189,9 +189,11 @@ namespace detail
 					t->parse_resume_data(t->resume_data, t->torrent_ptr->torrent_file()
 						, error_msg);
 
+					// lock the session to add the new torrent
+					session_impl::mutex_t::scoped_lock l(m_ses.m_mutex);
+
 					if (!error_msg.empty() && m_ses.m_alerts.should_post(alert::warning))
 					{
-						session_impl::mutex_t::scoped_lock l(m_ses.m_mutex);
 						m_ses.m_alerts.post_alert(fastresume_rejected_alert(
 							t->torrent_ptr->get_handle()
 							, error_msg));
@@ -202,8 +204,6 @@ namespace detail
 #endif
 					}
 
-					// lock the session to add the new torrent
-					session_impl::mutex_t::scoped_lock l(m_ses.m_mutex);
 					mutex::scoped_lock l2(m_mutex);
 
 					if (m_torrents.empty() || m_torrents.front() != t)
@@ -1623,7 +1623,7 @@ namespace detail
 		boost::intrusive_ptr<torrent_info> ti
 		, fs::path const& save_path
 		, entry const& resume_data
-		, bool compact_mode
+		, storage_mode_t storage_mode
 		, storage_constructor_type sc
 		, bool paused
 		, void* userdata)
@@ -1655,7 +1655,7 @@ namespace detail
 		// the thread
 		boost::shared_ptr<torrent> torrent_ptr(
 			new torrent(*this, m_checker_impl, ti, save_path
-				, m_listen_interface, compact_mode, 16 * 1024
+				, m_listen_interface, storage_mode, 16 * 1024
 				, sc, paused));
 		torrent_ptr->start();
 
@@ -1701,7 +1701,7 @@ namespace detail
 		, char const* name
 		, fs::path const& save_path
 		, entry const&
-		, bool compact_mode
+		, storage_mode_t storage_mode
 		, storage_constructor_type sc
 		, bool paused
 		, void* userdata)
@@ -1735,7 +1735,7 @@ namespace detail
 		// the thread
 		boost::shared_ptr<torrent> torrent_ptr(
 			new torrent(*this, m_checker_impl, tracker_url, info_hash, name
-			, save_path, m_listen_interface, compact_mode, 16 * 1024
+			, save_path, m_listen_interface, storage_mode, 16 * 1024
 			, sc, paused));
 		torrent_ptr->start();
 

libtorrent/src/storage.cpp

@@ -447,7 +447,7 @@ namespace libtorrent
 			}
 
 			// if the file is empty, just create it. But also make sure
-			// the directory exits.
+			// the directory exists.
 			if (file_iter->size == 0)
 			{
 				file(m_save_path / file_iter->path, file::out);
@@ -931,107 +931,6 @@ namespace libtorrent
 		return new storage(ti, path, fp);
 	}
 
-	bool supports_sparse_files(fs::path const& p)
-	{
-		TORRENT_ASSERT(p.is_complete());
-#if defined(_WIN32)
-		// assume windows API is available
-		DWORD max_component_len = 0;
-		DWORD volume_flags = 0;
-		std::string root_device = p.root_name() + "\\";
-#if defined(UNICODE)
-		std::wstring wph(safe_convert(root_device));
-		bool ret = ::GetVolumeInformation(wph.c_str(), 0
-			, 0, 0, &max_component_len, &volume_flags, 0, 0);
-#else
-		bool ret = ::GetVolumeInformation(root_device.c_str(), 0
-			, 0, 0, &max_component_len, &volume_flags, 0, 0);
-#endif
-
-		if (!ret) return false;
-		if (volume_flags & FILE_SUPPORTS_SPARSE_FILES)
-			return true;
-#endif
-
-#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__)
-		// find the last existing directory of the save path
-		fs::path query_path = p;
-		while (!query_path.empty() && !exists(query_path))
-			query_path = query_path.branch_path();
-#endif
-
-#if defined(__APPLE__)
-
-		struct statfs fsinfo;
-		int ret = statfs(query_path.native_directory_string().c_str(), &fsinfo);
-		if (ret != 0) return false;
-
-		attrlist request;
-		request.bitmapcount = ATTR_BIT_MAP_COUNT;
-		request.reserved = 0;
-		request.commonattr = 0;
-		request.volattr = ATTR_VOL_CAPABILITIES;
-		request.dirattr = 0;
-		request.fileattr = 0;
-		request.forkattr = 0;
-	
-		struct vol_capabilities_attr_buf
-		{
-			unsigned long length;
-			vol_capabilities_attr_t info;
-		} vol_cap;
-
-		ret = getattrlist(fsinfo.f_mntonname, &request, &vol_cap
-			, sizeof(vol_cap), 0);
-		if (ret != 0) return false;
-
-		if (vol_cap.info.capabilities[VOL_CAPABILITIES_FORMAT]
-			& (VOL_CAP_FMT_SPARSE_FILES | VOL_CAP_FMT_ZERO_RUNS))
-		{
-			return true;
-		}
-
-		// workaround for bugs in Mac OS X where zero run is not reported
-		if (!strcmp(fsinfo.f_fstypename, "hfs")
-			|| !strcmp(fsinfo.f_fstypename, "ufs"))
-			return true;
-
-		return false;
-#endif
-
-#if defined(__linux__) || defined(__FreeBSD__)
-		struct statfs buf;
-		int err = statfs(query_path.native_directory_string().c_str(), &buf);
-		if (err == 0)
-		{
-			switch (buf.f_type)
-			{
-				case 0x5346544e: // NTFS
-				case 0xEF51: // EXT2 OLD
-				case 0xEF53: // EXT2 and EXT3
-				case 0x00011954: // UFS
-				case 0x52654973: // ReiserFS
-				case 0x52345362: // Reiser4
-				case 0x58465342: // XFS
-				case 0x65735546: // NTFS-3G
-				case 0x19540119: // UFS2
-					return true;
-			}
-		}
-#ifndef NDEBUG
-		else
-		{
-			std::cerr << "statfs returned " << err << std::endl;
-			std::cerr << "errno: " << errno << std::endl;
-			std::cerr << "path: " << query_path.native_directory_string() << std::endl;
-		}
-#endif
-#endif
-
-		// TODO: POSIX implementation
-		return false;
-	}
-
 	// -- piece_manager -----------------------------------------------------
 
 	piece_manager::piece_manager(
@@ -1042,15 +941,16 @@ namespace libtorrent
 		, disk_io_thread& io
 		, storage_constructor_type sc)
 		: m_storage(sc(ti, save_path, fp))
-		, m_compact_mode(false)
+		, m_storage_mode(storage_mode_sparse)
-		, m_fill_mode(true)
 		, m_info(ti)
 		, m_save_path(complete(save_path))
+		, m_current_slot(0)
+		, m_out_of_place(false)
+		, m_scratch_piece(-1)
 		, m_storage_constructor(sc)
 		, m_io_thread(io)
 		, m_torrent(torrent)
 	{
-		m_fill_mode = !supports_sparse_files(save_path);
 	}
 
 	piece_manager::~piece_manager()
@@ -1158,7 +1058,7 @@ namespace libtorrent
 			m_piece_hasher.erase(i);
 		}
 
-		int slot = m_piece_to_slot[piece];
+		int slot = slot_for(piece);
 		TORRENT_ASSERT(slot != has_no_slot);
 		return m_storage->hash_for_slot(slot, ph, m_info->piece_size(piece));
 	}
@@ -1178,25 +1078,37 @@ namespace libtorrent
 		return false;
 	}
 	void piece_manager::export_piece_map(
-			std::vector<int>& p) const
+			std::vector<int>& p, std::vector<bool> const& have) const
 	{
 		boost::recursive_mutex::scoped_lock lock(m_mutex);
 
 		INVARIANT_CHECK;
 
+		if (m_storage_mode == storage_mode_compact)
+		{
 			p.clear();
+			p.reserve(m_info->num_pieces());
 			std::vector<int>::const_reverse_iterator last; 
 			for (last = m_slot_to_piece.rbegin();
 				last != m_slot_to_piece.rend(); ++last)
 			{
-			if (*last != unallocated) break;
+				if (*last != unallocated && have[*last]) break;
 			}
 
 			for (std::vector<int>::const_iterator i =
 				m_slot_to_piece.begin();
 				i != last.base(); ++i)
 			{
-			p.push_back(*i);
+				p.push_back(have[*i] ? *i : unassigned);
+			}
+		}
+		else
+		{
+			p.reserve(m_info->num_pieces());
+			for (int i = 0; i < m_info->num_pieces(); ++i)
+			{
+				p.push_back(have[i] ? i : unassigned);
+			}
 		}
 	}
 
@@ -1206,11 +1118,10 @@ namespace libtorrent
 
 		INVARIANT_CHECK;
 
+		if (m_storage_mode != storage_mode_compact) return;
+
 		TORRENT_ASSERT(piece_index >= 0 && piece_index < (int)m_piece_to_slot.size());
-		TORRENT_ASSERT(m_piece_to_slot[piece_index] >= 0);
-
 		int slot_index = m_piece_to_slot[piece_index];
-
 		TORRENT_ASSERT(slot_index >= 0);
 
 		m_slot_to_piece[slot_index] = unassigned;
@@ -1218,12 +1129,6 @@ namespace libtorrent
 		m_free_slots.push_back(slot_index);
 	}
 
-	int piece_manager::slot_for_piece(int piece_index) const
-	{
-		TORRENT_ASSERT(piece_index >= 0 && piece_index < m_info->num_pieces());
-		return m_piece_to_slot[piece_index];
-	}
-
 	unsigned long piece_manager::piece_crc(
 		int slot_index
 		, int block_size
@@ -1275,11 +1180,7 @@ namespace libtorrent
 		TORRENT_ASSERT(buf);
 		TORRENT_ASSERT(offset >= 0);
 		TORRENT_ASSERT(size > 0);
-		TORRENT_ASSERT(piece_index >= 0 && piece_index < (int)m_piece_to_slot.size());
+		int slot = slot_for(piece_index);
-		TORRENT_ASSERT(m_piece_to_slot[piece_index] >= 0
-			&& m_piece_to_slot[piece_index] < (int)m_slot_to_piece.size());
-		int slot = m_piece_to_slot[piece_index];
-		TORRENT_ASSERT(slot >= 0 && slot < (int)m_slot_to_piece.size());
 		return m_storage->read(buf, slot, offset, size);
 	}
 
@@ -1292,7 +1193,7 @@ namespace libtorrent
 		TORRENT_ASSERT(buf);
 		TORRENT_ASSERT(offset >= 0);
 		TORRENT_ASSERT(size > 0);
-		TORRENT_ASSERT(piece_index >= 0 && piece_index < (int)m_piece_to_slot.size());
+		TORRENT_ASSERT(piece_index >= 0 && piece_index < m_info->num_pieces());
 
 		if (offset == 0)
 		{
@@ -1317,7 +1218,6 @@ namespace libtorrent
 		}
 		
 		int slot = allocate_slot_for_piece(piece_index);
-		TORRENT_ASSERT(slot >= 0 && slot < (int)m_slot_to_piece.size());
 		m_storage->write(buf, slot, offset, size);
 	}
 
@@ -1426,6 +1326,7 @@ namespace libtorrent
 					// that piece as unassigned, since this slot
 					// is the correct place for the piece.
 					m_slot_to_piece[other_slot] = unassigned;
+					if (m_storage_mode == storage_mode_compact)
 						m_free_slots.push_back(other_slot);
 				}
 				TORRENT_ASSERT(m_piece_to_slot[piece_index] != current_slot);
@@ -1485,7 +1386,8 @@ namespace libtorrent
 	bool piece_manager::check_fastresume(
 		aux::piece_checker_data& data
 		, std::vector<bool>& pieces
-		, int& num_pieces, bool compact_mode)
+		, int& num_pieces, storage_mode_t storage_mode
+		, std::string& error_msg)
 	{
 		boost::recursive_mutex::scoped_lock lock(m_mutex);
 
@@ -1493,7 +1395,7 @@ namespace libtorrent
 
 		TORRENT_ASSERT(m_info->piece_length() > 0);
 
-		m_compact_mode = compact_mode;
+		m_storage_mode = storage_mode;
 
 		// This will corrupt the storage
 		// use while debugging to find
@@ -1503,8 +1405,12 @@ namespace libtorrent
 
 		m_piece_to_slot.resize(m_info->num_pieces(), has_no_slot);
 		m_slot_to_piece.resize(m_info->num_pieces(), unallocated);
-		m_free_slots.clear();
+		TORRENT_ASSERT(m_free_slots.empty());
-		m_unallocated_slots.clear();
+		TORRENT_ASSERT(m_unallocated_slots.empty());
+
+		// assume no piece is out of place (i.e. in a slot
+		// other than the one it should be in)
+		bool out_of_place = false;
 		
 		pieces.clear();
 		pieces.resize(m_info->num_pieces(), false);
@@ -1513,13 +1419,14 @@ namespace libtorrent
 		// if we have fast-resume info
 		// use it instead of doing the actual checking
 		if (!data.piece_map.empty()
-			&& data.piece_map.size() <= m_slot_to_piece.size())
+			&& int(data.piece_map.size()) <= m_info->num_pieces())
 		{
 			for (int i = 0; i < (int)data.piece_map.size(); ++i)
 			{
 				m_slot_to_piece[i] = data.piece_map[i];
 				if (data.piece_map[i] >= 0)
 				{
+					if (data.piece_map[i] != i) out_of_place = true;
 					m_piece_to_slot[data.piece_map[i]] = i;
 					int found_piece = data.piece_map[i];
 
@@ -1537,30 +1444,58 @@ namespace libtorrent
 				}
 				else if (data.piece_map[i] == unassigned)
 				{
+					if (m_storage_mode == storage_mode_compact)
 						m_free_slots.push_back(i);
 				}
 				else
 				{
 					TORRENT_ASSERT(data.piece_map[i] == unallocated);
+					if (m_storage_mode == storage_mode_compact)
 						m_unallocated_slots.push_back(i);
 				}
 			}
 
-			m_unallocated_slots.reserve(int(pieces.size() - data.piece_map.size()));
+			if (m_storage_mode == storage_mode_compact)
-			for (int i = (int)data.piece_map.size(); i < (int)pieces.size(); ++i)
+			{
+				m_unallocated_slots.reserve(int(m_info->num_pieces() - data.piece_map.size()));
+				for (int i = (int)data.piece_map.size(); i < (int)m_info->num_pieces(); ++i)
 				{
 					m_unallocated_slots.push_back(i);
 				}
-
 				if (m_unallocated_slots.empty())
-				m_state = state_create_files;
+				{
-			else if (m_compact_mode)
+					switch_to_full_mode();
-				m_state = state_create_files;
+				}
+			}
 			else
-				m_state = state_allocating;
+			{
+				if (!out_of_place)
+				{
+					// if no piece is out of place
+					// since we're in full allocation mode, we can
+					// forget the piece allocation tables
+
+					std::vector<int>().swap(m_piece_to_slot);
+					std::vector<int>().swap(m_slot_to_piece);
+					m_state = state_create_files;
+					return false;
+				}
+				else
+				{
+					// in this case we're in full allocation mode, but
+					// we're resuming a compact allocated storage
+					m_state = state_expand_pieces;
+					m_current_slot = 0;
+					error_msg = "pieces needs to be reordered";
+					return false;
+				}
+			}
+
+			m_state = state_create_files;
 			return false;
 		}
 
+		error_msg = "empty piece map";
 		m_state = state_full_check;
 		return false;
 	}
@@ -1572,18 +1507,13 @@ namespace libtorrent
 
       |        |
       |        v
-      |  +------------+
+      |  +------------+   +---------------+
-      |  | full_check |
+      |  | full_check |-->| expand_pieses |
-      |  +------------+
+      |  +------------+   +---------------+
-      |        |
+      |        |                 |
-      |        v
+      |        v                 |
-      |  +------------+
+      |  +--------------+        |
-      |->| allocating |
+      +->| create_files | <------+
-      |  +------------+
-      |        |
-      |        v
-      |  +--------------+
-      |->| create_files |
          +--------------+
                |
                v
@@ -1602,67 +1532,97 @@ namespace libtorrent
 	std::pair<bool, float> piece_manager::check_files(
 		std::vector<bool>& pieces, int& num_pieces, boost::recursive_mutex& mutex)
 	{
+#ifndef NDEBUG
+		boost::recursive_mutex::scoped_lock l_(mutex);
 		TORRENT_ASSERT(num_pieces == std::count(pieces.begin(), pieces.end(), true));
-
+		l_.unlock();
-		if (m_state == state_allocating)
+#endif
-		{
-			if (m_compact_mode || m_unallocated_slots.empty())
-			{
-				m_state = state_create_files;
-				return std::make_pair(false, 1.f);
-			}
-		
-			if (int(m_unallocated_slots.size()) == m_info->num_pieces()
-				&& !m_fill_mode)
-			{
-				// if there is not a single file on disk, just
-				// create the files
-				m_state = state_create_files;
-				return std::make_pair(false, 1.f);
-			}
-
-			// if we're not in compact mode, make sure the
-			// pieces are spread out and placed at their
-			// final position.
-			TORRENT_ASSERT(!m_unallocated_slots.empty());
-
-			if (!m_fill_mode)
-			{
-				// if we're not filling the allocation
-				// just make sure we move the current pieces
-				// into place, and just skip all other
-				// allocation
-				// allocate_slots returns true if it had to
-				// move any data
-				allocate_slots(m_unallocated_slots.size(), true);
-			}
-			else
-			{
-				allocate_slots(1);
-			}
-
-			return std::make_pair(false, 1.f - (float)m_unallocated_slots.size()
-				/ (float)m_slot_to_piece.size());
-		}
 
 		if (m_state == state_create_files)
 		{
-			m_storage->initialize(!m_fill_mode && !m_compact_mode);
+			m_storage->initialize(m_storage_mode == storage_mode_allocate);
-
-			if (!m_unallocated_slots.empty() && !m_compact_mode)
-			{
-				TORRENT_ASSERT(!m_fill_mode);
-				std::vector<int>().swap(m_unallocated_slots);
-				std::fill(m_slot_to_piece.begin(), m_slot_to_piece.end(), int(unassigned));
-				m_free_slots.resize(m_info->num_pieces());
-				for (int i = 0; i < m_info->num_pieces(); ++i)
-					m_free_slots[i] = i;
-			}
-
 			m_state = state_finished;
 			return std::make_pair(true, 1.f);
 		}
 
+		if (m_state == state_expand_pieces)
+		{
+			INVARIANT_CHECK;
+
+			if (m_scratch_piece >= 0)
+			{
+				int piece = m_scratch_piece;
+				int other_piece = m_slot_to_piece[piece];
+				m_scratch_piece = -1;
+
+				if (other_piece >= 0)
+				{
+					if (m_scratch_buffer2.empty())
+						m_scratch_buffer2.resize(m_info->piece_length());
+
+					m_storage->read(&m_scratch_buffer2[0], piece, 0, m_info->piece_size(other_piece));
+					m_scratch_piece = other_piece;
+					m_piece_to_slot[other_piece] = unassigned;
+				}
+				
+				// the slot where this piece belongs is
+				// free. Just move the piece there.
+				m_storage->write(&m_scratch_buffer[0], piece, 0, m_info->piece_size(piece));
+				m_piece_to_slot[piece] = piece;
+				m_slot_to_piece[piece] = piece;
+
+				if (other_piece >= 0)
+					m_scratch_buffer.swap(m_scratch_buffer2);
+		
+				return std::make_pair(false, (float)m_current_slot / m_info->num_pieces());
+			}
+
+			while (m_current_slot < m_info->num_pieces()
+				&& (m_slot_to_piece[m_current_slot] == m_current_slot
+				|| m_slot_to_piece[m_current_slot] < 0))
+			{
+				++m_current_slot;
+			}
+
+			if (m_current_slot == m_info->num_pieces())
+			{
+				m_state = state_create_files;
+				std::vector<char>().swap(m_scratch_buffer);
+				std::vector<char>().swap(m_scratch_buffer2);
+				if (m_storage_mode != storage_mode_compact)
+				{
+					std::vector<int>().swap(m_piece_to_slot);
+					std::vector<int>().swap(m_slot_to_piece);
+				}
+				return std::make_pair(false, 1.f);
+			}
+
+			int piece = m_slot_to_piece[m_current_slot];
+			TORRENT_ASSERT(piece >= 0);
+			int other_piece = m_slot_to_piece[piece];
+			if (other_piece >= 0)
+			{
+				// there is another piece in the slot
+				// where this one goes. Store it in the scratch
+				// buffer until next iteration.
+				if (m_scratch_buffer.empty())
+					m_scratch_buffer.resize(m_info->piece_length());
+			
+				m_storage->read(&m_scratch_buffer[0], piece, 0, m_info->piece_size(other_piece));
+				m_scratch_piece = other_piece;
+				m_piece_to_slot[other_piece] = unassigned;
+			}
+
+			// the slot where this piece belongs is
+			// free. Just move the piece there.
+			m_storage->move_slot(m_current_slot, piece);
+			m_piece_to_slot[piece] = piece;
+			m_slot_to_piece[m_current_slot] = unassigned;
+			m_slot_to_piece[piece] = piece;
+		
+			return std::make_pair(false, (float)m_current_slot / m_info->num_pieces());
+		}
+
 		TORRENT_ASSERT(m_state == state_full_check);
 
 		// ------------------------
@@ -1674,12 +1634,13 @@ namespace libtorrent
 			// initialization for the full check
 			if (m_hash_to_piece.empty())
 			{
-				m_current_slot = 0;
 				for (int i = 0; i < m_info->num_pieces(); ++i)
 				{
 					m_hash_to_piece.insert(std::make_pair(m_info->hash_for_piece(i), i));
 				}
+				boost::recursive_mutex::scoped_lock l(mutex);
 				std::fill(pieces.begin(), pieces.end(), false);
+				num_pieces = 0;
 			}
 
 			m_piece_data.resize(int(m_info->piece_length()));
@@ -1694,6 +1655,10 @@ namespace libtorrent
 			int piece_index = identify_data(m_piece_data, m_current_slot
 				, pieces, num_pieces, m_hash_to_piece, mutex);
 
+			if (piece_index != m_current_slot
+				&& piece_index >= 0)
+				m_out_of_place = true;
+
 			TORRENT_ASSERT(num_pieces == std::count(pieces.begin(), pieces.end(), true));
 			TORRENT_ASSERT(piece_index == unassigned || piece_index >= 0);
 
@@ -1745,9 +1710,12 @@ namespace libtorrent
 					std::vector<int>::iterator i =
 						std::find(m_free_slots.begin(), m_free_slots.end(), other_slot);
 					TORRENT_ASSERT(i != m_free_slots.end());
+					if (m_storage_mode == storage_mode_compact)
+					{
 						m_free_slots.erase(i);
 						m_free_slots.push_back(m_current_slot);
 					}
+				}
 
 				if (other_piece >= 0)
 					m_storage->swap_slots(other_slot, m_current_slot);
@@ -1770,7 +1738,8 @@ namespace libtorrent
 				m_slot_to_piece[other_slot] = piece_index;
 				m_piece_to_slot[other_piece] = m_current_slot;
 
-				if (piece_index == unassigned)
+				if (piece_index == unassigned
+					&& m_storage_mode == storage_mode_compact)
 					m_free_slots.push_back(other_slot);
 
 				if (piece_index >= 0)
@@ -1845,9 +1814,12 @@ namespace libtorrent
 						std::vector<int>::iterator i =
 							std::find(m_free_slots.begin(), m_free_slots.end(), slot1);
 						TORRENT_ASSERT(i != m_free_slots.end());
+						if (m_storage_mode == storage_mode_compact)
+						{
 							m_free_slots.erase(i);
 							m_free_slots.push_back(slot2);
 						}
+					}
 
 					if (piece1 >= 0)
 					{
@@ -1873,7 +1845,7 @@ namespace libtorrent
 				// the slot was identified as piece 'piece_index'
 				if (piece_index != unassigned)
 					m_piece_to_slot[piece_index] = m_current_slot;
-				else
+				else if (m_storage_mode == storage_mode_compact)
 					m_free_slots.push_back(m_current_slot);
 
 				m_slot_to_piece[m_current_slot] = piece_index;
@@ -1899,11 +1871,14 @@ namespace libtorrent
 					(file_offset - current_offset + m_info->piece_length() - 1)
 					/ m_info->piece_length());
 
+			if (m_storage_mode == storage_mode_compact)
+			{
 				for (int i = m_current_slot; i < m_current_slot + skip_blocks; ++i)
 				{
 					TORRENT_ASSERT(m_slot_to_piece[i] == unallocated);
 					m_unallocated_slots.push_back(i);
 				}
+			}
 
 			// current slot will increase by one at the end of the for-loop too
 			m_current_slot += skip_blocks - 1;
@@ -1917,8 +1892,39 @@ namespace libtorrent
 			// clear the memory we've been using
 			std::vector<char>().swap(m_piece_data);
 			std::multimap<sha1_hash, int>().swap(m_hash_to_piece);
-			m_state = state_allocating;
+
+			if (m_storage_mode != storage_mode_compact)
+			{
+				if (!m_out_of_place)
+				{
+					// if no piece is out of place
+					// since we're in full allocation mode, we can
+					// forget the piece allocation tables
+
+					std::vector<int>().swap(m_piece_to_slot);
+					std::vector<int>().swap(m_slot_to_piece);
+					m_state = state_create_files;
+					return std::make_pair(false, 1.f);
+				}
+				else
+				{
+					// in this case we're in full allocation mode, but
+					// we're resuming a compact allocated storage
+					m_state = state_expand_pieces;
+					m_current_slot = 0;
+					return std::make_pair(false, 0.f);
+				}
+			}
+			else if (m_unallocated_slots.empty())
+			{
+				switch_to_full_mode();
+			}
+			m_state = state_create_files;
+
+#ifndef NDEBUG
+			boost::recursive_mutex::scoped_lock l(mutex);
 			TORRENT_ASSERT(num_pieces == std::count(pieces.begin(), pieces.end(), true));
+#endif
 			return std::make_pair(false, 1.f);
 		}
 
@@ -1927,10 +1933,26 @@ namespace libtorrent
 		return std::make_pair(false, (float)m_current_slot / m_info->num_pieces());
 	}
 
+	void piece_manager::switch_to_full_mode()
+	{
+		TORRENT_ASSERT(m_storage_mode == storage_mode_compact);	
+		TORRENT_ASSERT(m_unallocated_slots.empty());	
+		// we have allocated all slots, switch to
+		// full allocation mode in order to free
+		// some unnecessary memory.
+		m_storage_mode = storage_mode_sparse;
+		std::vector<int>().swap(m_unallocated_slots);
+		std::vector<int>().swap(m_free_slots);
+		std::vector<int>().swap(m_piece_to_slot);
+		std::vector<int>().swap(m_slot_to_piece);
+	}
+
 	int piece_manager::allocate_slot_for_piece(int piece_index)
 	{
 		boost::recursive_mutex::scoped_lock lock(m_mutex);
 
+		if (m_storage_mode != storage_mode_compact) return piece_index;
+
 //		INVARIANT_CHECK;
 
 		TORRENT_ASSERT(piece_index >= 0);
@@ -2030,25 +2052,26 @@ namespace libtorrent
 			debug_log();
 #endif
 		}
-
 		TORRENT_ASSERT(slot_index >= 0);
 		TORRENT_ASSERT(slot_index < (int)m_slot_to_piece.size());
+
+		if (m_unallocated_slots.empty())
+		{
+			switch_to_full_mode();
+		}
+		
 		return slot_index;
 	}
 
 	bool piece_manager::allocate_slots(int num_slots, bool abort_on_disk)
 	{
-		TORRENT_ASSERT(num_slots > 0);
-
 		boost::recursive_mutex::scoped_lock lock(m_mutex);
+		TORRENT_ASSERT(num_slots > 0);
 
 //		INVARIANT_CHECK;
 
 		TORRENT_ASSERT(!m_unallocated_slots.empty());
-
+		TORRENT_ASSERT(m_storage_mode == storage_mode_compact);
-		const int stack_buffer_size = 16*1024;
-		char zeroes[stack_buffer_size];
-		memset(zeroes, 0, stack_buffer_size);
 
 		bool written = false;
 
@@ -2069,32 +2092,57 @@ namespace libtorrent
 				m_piece_to_slot[pos] = pos;
 				written = true;
 			}
-			else if (m_fill_mode)
-			{
-				int piece_size = int(m_info->piece_size(pos));
-				int offset = 0;
-				for (; piece_size > 0; piece_size -= stack_buffer_size
-					, offset += stack_buffer_size)
-				{
-					m_storage->write(zeroes, pos, offset
-						, (std::min)(piece_size, stack_buffer_size));
-				}
-				written = true;
-			}
 			m_unallocated_slots.erase(m_unallocated_slots.begin());
 			m_slot_to_piece[new_free_slot] = unassigned;
 			m_free_slots.push_back(new_free_slot);
-			if (abort_on_disk && written) return true;
+			if (abort_on_disk && written) break;
 		}
 
 		TORRENT_ASSERT(m_free_slots.size() > 0);
 		return written;
 	}
 
+	int piece_manager::slot_for(int piece) const
+	{
+		if (m_storage_mode != storage_mode_compact) return piece;
+		TORRENT_ASSERT(piece < int(m_piece_to_slot.size()));
+		TORRENT_ASSERT(piece >= 0);
+		return m_piece_to_slot[piece];
+	}
+
+	int piece_manager::piece_for(int slot) const
+	{
+		if (m_storage_mode != storage_mode_compact) return slot;
+		TORRENT_ASSERT(slot < int(m_slot_to_piece.size()));
+		TORRENT_ASSERT(slot >= 0);
+		return m_slot_to_piece[slot];
+	}
+		
 #ifndef NDEBUG
 	void piece_manager::check_invariant() const
 	{
 		boost::recursive_mutex::scoped_lock lock(m_mutex);
+
+		if (m_unallocated_slots.empty() && m_state == state_finished)
+		{
+			TORRENT_ASSERT(m_storage_mode != storage_mode_compact);
+		}
+		
+		if (m_storage_mode != storage_mode_compact)
+		{
+			TORRENT_ASSERT(m_unallocated_slots.empty());
+			TORRENT_ASSERT(m_free_slots.empty());
+		}
+		
+		if (m_storage_mode != storage_mode_compact
+			&& m_state != state_expand_pieces
+			&& m_state != state_full_check)
+		{
+			TORRENT_ASSERT(m_piece_to_slot.empty());
+			TORRENT_ASSERT(m_slot_to_piece.empty());
+		}
+		else
+		{
 			if (m_piece_to_slot.empty()) return;
 
 			TORRENT_ASSERT((int)m_piece_to_slot.size() == m_info->num_pieces());
@@ -2200,6 +2248,7 @@ namespace libtorrent
 				}
 			}
 		}
+	}
 
 #ifdef TORRENT_STORAGE_DEBUG
 	void piece_manager::debug_log() const

libtorrent/src/torrent.cpp

@@ -154,7 +154,7 @@ namespace libtorrent
 		, boost::intrusive_ptr<torrent_info> tf
 		, fs::path const& save_path
 		, tcp::endpoint const& net_interface
-		, bool compact_mode
+		, storage_mode_t storage_mode
 		, int block_size
 		, storage_constructor_type sc
 		, bool paused)
@@ -195,7 +195,7 @@ namespace libtorrent
 		, m_total_redundant_bytes(0)
 		, m_net_interface(net_interface.address(), 0)
 		, m_save_path(complete(save_path))
-		, m_compact_mode(compact_mode)
+		, m_storage_mode(storage_mode)
 		, m_default_block_size(block_size)
 		, m_connections_initialized(true)
 		, m_settings(ses.settings())
@@ -215,7 +215,7 @@ namespace libtorrent
 		, char const* name
 		, fs::path const& save_path
 		, tcp::endpoint const& net_interface
-		, bool compact_mode
+		, storage_mode_t storage_mode
 		, int block_size
 		, storage_constructor_type sc
 		, bool paused)
@@ -255,7 +255,7 @@ namespace libtorrent
 		, m_total_redundant_bytes(0)
 		, m_net_interface(net_interface.address(), 0)
 		, m_save_path(complete(save_path))
-		, m_compact_mode(compact_mode)
+		, m_storage_mode(storage_mode)
 		, m_default_block_size(block_size)
 		, m_connections_initialized(false)
 		, m_settings(ses.settings())
@@ -2215,10 +2215,22 @@ namespace libtorrent
 		bool done = true;
 		try
 		{
+			std::string error_msg;
 			TORRENT_ASSERT(m_storage);
 			TORRENT_ASSERT(m_owning_storage.get());
 			done = m_storage->check_fastresume(data, m_have_pieces, m_num_pieces
-				, m_compact_mode);
+				, m_storage_mode, error_msg);
+
+			if (!error_msg.empty() && m_ses.m_alerts.should_post(alert::warning))
+			{
+				m_ses.m_alerts.post_alert(fastresume_rejected_alert(
+					get_handle(), error_msg));
+#if defined(TORRENT_VERBOSE_LOGGING) || defined(TORRENT_LOGGING)
+				(*m_ses.m_logger) << "fastresume data for "
+					<< torrent_file().name() << " rejected: "
+					<< error_msg << "\n";
+#endif
+			}
 		}
 		catch (std::exception& e)
 		{
@@ -2768,7 +2780,7 @@ namespace libtorrent
 			!boost::bind(&peer_connection::is_connecting
 			, boost::bind(&std::map<tcp::endpoint,peer_connection*>::value_type::second, _1)));
 
-		st.compact_mode = m_compact_mode;
+		st.storage_mode = m_storage_mode;
 
 		st.num_complete = m_complete;
 		st.num_incomplete = m_incomplete;

libtorrent/src/torrent_handle.cpp

@@ -661,7 +661,7 @@ namespace libtorrent
 
 		if (!t->valid_metadata()) return entry();
 
-		t->filesystem().export_piece_map(piece_index);
+		std::vector<bool> have_pieces = t->pieces();
 
 		entry ret(entry::dictionary_t);
 
@@ -673,10 +673,6 @@ namespace libtorrent
 		const sha1_hash& info_hash = t->torrent_file().info_hash();
 		ret["info-hash"] = std::string((char*)info_hash.begin(), (char*)info_hash.end());
 
-		ret["slots"] = entry(entry::list_t);
-		entry::list_type& slots = ret["slots"].list();
-		std::copy(piece_index.begin(), piece_index.end(), std::back_inserter(slots));
-
 		// blocks per piece
 		int num_blocks_per_piece =
 			static_cast<int>(t->torrent_file().piece_length()) / t->block_size();
@@ -706,6 +702,8 @@ namespace libtorrent
 				// the unfinished piece's index
 				piece_struct["piece"] = i->index;
 
+				have_pieces[i->index] = true;
+
 				std::string bitmask;
 				const int num_bitmask_bytes
 					= (std::max)(num_blocks_per_piece / 8, 1);
@@ -722,10 +720,10 @@ namespace libtorrent
 				}
 				piece_struct["bitmask"] = bitmask;
 
-				TORRENT_ASSERT(t->filesystem().slot_for_piece(i->index) >= 0);
+				TORRENT_ASSERT(t->filesystem().slot_for(i->index) >= 0);
 				unsigned long adler
 					= t->filesystem().piece_crc(
-						t->filesystem().slot_for_piece(i->index)
+						t->filesystem().slot_for(i->index)
 						, t->block_size()
 						, i->info);
 
@@ -735,6 +733,11 @@ namespace libtorrent
 				up.push_back(piece_struct);
 			}
 		}
+
+		t->filesystem().export_piece_map(piece_index, have_pieces);
+		entry::list_type& slots = ret["slots"].list();
+		std::copy(piece_index.begin(), piece_index.end(), std::back_inserter(slots));
+
 		// write local peers
 
 		entry::list_type& peer_list = ret["peers"].list();

src/deluge_core.cpp

@@ -242,8 +242,13 @@ boost::filesystem::path const& save_path)
     // Create new torrent object
 
     torrent_t new_torrent;
-
+    libtorrent::storage_mode_t storage_mode;
-    torrent_handle h = M_ses->add_torrent(t, save_path, resume_data, compact_mode);
+    if (compact_mode){
+        storage_mode = storage_mode_compact;
+    } else {
+        storage_mode = storage_mode_sparse;
+    }
+    torrent_handle h = M_ses->add_torrent(t, save_path, resume_data, storage_mode);
     //    h.set_max_connections(60); // at some point we should use this
     h.set_max_uploads(-1);
     h.set_ratio(preferred_ratio);
@@ -893,7 +898,7 @@ static PyObject *torrent_get_torrent_state(PyObject *self, PyObject *args)
         "num_seeds",          connected_seeds,
         "distributed_copies", s.distributed_copies == -1.0 ? 0.0 : s.distributed_copies,
         "download_rate",      s.download_payload_rate,
-        "compact_mode",       s.compact_mode,
+        "storage_mode",       s.storage_mode,
         "upload_rate",        s.upload_payload_rate,
         "total_download",     s.total_download,
         "total_upload",       s.total_upload,