From 2aa20b3e222e218541b30d9e063cd2e3c855e9ca Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 15 Sep 2017 19:22:32 +1000 Subject: [PATCH] Don't verify data at startup, add Torrent.Piece.VerifyData and Torrent.VerifyData for this purpose This has the side effect of deflaking a lot of tests that race to verify data when a torrent is added. --- client_test.go | 25 ++++++++++++++----------- piece.go | 24 ++++++++++++++++++++---- t.go | 6 ++++++ torrent.go | 19 +++++++++++++------ 4 files changed, 53 insertions(+), 21 deletions(-) diff --git a/client_test.go b/client_test.go index b31720a3..2804d9c8 100644 --- a/client_test.go +++ b/client_test.go @@ -381,10 +381,8 @@ func testClientTransfer(t *testing.T, ps testClientTransferParams) { if ps.ExportClientStatus { testutil.ExportStatusWriter(seeder, "s") } - // seederTorrent, new, err := seeder.AddTorrentSpec(TorrentSpecFromMetaInfo(mi)) - _, new, err := seeder.AddTorrentSpec(TorrentSpecFromMetaInfo(mi)) - require.NoError(t, err) - assert.True(t, new) + seederTorrent, _, _ := seeder.AddTorrentSpec(TorrentSpecFromMetaInfo(mi)) + seederTorrent.VerifyData() // Create leecher and a Torrent. leecherDataDir, err := ioutil.TempDir("", "") require.NoError(t, err) @@ -454,7 +452,8 @@ func TestSeedAfterDownloading(t *testing.T) { require.NoError(t, err) defer seeder.Close() testutil.ExportStatusWriter(seeder, "s") - seeder.AddTorrentSpec(TorrentSpecFromMetaInfo(mi)) + seederTorrent, _, _ := seeder.AddTorrentSpec(TorrentSpecFromMetaInfo(mi)) + seederTorrent.VerifyData() cfg.DataDir, err = ioutil.TempDir("", "") require.NoError(t, err) defer os.RemoveAll(cfg.DataDir) @@ -619,7 +618,8 @@ func TestResponsive(t *testing.T) { seeder, err := NewClient(cfg) require.Nil(t, err) defer seeder.Close() - seeder.AddTorrentSpec(TorrentSpecFromMetaInfo(mi)) + seederTorrent, _, _ := seeder.AddTorrentSpec(TorrentSpecFromMetaInfo(mi)) + seederTorrent.VerifyData() leecherDataDir, err := ioutil.TempDir("", "") require.Nil(t, err) defer os.RemoveAll(leecherDataDir) @@ -661,7 +661,8 @@ func TestTorrentDroppedDuringResponsiveRead(t *testing.T) { seeder, err := NewClient(cfg) require.Nil(t, err) defer seeder.Close() - seeder.AddTorrentSpec(TorrentSpecFromMetaInfo(mi)) + seederTorrent, _, _ := seeder.AddTorrentSpec(TorrentSpecFromMetaInfo(mi)) + seederTorrent.VerifyData() leecherDataDir, err := ioutil.TempDir("", "") require.Nil(t, err) defer os.RemoveAll(leecherDataDir) @@ -759,6 +760,7 @@ func testAddTorrentPriorPieceCompletion(t *testing.T, alreadyCompleted bool, csf greetingDataTempDir, greetingMetainfo := testutil.GreetingTestTorrent() defer os.RemoveAll(greetingDataTempDir) filePieceStore := csf(fileCache) + defer filePieceStore.Close() info, err := greetingMetainfo.UnmarshalInfo() require.NoError(t, err) ih := greetingMetainfo.HashInfoBytes() @@ -770,8 +772,7 @@ func testAddTorrentPriorPieceCompletion(t *testing.T, alreadyCompleted bool, csf for i := 0; i < info.NumPieces(); i++ { p := info.Piece(i) if alreadyCompleted { - err := greetingData.Piece(p).MarkComplete() - assert.NoError(t, err) + require.NoError(t, greetingData.Piece(p).MarkComplete()) } } cfg := TestingConfig() @@ -844,7 +845,8 @@ func testDownloadCancel(t *testing.T, ps testDownloadCancelParams) { if ps.ExportClientStatus { testutil.ExportStatusWriter(seeder, "s") } - seeder.AddTorrentSpec(TorrentSpecFromMetaInfo(mi)) + seederTorrent, _, _ := seeder.AddTorrentSpec(TorrentSpecFromMetaInfo(mi)) + seederTorrent.VerifyData() leecherDataDir, err := ioutil.TempDir("", "") require.NoError(t, err) defer os.RemoveAll(leecherDataDir) @@ -1064,7 +1066,8 @@ func makeMagnet(t *testing.T, cl *Client, dir string, name string) string { magnet := mi.Magnet(name, mi.HashInfoBytes()).String() tr, err := cl.AddTorrent(&mi) require.NoError(t, err) - assert.True(t, tr.Seeding()) + require.True(t, tr.Seeding()) + tr.VerifyData() return magnet } diff --git a/piece.go b/piece.go index ec24b7db..cfdb8ef2 100644 --- a/piece.go +++ b/piece.go @@ -36,10 +36,13 @@ type piece struct { index int // Chunks we've written to since the last check. The chunk offset and // length can be determined by the request chunkSize in use. - DirtyChunks bitmap.Bitmap - Hashing bool - QueuedForHash bool - EverHashed bool + DirtyChunks bitmap.Bitmap + + Hashing bool + QueuedForHash bool + EverHashed bool + numVerifies int64 + PublicPieceState PieceState priority piecePriority @@ -157,3 +160,16 @@ func (p *piece) bytesLeft() (ret pp.Integer) { } return p.length() - p.numDirtyBytes() } + +func (p *piece) VerifyData() { + p.t.cl.mu.Lock() + defer p.t.cl.mu.Unlock() + target := p.numVerifies + 1 + if p.Hashing { + target++ + } + p.t.queuePieceCheck(p.index) + for p.numVerifies < target { + p.t.cl.event.Wait() + } +} diff --git a/t.go b/t.go index 13722d8a..ce0c007f 100644 --- a/t.go +++ b/t.go @@ -209,3 +209,9 @@ func (t *Torrent) AddTrackers(announceList [][]string) { defer t.cl.mu.Unlock() t.addTrackers(announceList) } + +func (t *Torrent) Piece(i int) *piece { + t.cl.mu.Lock() + defer t.cl.mu.Unlock() + return &t.pieces[i] +} diff --git a/torrent.go b/torrent.go index 5d6be261..29ca25a6 100644 --- a/torrent.go +++ b/torrent.go @@ -284,13 +284,13 @@ func (t *Torrent) setInfoBytes(b []byte) error { } for i := range t.pieces { t.updatePieceCompletion(i) - t.pieces[i].QueuedForHash = true + // t.pieces[i].QueuedForHash = true } - go func() { - for i := range t.pieces { - t.verifyPiece(i) - } - }() + // go func() { + // for i := range t.pieces { + // t.verifyPiece(i) + // } + // }() return nil } @@ -1485,6 +1485,7 @@ func (t *Torrent) verifyPiece(piece int) { cl.mu.Unlock() sum := t.hashPiece(piece) cl.mu.Lock() + p.numVerifies++ p.Hashing = false t.pieceHashed(piece, sum == p.Hash) } @@ -1518,3 +1519,9 @@ func (t *Torrent) queuePieceCheck(pieceIndex int) { t.publishPieceChange(pieceIndex) go t.verifyPiece(pieceIndex) } + +func (t *Torrent) VerifyData() { + for i := range iter.N(t.NumPieces()) { + t.Piece(i).VerifyData() + } +}