From 0fac705ed0a2efcda4da9dce8971bbda73f299d5 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 9 Apr 2018 13:47:06 +0200 Subject: [PATCH] compression/rle: delete RLE compression (#16468) --- compression/rle/read_write.go | 101 ----------------------------- compression/rle/read_write_test.go | 50 -------------- ethdb/database.go | 6 -- swarm/storage/database.go | 15 +---- 4 files changed, 2 insertions(+), 170 deletions(-) delete mode 100644 compression/rle/read_write.go delete mode 100644 compression/rle/read_write_test.go diff --git a/compression/rle/read_write.go b/compression/rle/read_write.go deleted file mode 100644 index 0e7ad90ae..000000000 --- a/compression/rle/read_write.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package rle implements the run-length encoding used for Ethereum data. -package rle - -import ( - "bytes" - "errors" - - "github.com/ethereum/go-ethereum/crypto" -) - -const ( - token byte = 0xfe - emptyShaToken = 0xfd - emptyListShaToken = 0xfe - tokenToken = 0xff -) - -var empty = crypto.Keccak256([]byte("")) -var emptyList = crypto.Keccak256([]byte{0x80}) - -func Decompress(dat []byte) ([]byte, error) { - buf := new(bytes.Buffer) - - for i := 0; i < len(dat); i++ { - if dat[i] == token { - if i+1 < len(dat) { - switch dat[i+1] { - case emptyShaToken: - buf.Write(empty) - case emptyListShaToken: - buf.Write(emptyList) - case tokenToken: - buf.WriteByte(token) - default: - buf.Write(make([]byte, int(dat[i+1]-2))) - } - i++ - } else { - return nil, errors.New("error reading bytes. token encountered without proceeding bytes") - } - } else { - buf.WriteByte(dat[i]) - } - } - - return buf.Bytes(), nil -} - -func compressChunk(dat []byte) (ret []byte, n int) { - switch { - case dat[0] == token: - return []byte{token, tokenToken}, 1 - case len(dat) > 1 && dat[0] == 0x0 && dat[1] == 0x0: - j := 0 - for j <= 254 && j < len(dat) { - if dat[j] != 0 { - break - } - j++ - } - return []byte{token, byte(j + 2)}, j - case len(dat) >= 32: - if dat[0] == empty[0] && bytes.Equal(dat[:32], empty) { - return []byte{token, emptyShaToken}, 32 - } else if dat[0] == emptyList[0] && bytes.Equal(dat[:32], emptyList) { - return []byte{token, emptyListShaToken}, 32 - } - fallthrough - default: - return dat[:1], 1 - } -} - -func Compress(dat []byte) []byte { - buf := new(bytes.Buffer) - - i := 0 - for i < len(dat) { - b, n := compressChunk(dat[i:]) - buf.Write(b) - i += n - } - - return buf.Bytes() -} diff --git a/compression/rle/read_write_test.go b/compression/rle/read_write_test.go deleted file mode 100644 index b36f7907b..000000000 --- a/compression/rle/read_write_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package rle - -import ( - "testing" - - checker "gopkg.in/check.v1" -) - -func Test(t *testing.T) { checker.TestingT(t) } - -type CompressionRleSuite struct{} - -var _ = checker.Suite(&CompressionRleSuite{}) - -func (s *CompressionRleSuite) TestDecompressSimple(c *checker.C) { - exp := []byte{0xc5, 0xd2, 0x46, 0x1, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x3, 0xc0, 0xe5, 0x0, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x4, 0x5d, 0x85, 0xa4, 0x70} - res, err := Decompress([]byte{token, 0xfd}) - c.Assert(err, checker.IsNil) - c.Assert(res, checker.DeepEquals, exp) - - exp = []byte{0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x1, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21} - res, err = Decompress([]byte{token, 0xfe}) - c.Assert(err, checker.IsNil) - c.Assert(res, checker.DeepEquals, exp) - - res, err = Decompress([]byte{token, 0xff}) - c.Assert(err, checker.IsNil) - c.Assert(res, checker.DeepEquals, []byte{token}) - - res, err = Decompress([]byte{token, 12}) - c.Assert(err, checker.IsNil) - c.Assert(res, checker.DeepEquals, make([]byte, 10)) - -} diff --git a/ethdb/database.go b/ethdb/database.go index 30ed37dc7..243b7f8d3 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -91,9 +91,6 @@ func (db *LDBDatabase) Path() string { // Put puts the given key / value to the queue func (db *LDBDatabase) Put(key []byte, value []byte) error { - // Generate the data to write to disk, update the meter and write - //value = rle.Compress(value) - return db.db.Put(key, value, nil) } @@ -103,18 +100,15 @@ func (db *LDBDatabase) Has(key []byte) (bool, error) { // Get returns the given key if it's present. func (db *LDBDatabase) Get(key []byte) ([]byte, error) { - // Retrieve the key and increment the miss counter if not found dat, err := db.db.Get(key, nil) if err != nil { return nil, err } return dat, nil - //return rle.Decompress(dat) } // Delete deletes the key from the queue and database func (db *LDBDatabase) Delete(key []byte) error { - // Execute the actual operation return db.db.Delete(key, nil) } diff --git a/swarm/storage/database.go b/swarm/storage/database.go index 2532490cc..f2ceb94e4 100644 --- a/swarm/storage/database.go +++ b/swarm/storage/database.go @@ -22,7 +22,6 @@ package storage import ( "fmt" - "github.com/ethereum/go-ethereum/compression/rle" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" @@ -31,8 +30,7 @@ import ( const openFileLimit = 128 type LDBDatabase struct { - db *leveldb.DB - comp bool + db *leveldb.DB } func NewLDBDatabase(file string) (*LDBDatabase, error) { @@ -42,16 +40,12 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) { return nil, err } - database := &LDBDatabase{db: db, comp: false} + database := &LDBDatabase{db: db} return database, nil } func (self *LDBDatabase) Put(key []byte, value []byte) { - if self.comp { - value = rle.Compress(value) - } - err := self.db.Put(key, value, nil) if err != nil { fmt.Println("Error put", err) @@ -63,11 +57,6 @@ func (self *LDBDatabase) Get(key []byte) ([]byte, error) { if err != nil { return nil, err } - - if self.comp { - return rle.Decompress(dat) - } - return dat, nil }