Add new key_store interface and two new key stores

* Add new generic key_store interface
* Add new plaintext key store storing unprotected keys on disk
* Add new encrypted key store storing encrypted keys on disk
* Add new entropy mixing function using OS and go runtime sources

.gitignore

@@ -10,3 +10,8 @@
 .DS_Store
 */**/.DS_Store
 .ethtest
+
+#*
+.#*
+*#
+*~

crypto/key.go

@@ -0,0 +1,205 @@
+/*
+	This file is part of go-ethereum
+
+	go-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Lesser General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	go-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU Lesser General Public License
+	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @authors
+ * 	Gustav Simonsson <gustav.simonsson@gmail.com>
+ * @date 2015
+ *
+ */
+
+package crypto
+
+import (
+	"bytes"
+	"code.google.com/p/go-uuid/uuid"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	crand "crypto/rand"
+	"encoding/binary"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"strings"
+	"time"
+)
+
+type Key struct {
+	Id    *uuid.UUID // Version 4 "random" for unique id not derived from key data
+	Flags [4]byte    // RFU
+	// we only store privkey as pubkey/address can be derived from it
+	// privkey in this struct is always in plaintext
+	PrivateKey *ecdsa.PrivateKey
+}
+
+type KeyPlainJSON struct {
+	Id         string
+	Flags      string
+	PrivateKey string
+}
+
+type CipherJSON struct {
+	Salt       string
+	IV         string
+	CipherText string
+}
+
+type KeyProtectedJSON struct {
+	Id     string
+	Flags  string
+	Crypto CipherJSON
+}
+
+func (k *Key) Address() []byte {
+	pubBytes := FromECDSAPub(&k.PrivateKey.PublicKey)
+	return Sha3(pubBytes)[12:]
+}
+
+func (k *Key) MarshalJSON() (j []byte, err error) {
+	stringStruct := KeyPlainJSON{
+		k.Id.String(),
+		hex.EncodeToString(k.Flags[:]),
+		hex.EncodeToString(FromECDSA(k.PrivateKey)),
+	}
+	j, _ = json.Marshal(stringStruct)
+	return
+}
+
+func (k *Key) UnmarshalJSON(j []byte) (err error) {
+	keyJSON := new(KeyPlainJSON)
+	err = json.Unmarshal(j, &keyJSON)
+	if err != nil {
+		return
+	}
+
+	u := new(uuid.UUID)
+	*u = uuid.Parse(keyJSON.Id)
+	if *u == nil {
+		err = errors.New("UUID parsing failed")
+		return
+	}
+	k.Id = u
+
+	flagsBytes, err := hex.DecodeString(keyJSON.Flags)
+	if err != nil {
+		return
+	}
+
+	PrivateKeyBytes, err := hex.DecodeString(keyJSON.PrivateKey)
+	if err != nil {
+		return
+	}
+
+	copy(k.Flags[:], flagsBytes[0:4])
+	k.PrivateKey = ToECDSA(PrivateKeyBytes)
+
+	return
+}
+
+func NewKey() *Key {
+	randBytes := GetEntropyCSPRNG(32)
+	reader := bytes.NewReader(randBytes)
+	_, x, y, err := elliptic.GenerateKey(S256(), reader)
+	if err != nil {
+		panic("key generation: elliptic.GenerateKey failed: " + err.Error())
+	}
+	privateKeyMarshalled := elliptic.Marshal(S256(), x, y)
+	privateKeyECDSA := ToECDSA(privateKeyMarshalled)
+
+	key := new(Key)
+	id := uuid.NewRandom()
+	key.Id = &id
+	// flags := new([4]byte)
+	// key.Flags = flags
+	key.PrivateKey = privateKeyECDSA
+	return key
+}
+
+// plain crypto/rand. this is /dev/urandom on Unix-like systems.
+func GetEntropyCSPRNG(n int) []byte {
+	mainBuff := make([]byte, n)
+	_, err := io.ReadFull(crand.Reader, mainBuff)
+	if err != nil {
+		panic("key generation: reading from crypto/rand failed: " + err.Error())
+	}
+	return mainBuff
+}
+
+// TODO: verify. Do not use until properly discussed.
+// we start with crypt/rand, then mix in additional sources of entropy.
+// These sources are from three types: OS, go runtime and ethereum client state.
+func GetEntropyTinFoilHat() []byte {
+	startTime := time.Now().UnixNano()
+	// for each source, we XOR in it's SHA3 hash.
+	mainBuff := GetEntropyCSPRNG(32)
+	// 1. OS entropy sources
+	startTimeBytes := make([]byte, 32)
+	binary.PutVarint(startTimeBytes, startTime)
+	startTimeHash := Sha3(startTimeBytes)
+	mix32Byte(mainBuff, startTimeHash)
+
+	pid := os.Getpid()
+	pidBytes := make([]byte, 32)
+	binary.PutUvarint(pidBytes, uint64(pid))
+	pidHash := Sha3(pidBytes)
+	mix32Byte(mainBuff, pidHash)
+
+	osEnv := os.Environ()
+	osEnvBytes := []byte(strings.Join(osEnv, ""))
+	osEnvHash := Sha3(osEnvBytes)
+	mix32Byte(mainBuff, osEnvHash)
+
+	// not all OS have hostname in env variables
+	osHostName, err := os.Hostname()
+	if err != nil {
+		osHostNameBytes := []byte(osHostName)
+		osHostNameHash := Sha3(osHostNameBytes)
+		mix32Byte(mainBuff, osHostNameHash)
+	}
+
+	// 2. go runtime entropy sources
+	memStats := new(runtime.MemStats)
+	runtime.ReadMemStats(memStats)
+	memStatsBytes := []byte(fmt.Sprintf("%v", memStats))
+	memStatsHash := Sha3(memStatsBytes)
+	mix32Byte(mainBuff, memStatsHash)
+
+	// 3. Mix in ethereum / client state
+	// TODO: list of network peers structs (IP, port, etc)
+	// TODO: merkle patricia tree root hash for world state and tx list
+
+	// 4. Yo dawg we heard you like entropy so we'll grab some entropy from how
+	//    long it took to grab the above entropy. And a yield, for good measure.
+	runtime.Gosched()
+	diffTime := time.Now().UnixNano() - startTime
+	diffTimeBytes := make([]byte, 32)
+	binary.PutVarint(diffTimeBytes, diffTime)
+	diffTimeHash := Sha3(diffTimeBytes)
+	mix32Byte(mainBuff, diffTimeHash)
+
+	return mainBuff
+}
+
+func mix32Byte(buff []byte, mixBuff []byte) []byte {
+	for i := 0; i < 32; i++ {
+		buff[i] ^= mixBuff[i]
+	}
+	return buff
+}

crypto/key_store_passphrase_encryped.go

@@ -0,0 +1,244 @@
+/*
+	This file is part of go-ethereum
+
+	go-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Lesser General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	go-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU Lesser General Public License
+	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @authors
+ * 	Gustav Simonsson <gustav.simonsson@gmail.com>
+ * @date 2015
+ *
+ */
+/*
+
+This key store behaves as KeyStorePlaintextFile with the difference that
+the private key is encrypted and encoded as a JSON object within the
+key JSON object.
+
+Cryptography:
+
+1. Encryption key is scrypt derived key from user passphrase. Scrypt parameters
+   (work factors) [1][2] are defined as constants below.
+2. Scrypt salt is 32 random bytes from CSPRNG. It is appended to ciphertext.
+3. Checksum is SHA3 of the private key bytes.
+4. Plaintext is concatenation of private key bytes and checksum.
+5. Encryption algo is AES 256 CBC [3][4]
+6. CBC IV is 16 random bytes from CSPRNG. It is appended to ciphertext.
+7. Plaintext padding is PKCS #7 [5][6]
+
+Encoding:
+
+1. On disk, ciphertext, salt and IV are encoded as a JSON object.
+   cat a key file to see the structure.
+2. byte arrays are ASCII HEX encoded as JSON strings.
+3. The EC private key bytes are in uncompressed form [7].
+   They are a big-endian byte slice of the absolute value of D [8][9].
+4. The checksum is the last 32 bytes of the plaintext byte array and the
+   private key is the preceeding bytes.
+
+References:
+
+1. http://www.tarsnap.com/scrypt/scrypt-slides.pdf
+2. http://stackoverflow.com/questions/11126315/what-are-optimal-scrypt-work-factors
+3. http://en.wikipedia.org/wiki/Advanced_Encryption_Standard
+4. http://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher-block_chaining_.28CBC.29
+5. https://leanpub.com/gocrypto/read#leanpub-auto-block-cipher-modes
+6. http://tools.ietf.org/html/rfc2315
+7. http://bitcoin.stackexchange.com/questions/3059/what-is-a-compressed-bitcoin-key
+8. http://golang.org/pkg/crypto/ecdsa/#PrivateKey
+9. https://golang.org/pkg/math/big/#Int.Bytes
+
+*/
+
+package crypto
+
+import (
+	"bytes"
+	"code.google.com/p/go-uuid/uuid"
+	"code.google.com/p/go.crypto/scrypt"
+	"crypto/aes"
+	"crypto/cipher"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"os"
+	"path"
+)
+
+const scryptN int = 262144 // 2^18
+const scryptr int = 8
+const scryptp int = 1
+const scryptdkLen int = 32
+
+type KeyStorePassphrase struct {
+	keysDirPath string
+}
+
+func (ks KeyStorePassphrase) GenerateNewKey(auth string) (key *Key, err error) {
+	key, err = GenerateNewKeyDefault(ks, auth)
+	return
+}
+
+func (ks KeyStorePassphrase) GetKey(keyId *uuid.UUID, auth string) (key *Key, err error) {
+	keyBytes, flags, err := DecryptKey(ks, keyId, auth)
+	key = new(Key)
+	key.Id = keyId
+	copy(key.Flags[:], flags[0:4])
+	key.PrivateKey = ToECDSA(keyBytes)
+	return
+}
+
+func (ks KeyStorePassphrase) StoreKey(key *Key, auth string) (err error) {
+	authArray := []byte(auth)
+	salt := GetEntropyCSPRNG(32)
+	derivedKey, err := scrypt.Key(authArray, salt, scryptN, scryptr, scryptp, scryptdkLen)
+	if err != nil {
+		return
+	}
+
+	keyBytes := FromECDSA(key.PrivateKey)
+	keyBytesHash := Sha3(keyBytes)
+	toEncrypt := PKCS7Pad(append(keyBytes, keyBytesHash...))
+
+	AES256Block, err := aes.NewCipher(derivedKey)
+	if err != nil {
+		return
+	}
+
+	iv := GetEntropyCSPRNG(aes.BlockSize) // 16
+	AES256CBCEncrypter := cipher.NewCBCEncrypter(AES256Block, iv)
+	cipherText := make([]byte, len(toEncrypt))
+	AES256CBCEncrypter.CryptBlocks(cipherText, toEncrypt)
+
+	cipherStruct := CipherJSON{
+		hex.EncodeToString(salt),
+		hex.EncodeToString(iv),
+		hex.EncodeToString(cipherText),
+	}
+	keyStruct := KeyProtectedJSON{
+		key.Id.String(),
+		hex.EncodeToString(key.Flags[:]),
+		cipherStruct,
+	}
+	keyJSON, err := json.Marshal(keyStruct)
+	if err != nil {
+		return
+	}
+
+	err = WriteKeyFile(key.Id.String(), ks.keysDirPath, keyJSON)
+	return
+}
+
+func (ks KeyStorePassphrase) DeleteKey(keyId *uuid.UUID, auth string) (err error) {
+	// only delete if correct passphrase is given
+	_, _, err = DecryptKey(ks, keyId, auth)
+	if err != nil {
+		return
+	}
+
+	keyDirPath := path.Join(ks.keysDirPath, keyId.String())
+	err = os.RemoveAll(keyDirPath)
+	return
+}
+
+func DecryptKey(ks KeyStorePassphrase, keyId *uuid.UUID, auth string) (keyBytes []byte, flags []byte, err error) {
+	fileContent, err := GetKeyFile(ks.keysDirPath, keyId)
+	if err != nil {
+		return
+	}
+
+	keyProtected := new(KeyProtectedJSON)
+	err = json.Unmarshal(fileContent, keyProtected)
+
+	flags, err = hex.DecodeString(keyProtected.Flags)
+	if err != nil {
+		return
+	}
+
+	salt, err := hex.DecodeString(keyProtected.Crypto.Salt)
+	if err != nil {
+		return
+	}
+
+	iv, err := hex.DecodeString(keyProtected.Crypto.IV)
+	if err != nil {
+		return
+	}
+
+	cipherText, err := hex.DecodeString(keyProtected.Crypto.CipherText)
+	if err != nil {
+		return
+	}
+
+	authArray := []byte(auth)
+	derivedKey, err := scrypt.Key(authArray, salt, scryptN, scryptr, scryptp, scryptdkLen)
+	if err != nil {
+		return
+	}
+
+	AES256Block, err := aes.NewCipher(derivedKey)
+	if err != nil {
+		return
+	}
+
+	AES256CBCDecrypter := cipher.NewCBCDecrypter(AES256Block, iv)
+	paddedPlainText := make([]byte, len(cipherText))
+	AES256CBCDecrypter.CryptBlocks(paddedPlainText, cipherText)
+
+	plainText := PKCS7Unpad(paddedPlainText)
+	if plainText == nil {
+		err = errors.New("Decryption failed: PKCS7Unpad failed after decryption")
+		return
+	}
+
+	keyBytes = plainText[:len(plainText)-32]
+	keyBytesHash := plainText[len(plainText)-32:]
+	if !bytes.Equal(Sha3(keyBytes), keyBytesHash) {
+		err = errors.New("Decryption failed: checksum mismatch")
+		return
+	}
+	return keyBytes, flags, err
+}
+
+// From https://leanpub.com/gocrypto/read#leanpub-auto-block-cipher-modes
+func PKCS7Pad(in []byte) []byte {
+	padding := 16 - (len(in) % 16)
+	if padding == 0 {
+		padding = 16
+	}
+	for i := 0; i < padding; i++ {
+		in = append(in, byte(padding))
+	}
+	return in
+}
+
+func PKCS7Unpad(in []byte) []byte {
+	if len(in) == 0 {
+		return nil
+	}
+
+	padding := in[len(in)-1]
+	if int(padding) > len(in) || padding > aes.BlockSize {
+		return nil
+	} else if padding == 0 {
+		return nil
+	}
+
+	for i := len(in) - 1; i > len(in)-int(padding)-1; i-- {
+		if in[i] != padding {
+			return nil
+		}
+	}
+	return in[:len(in)-int(padding)]
+}

crypto/key_store_plaintext_file.go

@@ -0,0 +1,113 @@
+/*
+	This file is part of go-ethereum
+
+	go-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Lesser General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	go-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU Lesser General Public License
+	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/**
+ * @authors
+ * 	Gustav Simonsson <gustav.simonsson@gmail.com>
+ * @date 2015
+ *
+ */
+
+package crypto
+
+import (
+	"code.google.com/p/go-uuid/uuid"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/user"
+	"path"
+)
+
+// TODO: rename to KeyStore when replacing existing KeyStore
+type KeyStore2 interface {
+	GenerateNewKey(string) (*Key, error)     // create and store new key, optionally using auth string
+	GetKey(*uuid.UUID, string) (*Key, error) // key from id and auth string
+	StoreKey(*Key, string) error             // store key optionally using auth string
+	DeleteKey(*uuid.UUID, string) error      // delete key by id and auth string
+}
+
+type KeyStorePlaintext struct {
+	keysDirPath string
+}
+
+// TODO: copied from cmd/ethereum/flags.go
+func DefaultDataDir() string {
+	usr, _ := user.Current()
+	return path.Join(usr.HomeDir, ".ethereum")
+}
+
+func (ks KeyStorePlaintext) GenerateNewKey(auth string) (key *Key, err error) {
+	key, err = GenerateNewKeyDefault(ks, auth)
+	return
+}
+
+func GenerateNewKeyDefault(ks KeyStore2, auth string) (key *Key, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("GenerateNewKey error: %v", r)
+		}
+	}()
+	key = NewKey()
+	err = ks.StoreKey(key, auth)
+	return
+}
+
+func (ks KeyStorePlaintext) GetKey(keyId *uuid.UUID, auth string) (key *Key, err error) {
+	fileContent, err := GetKeyFile(ks.keysDirPath, keyId)
+	if err != nil {
+		return
+	}
+
+	key = new(Key)
+	err = json.Unmarshal(fileContent, key)
+	return
+}
+
+func (ks KeyStorePlaintext) StoreKey(key *Key, auth string) (err error) {
+	keyJSON, err := json.Marshal(key)
+	if err != nil {
+		return
+	}
+	err = WriteKeyFile(key.Id.String(), ks.keysDirPath, keyJSON)
+	return
+}
+
+func (ks KeyStorePlaintext) DeleteKey(keyId *uuid.UUID, auth string) (err error) {
+	keyDirPath := path.Join(ks.keysDirPath, keyId.String())
+	err = os.RemoveAll(keyDirPath)
+	return
+}
+
+func GetKeyFile(keysDirPath string, keyId *uuid.UUID) (fileContent []byte, err error) {
+	idString := keyId.String()
+	keyDirPath := path.Join(keysDirPath, idString)
+	keyFilePath := path.Join(keyDirPath, idString)
+	fileContent, err = ioutil.ReadFile(keyFilePath)
+	return
+}
+
+func WriteKeyFile(idString string, keysDirPath string, content []byte) (err error) {
+	keyDirPath := path.Join(keysDirPath, idString)
+	keyFilePath := path.Join(keyDirPath, idString)
+	err = os.MkdirAll(keyDirPath, 0700) // read, write and dir search for user
+	if err != nil {
+		return
+	}
+	err = ioutil.WriteFile(keyFilePath, content, 0600) // read, write for user
+	return
+}

crypto/key_store_test.go

@@ -0,0 +1,118 @@
+package crypto
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestKeyStorePlaintext(t *testing.T) {
+	ks := new(KeyStorePlaintext)
+	ks.keysDirPath = DefaultDataDir()
+	pass := "" // not used but required by API
+	k1, err := ks.GenerateNewKey(pass)
+	if err != nil {
+		fmt.Println(err.Error())
+		t.FailNow()
+	}
+
+	k2 := new(Key)
+	k2, err = ks.GetKey(k1.Id, pass)
+	if err != nil {
+		fmt.Println(err.Error())
+		t.FailNow()
+	}
+
+	if !reflect.DeepEqual(k1.Id, k2.Id) {
+		fmt.Println("key Id mismatch")
+		t.FailNow()
+	}
+
+	if k1.Flags != k2.Flags {
+		fmt.Println("key Flags mismatch")
+		t.FailNow()
+	}
+
+	if !reflect.DeepEqual(k1.PrivateKey, k2.PrivateKey) {
+		fmt.Println("key PrivateKey mismatch")
+		t.FailNow()
+	}
+
+	err = ks.DeleteKey(k2.Id, pass)
+	if err != nil {
+		fmt.Println(err.Error())
+		t.FailNow()
+	}
+}
+
+func TestKeyStorePassphrase(t *testing.T) {
+	ks := new(KeyStorePassphrase)
+	ks.keysDirPath = DefaultDataDir()
+	pass := "foo"
+	k1, err := ks.GenerateNewKey(pass)
+	if err != nil {
+		fmt.Println(err.Error())
+		t.FailNow()
+	}
+
+	k2 := new(Key)
+	k2, err = ks.GetKey(k1.Id, pass)
+	if err != nil {
+		fmt.Println(err.Error())
+		t.FailNow()
+	}
+
+	if !reflect.DeepEqual(k1.Id, k2.Id) {
+		fmt.Println("key Id mismatch")
+		t.FailNow()
+	}
+
+	if k1.Flags != k2.Flags {
+		fmt.Println("key Flags mismatch")
+		t.FailNow()
+	}
+
+	if !reflect.DeepEqual(k1.PrivateKey, k2.PrivateKey) {
+		fmt.Println("key PrivateKey mismatch")
+		t.FailNow()
+	}
+
+	err = ks.DeleteKey(k2.Id, pass) // also to clean up created files
+	if err != nil {
+		fmt.Println(err.Error())
+		t.FailNow()
+	}
+}
+
+func TestKeyStorePassphraseDecryptionFail(t *testing.T) {
+	ks := new(KeyStorePassphrase)
+	ks.keysDirPath = DefaultDataDir()
+	pass := "foo"
+	k1, err := ks.GenerateNewKey(pass)
+	if err != nil {
+		fmt.Println(err.Error())
+		t.FailNow()
+	}
+
+	_, err = ks.GetKey(k1.Id, "bar") // wrong passphrase
+	// fmt.Println(err.Error())
+	if err == nil {
+		t.FailNow()
+	}
+
+	err = ks.DeleteKey(k1.Id, "bar") // wrong passphrase
+	if err == nil {
+		fmt.Println(err.Error())
+		t.FailNow()
+	}
+
+	err = ks.DeleteKey(k1.Id, pass) // to clean up
+	if err != nil {
+		fmt.Println(err.Error())
+		t.FailNow()
+	}
+}
+
+func TestKeyMixedEntropy(t *testing.T) {
+	GetEntropyTinFoilHat()
+}