diff options
Diffstat (limited to '')
-rw-r--r-- | src/hash/fnv/fnv.go | 371 | ||||
-rw-r--r-- | src/hash/fnv/fnv_test.go | 255 |
2 files changed, 626 insertions, 0 deletions
diff --git a/src/hash/fnv/fnv.go b/src/hash/fnv/fnv.go new file mode 100644 index 0000000..0fce177 --- /dev/null +++ b/src/hash/fnv/fnv.go @@ -0,0 +1,371 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package fnv implements FNV-1 and FNV-1a, non-cryptographic hash functions +// created by Glenn Fowler, Landon Curt Noll, and Phong Vo. +// See +// https://en.wikipedia.org/wiki/Fowler-Noll-Vo_hash_function. +// +// All the hash.Hash implementations returned by this package also +// implement encoding.BinaryMarshaler and encoding.BinaryUnmarshaler to +// marshal and unmarshal the internal state of the hash. +package fnv + +import ( + "errors" + "hash" + "math/bits" +) + +type ( + sum32 uint32 + sum32a uint32 + sum64 uint64 + sum64a uint64 + sum128 [2]uint64 + sum128a [2]uint64 +) + +const ( + offset32 = 2166136261 + offset64 = 14695981039346656037 + offset128Lower = 0x62b821756295c58d + offset128Higher = 0x6c62272e07bb0142 + prime32 = 16777619 + prime64 = 1099511628211 + prime128Lower = 0x13b + prime128Shift = 24 +) + +// New32 returns a new 32-bit FNV-1 hash.Hash. +// Its Sum method will lay the value out in big-endian byte order. +func New32() hash.Hash32 { + var s sum32 = offset32 + return &s +} + +// New32a returns a new 32-bit FNV-1a hash.Hash. +// Its Sum method will lay the value out in big-endian byte order. +func New32a() hash.Hash32 { + var s sum32a = offset32 + return &s +} + +// New64 returns a new 64-bit FNV-1 hash.Hash. +// Its Sum method will lay the value out in big-endian byte order. +func New64() hash.Hash64 { + var s sum64 = offset64 + return &s +} + +// New64a returns a new 64-bit FNV-1a hash.Hash. +// Its Sum method will lay the value out in big-endian byte order. +func New64a() hash.Hash64 { + var s sum64a = offset64 + return &s +} + +// New128 returns a new 128-bit FNV-1 hash.Hash. +// Its Sum method will lay the value out in big-endian byte order. +func New128() hash.Hash { + var s sum128 + s[0] = offset128Higher + s[1] = offset128Lower + return &s +} + +// New128a returns a new 128-bit FNV-1a hash.Hash. +// Its Sum method will lay the value out in big-endian byte order. +func New128a() hash.Hash { + var s sum128a + s[0] = offset128Higher + s[1] = offset128Lower + return &s +} + +func (s *sum32) Reset() { *s = offset32 } +func (s *sum32a) Reset() { *s = offset32 } +func (s *sum64) Reset() { *s = offset64 } +func (s *sum64a) Reset() { *s = offset64 } +func (s *sum128) Reset() { s[0] = offset128Higher; s[1] = offset128Lower } +func (s *sum128a) Reset() { s[0] = offset128Higher; s[1] = offset128Lower } + +func (s *sum32) Sum32() uint32 { return uint32(*s) } +func (s *sum32a) Sum32() uint32 { return uint32(*s) } +func (s *sum64) Sum64() uint64 { return uint64(*s) } +func (s *sum64a) Sum64() uint64 { return uint64(*s) } + +func (s *sum32) Write(data []byte) (int, error) { + hash := *s + for _, c := range data { + hash *= prime32 + hash ^= sum32(c) + } + *s = hash + return len(data), nil +} + +func (s *sum32a) Write(data []byte) (int, error) { + hash := *s + for _, c := range data { + hash ^= sum32a(c) + hash *= prime32 + } + *s = hash + return len(data), nil +} + +func (s *sum64) Write(data []byte) (int, error) { + hash := *s + for _, c := range data { + hash *= prime64 + hash ^= sum64(c) + } + *s = hash + return len(data), nil +} + +func (s *sum64a) Write(data []byte) (int, error) { + hash := *s + for _, c := range data { + hash ^= sum64a(c) + hash *= prime64 + } + *s = hash + return len(data), nil +} + +func (s *sum128) Write(data []byte) (int, error) { + for _, c := range data { + // Compute the multiplication + s0, s1 := bits.Mul64(prime128Lower, s[1]) + s0 += s[1]<<prime128Shift + prime128Lower*s[0] + // Update the values + s[1] = s1 + s[0] = s0 + s[1] ^= uint64(c) + } + return len(data), nil +} + +func (s *sum128a) Write(data []byte) (int, error) { + for _, c := range data { + s[1] ^= uint64(c) + // Compute the multiplication + s0, s1 := bits.Mul64(prime128Lower, s[1]) + s0 += s[1]<<prime128Shift + prime128Lower*s[0] + // Update the values + s[1] = s1 + s[0] = s0 + } + return len(data), nil +} + +func (s *sum32) Size() int { return 4 } +func (s *sum32a) Size() int { return 4 } +func (s *sum64) Size() int { return 8 } +func (s *sum64a) Size() int { return 8 } +func (s *sum128) Size() int { return 16 } +func (s *sum128a) Size() int { return 16 } + +func (s *sum32) BlockSize() int { return 1 } +func (s *sum32a) BlockSize() int { return 1 } +func (s *sum64) BlockSize() int { return 1 } +func (s *sum64a) BlockSize() int { return 1 } +func (s *sum128) BlockSize() int { return 1 } +func (s *sum128a) BlockSize() int { return 1 } + +func (s *sum32) Sum(in []byte) []byte { + v := uint32(*s) + return append(in, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) +} + +func (s *sum32a) Sum(in []byte) []byte { + v := uint32(*s) + return append(in, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) +} + +func (s *sum64) Sum(in []byte) []byte { + v := uint64(*s) + return append(in, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) +} + +func (s *sum64a) Sum(in []byte) []byte { + v := uint64(*s) + return append(in, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) +} + +func (s *sum128) Sum(in []byte) []byte { + return append(in, + byte(s[0]>>56), byte(s[0]>>48), byte(s[0]>>40), byte(s[0]>>32), byte(s[0]>>24), byte(s[0]>>16), byte(s[0]>>8), byte(s[0]), + byte(s[1]>>56), byte(s[1]>>48), byte(s[1]>>40), byte(s[1]>>32), byte(s[1]>>24), byte(s[1]>>16), byte(s[1]>>8), byte(s[1]), + ) +} + +func (s *sum128a) Sum(in []byte) []byte { + return append(in, + byte(s[0]>>56), byte(s[0]>>48), byte(s[0]>>40), byte(s[0]>>32), byte(s[0]>>24), byte(s[0]>>16), byte(s[0]>>8), byte(s[0]), + byte(s[1]>>56), byte(s[1]>>48), byte(s[1]>>40), byte(s[1]>>32), byte(s[1]>>24), byte(s[1]>>16), byte(s[1]>>8), byte(s[1]), + ) +} + +const ( + magic32 = "fnv\x01" + magic32a = "fnv\x02" + magic64 = "fnv\x03" + magic64a = "fnv\x04" + magic128 = "fnv\x05" + magic128a = "fnv\x06" + marshaledSize32 = len(magic32) + 4 + marshaledSize64 = len(magic64) + 8 + marshaledSize128 = len(magic128) + 8*2 +) + +func (s *sum32) MarshalBinary() ([]byte, error) { + b := make([]byte, 0, marshaledSize32) + b = append(b, magic32...) + b = appendUint32(b, uint32(*s)) + return b, nil +} + +func (s *sum32a) MarshalBinary() ([]byte, error) { + b := make([]byte, 0, marshaledSize32) + b = append(b, magic32a...) + b = appendUint32(b, uint32(*s)) + return b, nil +} + +func (s *sum64) MarshalBinary() ([]byte, error) { + b := make([]byte, 0, marshaledSize64) + b = append(b, magic64...) + b = appendUint64(b, uint64(*s)) + return b, nil + +} + +func (s *sum64a) MarshalBinary() ([]byte, error) { + b := make([]byte, 0, marshaledSize64) + b = append(b, magic64a...) + b = appendUint64(b, uint64(*s)) + return b, nil +} + +func (s *sum128) MarshalBinary() ([]byte, error) { + b := make([]byte, 0, marshaledSize128) + b = append(b, magic128...) + b = appendUint64(b, s[0]) + b = appendUint64(b, s[1]) + return b, nil +} + +func (s *sum128a) MarshalBinary() ([]byte, error) { + b := make([]byte, 0, marshaledSize128) + b = append(b, magic128a...) + b = appendUint64(b, s[0]) + b = appendUint64(b, s[1]) + return b, nil +} + +func (s *sum32) UnmarshalBinary(b []byte) error { + if len(b) < len(magic32) || string(b[:len(magic32)]) != magic32 { + return errors.New("hash/fnv: invalid hash state identifier") + } + if len(b) != marshaledSize32 { + return errors.New("hash/fnv: invalid hash state size") + } + *s = sum32(readUint32(b[4:])) + return nil +} + +func (s *sum32a) UnmarshalBinary(b []byte) error { + if len(b) < len(magic32a) || string(b[:len(magic32a)]) != magic32a { + return errors.New("hash/fnv: invalid hash state identifier") + } + if len(b) != marshaledSize32 { + return errors.New("hash/fnv: invalid hash state size") + } + *s = sum32a(readUint32(b[4:])) + return nil +} + +func (s *sum64) UnmarshalBinary(b []byte) error { + if len(b) < len(magic64) || string(b[:len(magic64)]) != magic64 { + return errors.New("hash/fnv: invalid hash state identifier") + } + if len(b) != marshaledSize64 { + return errors.New("hash/fnv: invalid hash state size") + } + *s = sum64(readUint64(b[4:])) + return nil +} + +func (s *sum64a) UnmarshalBinary(b []byte) error { + if len(b) < len(magic64a) || string(b[:len(magic64a)]) != magic64a { + return errors.New("hash/fnv: invalid hash state identifier") + } + if len(b) != marshaledSize64 { + return errors.New("hash/fnv: invalid hash state size") + } + *s = sum64a(readUint64(b[4:])) + return nil +} + +func (s *sum128) UnmarshalBinary(b []byte) error { + if len(b) < len(magic128) || string(b[:len(magic128)]) != magic128 { + return errors.New("hash/fnv: invalid hash state identifier") + } + if len(b) != marshaledSize128 { + return errors.New("hash/fnv: invalid hash state size") + } + s[0] = readUint64(b[4:]) + s[1] = readUint64(b[12:]) + return nil +} + +func (s *sum128a) UnmarshalBinary(b []byte) error { + if len(b) < len(magic128a) || string(b[:len(magic128a)]) != magic128a { + return errors.New("hash/fnv: invalid hash state identifier") + } + if len(b) != marshaledSize128 { + return errors.New("hash/fnv: invalid hash state size") + } + s[0] = readUint64(b[4:]) + s[1] = readUint64(b[12:]) + return nil +} + +func readUint32(b []byte) uint32 { + _ = b[3] + return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 +} + +func appendUint32(b []byte, x uint32) []byte { + a := [4]byte{ + byte(x >> 24), + byte(x >> 16), + byte(x >> 8), + byte(x), + } + return append(b, a[:]...) +} + +func appendUint64(b []byte, x uint64) []byte { + a := [8]byte{ + byte(x >> 56), + byte(x >> 48), + byte(x >> 40), + byte(x >> 32), + byte(x >> 24), + byte(x >> 16), + byte(x >> 8), + byte(x), + } + return append(b, a[:]...) +} + +func readUint64(b []byte) uint64 { + _ = b[7] + return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | + uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 +} diff --git a/src/hash/fnv/fnv_test.go b/src/hash/fnv/fnv_test.go new file mode 100644 index 0000000..7b1f7a3 --- /dev/null +++ b/src/hash/fnv/fnv_test.go @@ -0,0 +1,255 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fnv + +import ( + "bytes" + "encoding" + "encoding/binary" + "hash" + "io" + "testing" +) + +type golden struct { + out []byte + in string + halfState string // marshaled hash state after first half of in written, used by TestGoldenMarshal +} + +var golden32 = []golden{ + {[]byte{0x81, 0x1c, 0x9d, 0xc5}, "", "fnv\x01\x81\x1c\x9d\xc5"}, + {[]byte{0x05, 0x0c, 0x5d, 0x7e}, "a", "fnv\x01\x81\x1c\x9d\xc5"}, + {[]byte{0x70, 0x77, 0x2d, 0x38}, "ab", "fnv\x01\x05\f]~"}, + {[]byte{0x43, 0x9c, 0x2f, 0x4b}, "abc", "fnv\x01\x05\f]~"}, +} + +var golden32a = []golden{ + {[]byte{0x81, 0x1c, 0x9d, 0xc5}, "", "fnv\x02\x81\x1c\x9d\xc5"}, + {[]byte{0xe4, 0x0c, 0x29, 0x2c}, "a", "fnv\x02\x81\x1c\x9d\xc5"}, + {[]byte{0x4d, 0x25, 0x05, 0xca}, "ab", "fnv\x02\xe4\f),"}, + {[]byte{0x1a, 0x47, 0xe9, 0x0b}, "abc", "fnv\x02\xe4\f),"}, +} + +var golden64 = []golden{ + {[]byte{0xcb, 0xf2, 0x9c, 0xe4, 0x84, 0x22, 0x23, 0x25}, "", "fnv\x03\xcb\xf2\x9c\xe4\x84\"#%"}, + {[]byte{0xaf, 0x63, 0xbd, 0x4c, 0x86, 0x01, 0xb7, 0xbe}, "a", "fnv\x03\xcb\xf2\x9c\xe4\x84\"#%"}, + {[]byte{0x08, 0x32, 0x67, 0x07, 0xb4, 0xeb, 0x37, 0xb8}, "ab", "fnv\x03\xafc\xbdL\x86\x01\xb7\xbe"}, + {[]byte{0xd8, 0xdc, 0xca, 0x18, 0x6b, 0xaf, 0xad, 0xcb}, "abc", "fnv\x03\xafc\xbdL\x86\x01\xb7\xbe"}, +} + +var golden64a = []golden{ + {[]byte{0xcb, 0xf2, 0x9c, 0xe4, 0x84, 0x22, 0x23, 0x25}, "", "fnv\x04\xcb\xf2\x9c\xe4\x84\"#%"}, + {[]byte{0xaf, 0x63, 0xdc, 0x4c, 0x86, 0x01, 0xec, 0x8c}, "a", "fnv\x04\xcb\xf2\x9c\xe4\x84\"#%"}, + {[]byte{0x08, 0x9c, 0x44, 0x07, 0xb5, 0x45, 0x98, 0x6a}, "ab", "fnv\x04\xafc\xdcL\x86\x01\xec\x8c"}, + {[]byte{0xe7, 0x1f, 0xa2, 0x19, 0x05, 0x41, 0x57, 0x4b}, "abc", "fnv\x04\xafc\xdcL\x86\x01\xec\x8c"}, +} + +var golden128 = []golden{ + {[]byte{0x6c, 0x62, 0x27, 0x2e, 0x07, 0xbb, 0x01, 0x42, 0x62, 0xb8, 0x21, 0x75, 0x62, 0x95, 0xc5, 0x8d}, "", "fnv\x05lb'.\a\xbb\x01Bb\xb8!ub\x95ō"}, + {[]byte{0xd2, 0x28, 0xcb, 0x69, 0x10, 0x1a, 0x8c, 0xaf, 0x78, 0x91, 0x2b, 0x70, 0x4e, 0x4a, 0x14, 0x1e}, "a", "fnv\x05lb'.\a\xbb\x01Bb\xb8!ub\x95ō"}, + {[]byte{0x8, 0x80, 0x94, 0x5a, 0xee, 0xab, 0x1b, 0xe9, 0x5a, 0xa0, 0x73, 0x30, 0x55, 0x26, 0xc0, 0x88}, "ab", "fnv\x05\xd2(\xcbi\x10\x1a\x8c\xafx\x91+pNJ\x14\x1e"}, + {[]byte{0xa6, 0x8b, 0xb2, 0xa4, 0x34, 0x8b, 0x58, 0x22, 0x83, 0x6d, 0xbc, 0x78, 0xc6, 0xae, 0xe7, 0x3b}, "abc", "fnv\x05\xd2(\xcbi\x10\x1a\x8c\xafx\x91+pNJ\x14\x1e"}, +} + +var golden128a = []golden{ + {[]byte{0x6c, 0x62, 0x27, 0x2e, 0x07, 0xbb, 0x01, 0x42, 0x62, 0xb8, 0x21, 0x75, 0x62, 0x95, 0xc5, 0x8d}, "", "fnv\x06lb'.\a\xbb\x01Bb\xb8!ub\x95ō"}, + {[]byte{0xd2, 0x28, 0xcb, 0x69, 0x6f, 0x1a, 0x8c, 0xaf, 0x78, 0x91, 0x2b, 0x70, 0x4e, 0x4a, 0x89, 0x64}, "a", "fnv\x06lb'.\a\xbb\x01Bb\xb8!ub\x95ō"}, + {[]byte{0x08, 0x80, 0x95, 0x44, 0xbb, 0xab, 0x1b, 0xe9, 0x5a, 0xa0, 0x73, 0x30, 0x55, 0xb6, 0x9a, 0x62}, "ab", "fnv\x06\xd2(\xcbio\x1a\x8c\xafx\x91+pNJ\x89d"}, + {[]byte{0xa6, 0x8d, 0x62, 0x2c, 0xec, 0x8b, 0x58, 0x22, 0x83, 0x6d, 0xbc, 0x79, 0x77, 0xaf, 0x7f, 0x3b}, "abc", "fnv\x06\xd2(\xcbio\x1a\x8c\xafx\x91+pNJ\x89d"}, +} + +func TestGolden32(t *testing.T) { + testGolden(t, New32(), golden32) +} + +func TestGolden32a(t *testing.T) { + testGolden(t, New32a(), golden32a) +} + +func TestGolden64(t *testing.T) { + testGolden(t, New64(), golden64) +} + +func TestGolden64a(t *testing.T) { + testGolden(t, New64a(), golden64a) +} + +func TestGolden128(t *testing.T) { + testGolden(t, New128(), golden128) +} + +func TestGolden128a(t *testing.T) { + testGolden(t, New128a(), golden128a) +} + +func testGolden(t *testing.T, hash hash.Hash, gold []golden) { + for _, g := range gold { + hash.Reset() + done, error := hash.Write([]byte(g.in)) + if error != nil { + t.Fatalf("write error: %s", error) + } + if done != len(g.in) { + t.Fatalf("wrote only %d out of %d bytes", done, len(g.in)) + } + if actual := hash.Sum(nil); !bytes.Equal(g.out, actual) { + t.Errorf("hash(%q) = 0x%x want 0x%x", g.in, actual, g.out) + } + } +} + +func TestGoldenMarshal(t *testing.T) { + tests := []struct { + name string + newHash func() hash.Hash + gold []golden + }{ + {"32", func() hash.Hash { return New32() }, golden32}, + {"32a", func() hash.Hash { return New32a() }, golden32a}, + {"64", func() hash.Hash { return New64() }, golden64}, + {"64a", func() hash.Hash { return New64a() }, golden64a}, + {"128", func() hash.Hash { return New128() }, golden128}, + {"128a", func() hash.Hash { return New128a() }, golden128a}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for _, g := range tt.gold { + h := tt.newHash() + h2 := tt.newHash() + + io.WriteString(h, g.in[:len(g.in)/2]) + + state, err := h.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + t.Errorf("could not marshal: %v", err) + continue + } + + if string(state) != g.halfState { + t.Errorf("checksum(%q) state = %q, want %q", g.in, state, g.halfState) + continue + } + + if err := h2.(encoding.BinaryUnmarshaler).UnmarshalBinary(state); err != nil { + t.Errorf("could not unmarshal: %v", err) + continue + } + + io.WriteString(h, g.in[len(g.in)/2:]) + io.WriteString(h2, g.in[len(g.in)/2:]) + + if actual, actual2 := h.Sum(nil), h2.Sum(nil); !bytes.Equal(actual, actual2) { + t.Errorf("hash(%q) = 0x%x != marshaled 0x%x", g.in, actual, actual2) + } + } + }) + } +} + +func TestIntegrity32(t *testing.T) { + testIntegrity(t, New32()) +} + +func TestIntegrity32a(t *testing.T) { + testIntegrity(t, New32a()) +} + +func TestIntegrity64(t *testing.T) { + testIntegrity(t, New64()) +} + +func TestIntegrity64a(t *testing.T) { + testIntegrity(t, New64a()) +} +func TestIntegrity128(t *testing.T) { + testIntegrity(t, New128()) +} + +func TestIntegrity128a(t *testing.T) { + testIntegrity(t, New128a()) +} + +func testIntegrity(t *testing.T, h hash.Hash) { + data := []byte{'1', '2', 3, 4, 5} + h.Write(data) + sum := h.Sum(nil) + + if size := h.Size(); size != len(sum) { + t.Fatalf("Size()=%d but len(Sum())=%d", size, len(sum)) + } + + if a := h.Sum(nil); !bytes.Equal(sum, a) { + t.Fatalf("first Sum()=0x%x, second Sum()=0x%x", sum, a) + } + + h.Reset() + h.Write(data) + if a := h.Sum(nil); !bytes.Equal(sum, a) { + t.Fatalf("Sum()=0x%x, but after Reset() Sum()=0x%x", sum, a) + } + + h.Reset() + h.Write(data[:2]) + h.Write(data[2:]) + if a := h.Sum(nil); !bytes.Equal(sum, a) { + t.Fatalf("Sum()=0x%x, but with partial writes, Sum()=0x%x", sum, a) + } + + switch h.Size() { + case 4: + sum32 := h.(hash.Hash32).Sum32() + if sum32 != binary.BigEndian.Uint32(sum) { + t.Fatalf("Sum()=0x%x, but Sum32()=0x%x", sum, sum32) + } + case 8: + sum64 := h.(hash.Hash64).Sum64() + if sum64 != binary.BigEndian.Uint64(sum) { + t.Fatalf("Sum()=0x%x, but Sum64()=0x%x", sum, sum64) + } + case 16: + // There's no Sum128 function, so we don't need to test anything here. + } +} + +func BenchmarkFnv32KB(b *testing.B) { + benchmarkKB(b, New32()) +} + +func BenchmarkFnv32aKB(b *testing.B) { + benchmarkKB(b, New32a()) +} + +func BenchmarkFnv64KB(b *testing.B) { + benchmarkKB(b, New64()) +} + +func BenchmarkFnv64aKB(b *testing.B) { + benchmarkKB(b, New64a()) +} + +func BenchmarkFnv128KB(b *testing.B) { + benchmarkKB(b, New128()) +} + +func BenchmarkFnv128aKB(b *testing.B) { + benchmarkKB(b, New128a()) +} + +func benchmarkKB(b *testing.B, h hash.Hash) { + b.SetBytes(1024) + data := make([]byte, 1024) + for i := range data { + data[i] = byte(i) + } + in := make([]byte, 0, h.Size()) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + h.Reset() + h.Write(data) + h.Sum(in) + } +} |