add all files from Hong

This commit is contained in:
zhangsz
2025-06-30 09:23:28 +08:00
parent ceb1fe2640
commit 9b7d32fbd9
69 changed files with 7280 additions and 0 deletions

View File

@@ -0,0 +1,48 @@
// Copyright 2018-present the CoreDHCP Authors. All rights reserved
// This source code is licensed under the MIT license found in the
// LICENSE file in the root directory of this source tree.
// Package allocators provides the interface and the algorithm(s) for allocation of ipv6
// prefixes of various sizes within a larger prefix.
// There are many many parallels with memory allocation.
package allocators
import (
"errors"
"fmt"
"net"
)
// Allocator is the interface to the address allocator. It only finds and
// allocates blocks and is not concerned with lease-specific questions like
// expiration (ie garbage collection needs to be handled separately)
type Allocator interface {
// Allocate finds a suitable prefix of the given size and returns it.
//
// hint is a prefix, which the client desires especially, and that the
// allocator MAY try to return; the allocator SHOULD try to return a prefix of
// the same size as the given hint prefix. The allocator MUST NOT return an
// error if a prefix was successfully assigned, even if the prefix has nothing
// in common with the hinted prefix
Allocate(hint net.IPNet) (net.IPNet, error)
// Free returns the prefix containing the given network to the pool
//
// Free may return a DoubleFreeError if the prefix being returned was not
// previously allocated
Free(net.IPNet) error
}
// ErrDoubleFree is an error type returned by Allocator.Free() when a
// non-allocated block is passed
type ErrDoubleFree struct {
Loc net.IPNet
}
// String returns a human-readable error message for a DoubleFree error
func (err *ErrDoubleFree) Error() string {
return fmt.Sprint("Attempted to free unallocated block at ", err.Loc.String())
}
// ErrNoAddrAvail is returned when we can't allocate an IP because there's no unallocated space left
var ErrNoAddrAvail = errors.New("no address available to allocate")

View File

@@ -0,0 +1,135 @@
// Copyright 2018-present the CoreDHCP Authors. All rights reserved
// This source code is licensed under the MIT license found in the
// LICENSE file in the root directory of this source tree.
// This allocator only returns prefixes of a single size
// This is much simpler to implement (reduces the problem to an equivalent of
// single ip allocations), probably makes sense in cases where the available
// range is much larger than the expected number of clients. Also is what KEA
// does so at least it's not worse than that
package bitmap
import (
"errors"
"fmt"
"net"
"strconv"
"sync"
"github.com/bits-and-blooms/bitset"
"github.com/coredhcp/coredhcp/logger"
"github.com/coredhcp/coredhcp/plugins/allocators"
)
var log = logger.GetLogger("plugins/allocators/bitmap")
// Allocator is a prefix allocator allocating in chunks of a fixed size
// regardless of the size requested by the client.
// It consumes an amount of memory proportional to the total amount of available prefixes
type Allocator struct {
containing net.IPNet
page int
bitmap *bitset.BitSet
l sync.Mutex
}
// prefix must verify: containing.Mask.Size < prefix.Mask.Size < page
func (a *Allocator) toIndex(base net.IP) (uint, error) {
value, err := allocators.Offset(base, a.containing.IP, a.page)
if err != nil {
return 0, fmt.Errorf("Cannot compute prefix index: %w", err)
}
return uint(value), nil
}
func (a *Allocator) toPrefix(idx uint) (net.IP, error) {
return allocators.AddPrefixes(a.containing.IP, uint64(idx), uint64(a.page))
}
// Allocate reserves a maxsize-sized block and returns a block of size
// min(maxsize, hint.size)
func (a *Allocator) Allocate(hint net.IPNet) (ret net.IPNet, err error) {
// Ensure size is max(maxsize, hint.size)
reqSize, hintErr := hint.Mask.Size()
if reqSize < a.page || hintErr != 128 {
reqSize = a.page
}
ret.Mask = net.CIDRMask(reqSize, 128)
// Try to allocate the requested prefix
a.l.Lock()
defer a.l.Unlock()
if hint.IP.To16() != nil && a.containing.Contains(hint.IP) {
idx, hintErr := a.toIndex(hint.IP)
if hintErr == nil && !a.bitmap.Test(idx) {
a.bitmap.Set(idx)
ret.IP, err = a.toPrefix(idx)
return
}
}
// Find a free prefix
next, ok := a.bitmap.NextClear(0)
if !ok {
err = allocators.ErrNoAddrAvail
return
}
a.bitmap.Set(next)
ret.IP, err = a.toPrefix(next)
if err != nil {
// This violates the assumption that every index in the bitmap maps back to a valid prefix
err = fmt.Errorf("BUG: could not get prefix from allocation: %w", err)
a.bitmap.Clear(next)
}
return
}
// Free returns the given prefix to the available pool if it was taken.
func (a *Allocator) Free(prefix net.IPNet) error {
idx, err := a.toIndex(prefix.IP.Mask(prefix.Mask))
if err != nil {
return fmt.Errorf("Could not find prefix in pool: %w", err)
}
a.l.Lock()
defer a.l.Unlock()
if !a.bitmap.Test(idx) {
return &allocators.ErrDoubleFree{Loc: prefix}
}
a.bitmap.Clear(idx)
return nil
}
// NewBitmapAllocator creates a new allocator, allocating /`size` prefixes
// carved out of the given `pool` prefix
func NewBitmapAllocator(pool net.IPNet, size int) (*Allocator, error) {
poolSize, _ := pool.Mask.Size()
allocOrder := size - poolSize
if allocOrder < 0 {
return nil, errors.New("The size of allocated prefixes cannot be larger than the pool they're allocated from")
} else if allocOrder >= strconv.IntSize {
return nil, fmt.Errorf("A pool with more than 2^%d items is not representable", size-poolSize)
} else if allocOrder >= 32 {
log.Warningln("Using a pool of more than 2^32 elements may result in large memory consumption")
}
if !(1<<uint(allocOrder) <= bitset.Cap()) {
return nil, errors.New("Can't fit this pool using the bitmap allocator")
}
alloc := Allocator{
containing: pool,
page: size,
bitmap: bitset.New(1 << uint(allocOrder)),
}
return &alloc, nil
}

View File

@@ -0,0 +1,122 @@
// Copyright 2018-present the CoreDHCP Authors. All rights reserved
// This source code is licensed under the MIT license found in the
// LICENSE file in the root directory of this source tree.
package bitmap
// This allocator handles IPv4 assignments with a similar logic to the base bitmap, but a simpler
// implementation due to the ability to just use uint32 for IPv4 addresses
import (
"encoding/binary"
"errors"
"fmt"
"net"
"sync"
"github.com/bits-and-blooms/bitset"
"github.com/coredhcp/coredhcp/plugins/allocators"
)
var (
errNotInRange = errors.New("IPv4 address outside of allowed range")
errInvalidIP = errors.New("invalid IPv4 address passed as input")
)
// IPv4Allocator allocates IPv4 addresses, tracking utilization with a bitmap
type IPv4Allocator struct {
start uint32
end uint32
// This bitset implementation isn't goroutine-safe, we protect it with a mutex for now
// until we can swap for another concurrent implementation
bitmap *bitset.BitSet
l sync.Mutex
}
func (a *IPv4Allocator) toIP(offset uint32) net.IP {
if offset > a.end-a.start {
panic("BUG: offset out of bounds")
}
r := make(net.IP, net.IPv4len)
binary.BigEndian.PutUint32(r, a.start+offset)
return r
}
func (a *IPv4Allocator) toOffset(ip net.IP) (uint, error) {
if ip.To4() == nil {
return 0, errInvalidIP
}
intIP := binary.BigEndian.Uint32(ip.To4())
if intIP < a.start || intIP > a.end {
return 0, errNotInRange
}
return uint(intIP - a.start), nil
}
// Allocate reserves an IP for a client
func (a *IPv4Allocator) Allocate(hint net.IPNet) (n net.IPNet, err error) {
n.Mask = net.CIDRMask(32, 32)
// This is just a hint, ignore any error with it
hintOffset, _ := a.toOffset(hint.IP)
a.l.Lock()
defer a.l.Unlock()
var next uint
// First try the exact match
if !a.bitmap.Test(hintOffset) {
next = hintOffset
} else {
// Then any available address
avail, ok := a.bitmap.NextClear(0)
if !ok {
return n, allocators.ErrNoAddrAvail
}
next = avail
}
a.bitmap.Set(next)
n.IP = a.toIP(uint32(next))
return
}
// Free releases the given IP
func (a *IPv4Allocator) Free(n net.IPNet) error {
offset, err := a.toOffset(n.IP)
if err != nil {
return errNotInRange
}
a.l.Lock()
defer a.l.Unlock()
if !a.bitmap.Test(uint(offset)) {
return &allocators.ErrDoubleFree{Loc: n}
}
a.bitmap.Clear(offset)
return nil
}
// NewIPv4Allocator creates a new allocator suitable for giving out IPv4 addresses
func NewIPv4Allocator(start, end net.IP) (*IPv4Allocator, error) {
if start.To4() == nil || end.To4() == nil {
return nil, fmt.Errorf("invalid IPv4 addresses given to create the allocator: [%s,%s]", start, end)
}
alloc := IPv4Allocator{
start: binary.BigEndian.Uint32(start.To4()),
end: binary.BigEndian.Uint32(end.To4()),
}
if alloc.start > alloc.end {
return nil, errors.New("no IPs in the given range to allocate")
}
alloc.bitmap = bitset.New(uint(alloc.end - alloc.start + 1))
return &alloc, nil
}

View File

@@ -0,0 +1,63 @@
// Copyright 2018-present the CoreDHCP Authors. All rights reserved
// This source code is licensed under the MIT license found in the
// LICENSE file in the root directory of this source tree.
package bitmap
import (
"net"
"testing"
)
func getv4Allocator() *IPv4Allocator {
alloc, err := NewIPv4Allocator(net.IPv4(192, 0, 2, 0), net.IPv4(192, 0, 2, 255))
if err != nil {
panic(err)
}
return alloc
}
func Test4Alloc(t *testing.T) {
alloc := getv4Allocator()
net1, err := alloc.Allocate(net.IPNet{})
if err != nil {
t.Fatal(err)
}
net2, err := alloc.Allocate(net.IPNet{})
if err != nil {
t.Fatal(err)
}
if net1.IP.Equal(net2.IP) {
t.Fatal("That address was already allocated")
}
err = alloc.Free(net1)
if err != nil {
t.Fatal(err)
}
err = alloc.Free(net1)
if err == nil {
t.Fatal("Expected DoubleFree error")
}
}
func Test4OutOfPool(t *testing.T) {
alloc := getv4Allocator()
hint := net.IPv4(198, 51, 100, 5)
res, err := alloc.Allocate(net.IPNet{IP: hint, Mask: net.CIDRMask(32, 32)})
if err != nil {
t.Fatalf("Failed to allocate with invalid hint: %v", err)
}
_, prefix, _ := net.ParseCIDR("192.0.2.0/24")
if !prefix.Contains(res.IP) {
t.Fatal("Obtained prefix outside of range: ", res)
}
if prefLen, totalLen := res.Mask.Size(); prefLen != 32 || totalLen != 32 {
t.Fatalf("Prefixes have wrong size %d/%d", prefLen, totalLen)
}
}

View File

@@ -0,0 +1,137 @@
// Copyright 2018-present the CoreDHCP Authors. All rights reserved
// This source code is licensed under the MIT license found in the
// LICENSE file in the root directory of this source tree.
package bitmap
import (
"math"
"math/rand"
"net"
"testing"
"github.com/bits-and-blooms/bitset"
)
func getAllocator(bits int) *Allocator {
_, prefix, err := net.ParseCIDR("2001:db8::/56")
if err != nil {
panic(err)
}
alloc, err := NewBitmapAllocator(*prefix, 56+bits)
if err != nil {
panic(err)
}
return alloc
}
func TestAlloc(t *testing.T) {
alloc := getAllocator(8)
net, err := alloc.Allocate(net.IPNet{})
if err != nil {
t.Fatal(err)
}
err = alloc.Free(net)
if err != nil {
t.Fatal(err)
}
err = alloc.Free(net)
if err == nil {
t.Fatal("Expected DoubleFree error")
}
}
func TestExhaust(t *testing.T) {
_, prefix, _ := net.ParseCIDR("2001:db8::/62")
alloc, _ := NewBitmapAllocator(*prefix, 64)
allocd := []net.IPNet{}
for i := 0; i < 4; i++ {
net, err := alloc.Allocate(net.IPNet{Mask: net.CIDRMask(64, 128)})
if err != nil {
t.Fatalf("Error before exhaustion: %v", err)
}
allocd = append(allocd, net)
}
_, err := alloc.Allocate(net.IPNet{})
if err == nil {
t.Fatalf("Successfully allocated more prefixes than there are in the pool")
}
err = alloc.Free(allocd[1])
if err != nil {
t.Fatalf("Could not free: %v", err)
}
net, err := alloc.Allocate(allocd[1])
if err != nil {
t.Fatalf("Could not reallocate after free: %v", err)
}
if !net.IP.Equal(allocd[1].IP) || net.Mask.String() != allocd[1].Mask.String() {
t.Fatalf("Did not obtain the right network after free: got %v, expected %v", net, allocd[1])
}
}
func TestOutOfPool(t *testing.T) {
alloc := getAllocator(8)
_, prefix, _ := net.ParseCIDR("fe80:abcd::/48")
res, err := alloc.Allocate(*prefix)
if err != nil {
t.Fatalf("Failed to allocate with invalid hint: %v", err)
}
if !alloc.containing.Contains(res.IP) {
t.Fatal("Obtained prefix outside of range: ", res)
}
if prefLen, totalLen := res.Mask.Size(); prefLen != 64 || totalLen != 128 {
t.Fatalf("Prefixes have wrong size %d/%d", prefLen, totalLen)
}
}
func prefixSizeForAllocs(allocs int) int {
return int(math.Ceil(math.Log2(float64(allocs))))
}
// Benchmark parallel Allocate, when the bitmap is mostly empty and we're allocating few values
// compared to the available allocations
func BenchmarkParallelAllocInitiallyEmpty(b *testing.B) {
// Run with -race to debug concurrency issues
alloc := getAllocator(prefixSizeForAllocs(b.N) + 2) // Use max 25% of the bitmap (initially empty)
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if net, err := alloc.Allocate(net.IPNet{}); err != nil {
b.Logf("Could not allocate (got %v and an error): %v", net, err)
b.Fail()
}
}
})
}
func BenchmarkParallelAllocPartiallyFilled(b *testing.B) {
// We'll make a bitmap with 2x the number of allocs we want to make.
// Then randomly fill it to about 50% utilization
alloc := getAllocator(prefixSizeForAllocs(b.N) + 1)
// Build a replacement bitmap that we'll put in the allocator, with approx. 50% of values filled
newbmap := make([]uint64, alloc.bitmap.Len())
for i := uint(0); i < alloc.bitmap.Len(); i++ {
newbmap[i] = rand.Uint64()
}
alloc.bitmap = bitset.From(newbmap)
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if net, err := alloc.Allocate(net.IPNet{}); err != nil {
b.Logf("Could not allocate (got %v and an error): %v", net, err)
b.Fail()
}
}
})
}

View File

@@ -0,0 +1,126 @@
// Copyright 2018-present the CoreDHCP Authors. All rights reserved
// This source code is licensed under the MIT license found in the
// LICENSE file in the root directory of this source tree.
// Provides functions to add/subtract ipv6 addresses, for use in offset
// calculations in allocators
package allocators
import (
"bytes"
"encoding/binary"
"errors"
"math/bits"
"net"
)
// ErrOverflow is returned when arithmetic operations on IPs carry bits
// over/under the 0th or 128th bit respectively
var ErrOverflow = errors.New("Operation overflows")
// Offset returns the absolute distance between addresses `a` and `b` in units
// of /`prefixLength` subnets.
// Both addresses will have a /`prefixLength` mask applied to them, any
// differences of less than that will be discarded
// If the distance is larger than 2^64 units of /`prefixLength` an error is returned
//
// This function is used in allocators to index bitmaps by an offset from the
// first ip of the range
func Offset(a, b net.IP, prefixLength int) (uint64, error) {
if prefixLength > 128 || prefixLength < 0 {
return 0, errors.New("prefix out of range")
}
reverse := bytes.Compare(a, b)
if reverse == 0 {
return 0, nil
} else if reverse < 0 {
a, b = b, a
}
// take an example of [a:b:c:d:e:f:g:h] [1:2:3:4:5:6:7:8]
// Cut the addresses as such: [a:b:c:d|e:f:g:h] [1:2:3:4|5:6:7:8] so we can use
// native integers for computation
ah, bh := binary.BigEndian.Uint64(a[:8]), binary.BigEndian.Uint64(b[:8])
if prefixLength <= 64 {
// [(a:b:c):d|e:f:g:h] - [(1:2:3):4|5:6:7:8]
// Only the high bits matter, so the distance always fits within 64 bits.
// We shift to remove anything to the right of the cut
// [(a:b:c):d] => [0:a:b:c]
return (ah - bh) >> (64 - uint(prefixLength)), nil
}
// General case where both high and low bits matter
al, bl := binary.BigEndian.Uint64(a[8:]), binary.BigEndian.Uint64(b[8:])
distanceLow, borrow := bits.Sub64(al, bl, 0)
// This is the distance between the high bits. depending on the prefix unit, we
// will shift this distance left or right
distanceHigh, _ := bits.Sub64(ah, bh, borrow) // [a:b:c:d] - [1:2:3:4]
// [a:b:c:(d|e:f:g):h] - [1:2:3:(4|5:6:7):8]
// we cut in the low bits (eg. between the parentheses)
// To ensure we stay within 64 bits, we need to ensure [a:b:c:d] - [1:2:3:4] = [0:0:0:d-4]
// so that we don't overflow when adding to the low bits
if distanceHigh >= (1 << (128 - uint(prefixLength))) {
return 0, ErrOverflow
}
// Schema of the carry and shifts:
// [a:b:c:(d]
// [e:f:g):h]
// <---------------> prefixLen
// <-> 128 - prefixLen (cut right)
// <-----> prefixLen - 64 (cut left)
//
// [a:b:c:(d] => [d:0:0:0]
distanceHigh <<= uint(prefixLength) - 64
// [e:f:g):h] => [0:e:f:g]
distanceLow >>= 128 - uint(prefixLength)
// [d:0:0:0] + [0:e:f:g] = (d:e:f:g)
return distanceHigh + distanceLow, nil
}
// AddPrefixes returns the `n`th /`unit` subnet after the `ip` base subnet. It
// is the converse operation of Offset(), used to retrieve a prefix from the
// index within the allocator table
func AddPrefixes(ip net.IP, n, unit uint64) (net.IP, error) {
if unit == 0 && n != 0 {
return net.IP{}, ErrOverflow
} else if n == 0 {
return ip, nil
}
if len(ip) != 16 {
// We don't actually care if they're true v6 or v4-mapped,
// but they need to be 128-bit to handle as 64-bit ints
return net.IP{}, errors.New("AddPrefixes needs 128-bit IPs")
}
// Compute as pairs of uint64 for easier operations
// This could all be 1 function call if go had 128-bit integers
iph, ipl := binary.BigEndian.Uint64(ip[:8]), binary.BigEndian.Uint64(ip[8:])
// Compute `n` /`unit` subnets as uint64 pair
var offh, offl uint64
if unit <= 64 {
offh = n << (64 - unit)
} else {
offh, offl = bits.Mul64(n, 1<<(128-unit))
}
// Now add the 2, check for overflow
ipl, carry := bits.Add64(offl, ipl, 0)
iph, carry = bits.Add64(offh, iph, carry)
if carry != 0 {
return net.IP{}, ErrOverflow
}
// Finally convert back to net.IP
ret := make(net.IP, net.IPv6len)
binary.BigEndian.PutUint64(ret[:8], iph)
binary.BigEndian.PutUint64(ret[8:], ipl)
return ret, nil
}

View File

@@ -0,0 +1,77 @@
// Copyright 2018-present the CoreDHCP Authors. All rights reserved
// This source code is licensed under the MIT license found in the
// LICENSE file in the root directory of this source tree.
package allocators
import (
"fmt"
"net"
"testing"
"math/rand"
)
func ExampleOffset() {
fmt.Println(Offset(net.ParseIP("2001:db8:0:aabb::"), net.ParseIP("2001:db8:ff::34"), 0))
fmt.Println(Offset(net.ParseIP("2001:db8:0:aabb::"), net.ParseIP("2001:db8:ff::34"), 16))
fmt.Println(Offset(net.ParseIP("2001:db8:0:aabb::"), net.ParseIP("2001:db8:ff::34"), 32))
fmt.Println(Offset(net.ParseIP("2001:db8:0:aabb::"), net.ParseIP("2001:db8:ff::34"), 48))
fmt.Println(Offset(net.ParseIP("2001:db8:0:aabb::"), net.ParseIP("2001:db8:ff::34"), 64))
fmt.Println(Offset(net.ParseIP("2001:db8:0:aabb::"), net.ParseIP("2001:db8:ff::34"), 73))
fmt.Println(Offset(net.ParseIP("2001:db8:0:aabb::"), net.ParseIP("2001:db8:ff::34"), 80))
fmt.Println(Offset(net.ParseIP("2001:db8:0:aabb::"), net.ParseIP("2001:db8:ff::34"), 96))
fmt.Println(Offset(net.ParseIP("2001:db8:0:aabb::"), net.ParseIP("2001:db8:ff::34"), 112))
fmt.Println(Offset(net.ParseIP("2001:db8:0:aabb::"), net.ParseIP("2001:db8:ff::34"), 128))
// Output:
// 0 <nil>
// 0 <nil>
// 0 <nil>
// 254 <nil>
// 16667973 <nil>
// 8534002176 <nil>
// 1092352278528 <nil>
// 71588398925611008 <nil>
// 0 Operation overflows
// 0 Operation overflows
}
func ExampleAddPrefixes() {
fmt.Println(AddPrefixes(net.ParseIP("2001:db8::"), 0xff, 64))
fmt.Println(AddPrefixes(net.ParseIP("2001:db8::"), 0x1, 128))
fmt.Println(AddPrefixes(net.ParseIP("2001:db8::"), 0xff, 32))
fmt.Println(AddPrefixes(net.ParseIP("2001:db8::"), 0x1, 16))
fmt.Println(AddPrefixes(net.ParseIP("2001:db8::"), 0xff, 65))
// Error cases
fmt.Println(AddPrefixes(net.ParseIP("2001:db8::"), 0xff, 8))
fmt.Println(AddPrefixes(net.IP{10, 0, 0, 1}, 64, 32))
// Output:
// 2001:db8:0:ff:: <nil>
// 2001:db8::1 <nil>
// 2001:eb7:: <nil>
// 2002:db8:: <nil>
// 2001:db8:0:7f:8000:: <nil>
// <nil> Operation overflows
// <nil> AddPrefixes needs 128-bit IPs
}
// Offset is used as a hash function, so it needs to be reasonably fast
func BenchmarkOffset(b *testing.B) {
// Need predictable randomness for benchmark reproducibility
rng := rand.New(rand.NewSource(0))
addresses := make([]byte, b.N*net.IPv6len*2)
_, err := rng.Read(addresses)
if err != nil {
b.Fatalf("Could not generate random addresses: %v", err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
// The arrays will be in cache, so this should amortize to measure mostly just the offset
// computation itself
_, _ = Offset(
addresses[i*2*net.IPv6len:(i*2+1)*net.IPv6len],
addresses[(i*2+1)*net.IPv6len:(i+1)*2*net.IPv6len],
(i*4)%128,
)
}
}