Kafka Golang Hash Strategy

ยท Around 1 minute

Nowadays, the default partition is apparently UniformStickyPartitioner although at the time of writing IBM/sarama defaults to HashPartitioner and segmentio/kafka-go has round-robin as its default.

When it comes to the hash strategy, I recently found myself wondering how to determine the assigned partition for a program using kafka-go's hash partition.

This lead to me slicing up the default hasher implementation into a small Go playground program.

NOTE: While I've manually run the testcases through this small program and they all passed, I haven't exercised it in any depth. It's just a reference for myself in the future. You should probably use the actual library directly. 

package main
 
import (
	"fmt"
	"hash"
	"hash/fnv"
)
 
func main() {
	partitions := generatePartitions(3)
	key := "blah"
 
	hasher := fnv.New32a().(hash.Hash32)
	hasher.Reset()
 
	if _, err := hasher.Write([]byte(key)); err != nil {
		panic(err)
	}
 
	partition := int32(hasher.Sum32()) % int32(len(partitions))
	if partition < 0 {
		partition = -partition
	}
	fmt.Println(partition)
}
 
func generatePartitions(partitionCount int) []int {
	partitions := []int{}
	for i := 0; i <= partitionCount-1; i++ {
		partitions = append(partitions, i)
	}
	return partitions
}
package main
 
import (
	"fmt"
	"hash"
	"hash/fnv"
)
 
func main() {
	partitions := generatePartitions(3)
	key := "blah"
 
	hasher := fnv.New32a().(hash.Hash32)
	hasher.Reset()
 
	if _, err := hasher.Write([]byte(key)); err != nil {
		panic(err)
	}
 
	partition := int32(hasher.Sum32()) % int32(len(partitions))
	if partition < 0 {
		partition = -partition
	}
	fmt.Println(partition)
}
 
func generatePartitions(partitionCount int) []int {
	partitions := []int{}
	for i := 0; i <= partitionCount-1; i++ {
		partitions = append(partitions, i)
	}
	return partitions
}