re-parameterized, and parallelized

This commit is contained in:
Cassandra Heart 2025-01-21 16:15:35 -06:00
parent c54c42c0ed
commit 059133fde6
No known key found for this signature in database
GPG Key ID: 6352152859385958
12 changed files with 119 additions and 66 deletions

View File

@ -8,6 +8,7 @@ import (
"go.uber.org/zap"
"golang.org/x/crypto/sha3"
"source.quilibrium.com/quilibrium/monorepo/node/config"
"source.quilibrium.com/quilibrium/monorepo/node/consensus"
"source.quilibrium.com/quilibrium/monorepo/node/consensus/master"
"source.quilibrium.com/quilibrium/monorepo/node/crypto"
@ -108,7 +109,7 @@ func (n *Node) VerifyProofIntegrity() {
panic(e)
}
dataProver := crypto.NewKZGInclusionProver(n.logger)
dataProver := crypto.NewKZGInclusionProver(n.logger, &config.EngineConfig{PendingCommitWorkers: 1})
wesoProver := crypto.NewWesolowskiFrameProver(n.logger)
for j := int(i); j >= 0; j-- {

View File

@ -46,8 +46,8 @@ func NewDebugNode(configConfig *config.Config, selfTestReport *protobufs.SelfTes
p2PConfig := configConfig.P2P
blossomSub := p2p.NewBlossomSub(p2PConfig, zapLogger)
frameProver := crypto.NewCachedWesolowskiFrameProver(zapLogger)
kzgInclusionProver := crypto.NewKZGInclusionProver(zapLogger)
engineConfig := configConfig.Engine
kzgInclusionProver := crypto.NewKZGInclusionProver(zapLogger, engineConfig)
masterTimeReel := time.NewMasterTimeReel(zapLogger, pebbleClockStore, engineConfig, frameProver)
inMemoryPeerInfoManager := p2p.NewInMemoryPeerInfoManager(zapLogger)
pebbleKeyStore := store.NewPebbleKeyStore(pebbleDB, zapLogger)
@ -72,8 +72,8 @@ func NewNode(configConfig *config.Config, selfTestReport *protobufs.SelfTestRepo
p2PConfig := configConfig.P2P
blossomSub := p2p.NewBlossomSub(p2PConfig, zapLogger)
frameProver := crypto.NewCachedWesolowskiFrameProver(zapLogger)
kzgInclusionProver := crypto.NewKZGInclusionProver(zapLogger)
engineConfig := configConfig.Engine
kzgInclusionProver := crypto.NewKZGInclusionProver(zapLogger, engineConfig)
masterTimeReel := time.NewMasterTimeReel(zapLogger, pebbleClockStore, engineConfig, frameProver)
inMemoryPeerInfoManager := p2p.NewInMemoryPeerInfoManager(zapLogger)
pebbleKeyStore := store.NewPebbleKeyStore(pebbleDB, zapLogger)

View File

@ -351,7 +351,7 @@ func (e *DataClockConsensusEngine) handleMint(
output := make([]byte, outputLen)
copy(output[:], data[16+inputLen+8:])
dataProver := crypto.NewKZGInclusionProver(e.logger)
dataProver := crypto.NewKZGInclusionProver(e.logger, e.config.Engine)
wesoProver := crypto.NewWesolowskiFrameProver(e.logger)
index = binary.BigEndian.Uint32(output[:4])
indexProof = output[4:520]

View File

@ -23,6 +23,7 @@ import (
"google.golang.org/protobuf/types/known/anypb"
"google.golang.org/protobuf/types/known/wrapperspb"
"source.quilibrium.com/quilibrium/monorepo/go-libp2p-blossomsub/pb"
"source.quilibrium.com/quilibrium/monorepo/node/config"
"source.quilibrium.com/quilibrium/monorepo/node/consensus"
qtime "source.quilibrium.com/quilibrium/monorepo/node/consensus/time"
qcrypto "source.quilibrium.com/quilibrium/monorepo/node/crypto"
@ -126,10 +127,13 @@ func TestHandlePreMidnightMint(t *testing.T) {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
},
currentReceivingSyncPeers: 0,
lastFrameReceivedAt: time.Time{},
frameProverTries: []*tries.RollingFrecencyCritbitTrie{},
inclusionProver: qcrypto.NewKZGInclusionProver(log),
currentReceivingSyncPeers: 0,
lastFrameReceivedAt: time.Time{},
frameProverTries: []*tries.RollingFrecencyCritbitTrie{},
inclusionProver: qcrypto.NewKZGInclusionProver(
log,
&config.EngineConfig{PendingCommitWorkers: 1},
),
syncingStatus: SyncStatusNotSyncing,
peerMap: map[string]*peerInfo{},
uncooperativePeersMap: map[string]*peerInfo{},

View File

@ -15,6 +15,10 @@ type InclusionAggregateProof struct {
}
type InclusionProver interface {
Commit(
data []byte,
polySize uint64,
) ([]byte, error)
VerifyFrame(frame *protobufs.ClockFrame) error
CommitRaw(
data []byte,

View File

@ -5,16 +5,19 @@ import (
"go.uber.org/zap"
"golang.org/x/crypto/sha3"
rbls48581 "source.quilibrium.com/quilibrium/monorepo/bls48581"
"source.quilibrium.com/quilibrium/monorepo/node/config"
"source.quilibrium.com/quilibrium/monorepo/node/protobufs"
)
type KZGInclusionProver struct {
logger *zap.Logger
logger *zap.Logger
workers chan struct{}
}
func NewKZGInclusionProver(logger *zap.Logger) *KZGInclusionProver {
func NewKZGInclusionProver(logger *zap.Logger, config *config.EngineConfig) *KZGInclusionProver {
return &KZGInclusionProver{
logger: logger,
logger: logger,
workers: make(chan struct{}, config.PendingCommitWorkers),
}
}
@ -99,6 +102,18 @@ func (k *KZGInclusionProver) VerifyFrame(
return nil
}
func (k *KZGInclusionProver) Commit(
data []byte,
polySize uint64,
) ([]byte, error) {
k.workers <- struct{}{}
defer func() {
<-k.workers
}()
return k.CommitRaw(data, polySize)
}
func (k *KZGInclusionProver) CommitRaw(
data []byte,
polySize uint64,

View File

@ -6,6 +6,7 @@ import (
"github.com/stretchr/testify/assert"
"go.uber.org/zap"
"source.quilibrium.com/quilibrium/monorepo/node/config"
"source.quilibrium.com/quilibrium/monorepo/node/crypto"
"source.quilibrium.com/quilibrium/monorepo/node/crypto/kzg"
)
@ -16,7 +17,7 @@ func TestKZGInclusionProverRawFuncs(t *testing.T) {
data, _ := hex.DecodeString("408f9f0a63a1c463579a1fdaf82b37e0f397476e87c524915870ce7f5ede9c248493ea4ffefae154b8a55f10add4d75846b273a7f57433b438ae72880a29ab7cab6c3187a14651bac085329778526ebb31d14c9beb7b0983ff5e71a47c96ed9e7149e9e896cd4d604191583a282bdb5a92ea71334f296fd06498323b0c5d0e60c04180a7141813f6f9a6c766c450898ffc437ebed07a2fbd9201207171a0a8f5006a83d9e2430687952dd42237b7d77de61c0655b91bb1943ed4b9337449ded69ef8f2f83fba58827be7b7082db048b799f1bb590f61c558976910e77357562eb4d66fc97636c26ea562fe18b4cc397e679acad23cfd003ae93efe2903534ce1fe475eba3c82fef71554b4d63b593f2da3fea3b1b3f91379c6ff1989c91eaab70e336d96f3c46de987ef7165d111f692fe8205f7df0eb854fc550aa0d10942049dec4c60d99a51b0a7cde49a6d5e9364d0162cb86af1a51efeffacf7935f796f18cb868756e693aa967339efb8e45071da835ff8b6897fe56dc14edb49352edc88d3a6866873ecfa2bf968907e86c0dd139ab9a23bae341ec6aa5f1fbac2390a9d7f5ef9346d5c433268bf85e34e98295233f5e0d2ceb35c47b33b93e8ae9445c3b9f6ec32d8e3a1a1bc95b013dd36a84d803e468e873420c71b6473e44300f4d2702ccb452146c675d5ac1511a0b0a61a857b58ed3365ecdc1cafafbdfe5f0f2420389ae5f54d2fb9d12de314b416fdb12786fb66d0517229347ecc347eb8207a88abeffbdb9acfc582047a9343efae6c21cf67566e2d949920bdff1f4cea376332dd503c9dcd72a776744724c29a25038ef582f1103b406321e14d0f232c709b3d5a3568c75a1bc244b65e18d9ca7c53e2e13bb5638c325f6d43601de131aa2e3b7ffcc23accf6c69e9c6360cf8f4d48de3f11354855ec281f8a9c85caec0b8284c99c66a43ed0c37d6ce0f5c349e4551da6a1d9edcfa02f6be27ed037c5ec79c0519ba60725f89b3fe7826ca1a7b157ef9360bc2007bc2b9dd2ba8fdc225047a9f66b832e2da1dc6019f480e3aadb46ba93cccbd1e7b221a5d36e0fc96cbf497bfb40ff0276f14b7d45c4738a1b755e2754c5c352ac4af96c1a9be1d92942200b325cc3c53e9b3099c99a466bdc6c001179f6c63f828936b1c33f651a150c080b2eac8ed7cb9cfe599daee477f9ba88a6d1cbdeb08995c3c7bcce18ee2946c2beb138b8c797f61c6c33800ffeda74b77dab186cc4c7e91e9aca954d4863de6b04a82ef563a6eefbedec8fdc9284fb33e15197d2512e4928019fc29aa9c0a199797ef02c8daeb8706dd21a0e6b25b0e73795bac18dfaac2abc1defddf530f6a14046c2a918fa581b7ab0240bbd4f2e570a527581cb0a39bb544ceeabeedf891bc2417ac1e1fa558c09a9ceffef108a5778ff99a8575b4fb69cbbfb2c474d58")
l, _ := zap.NewProduction()
inclusionProver := crypto.NewKZGInclusionProver(l)
inclusionProver := crypto.NewKZGInclusionProver(l, &config.EngineConfig{PendingCommitWorkers: 1})
c, err := inclusionProver.CommitRaw(data, 1024)
assert.NoError(t, err)

View File

@ -7,6 +7,7 @@ import (
"encoding/gob"
"errors"
"fmt"
"sync"
rbls48581 "source.quilibrium.com/quilibrium/monorepo/bls48581"
)
@ -17,13 +18,13 @@ func init() {
}
const (
BranchNodes = 1024
BranchBits = 10 // log2(1024)
BranchNodes = 256
BranchBits = 10 // log2(256)
BranchMask = BranchNodes - 1
)
type VectorCommitmentNode interface {
Commit() []byte
Commit(prover InclusionProver) []byte
}
type VectorCommitmentLeafNode struct {
@ -38,7 +39,7 @@ type VectorCommitmentBranchNode struct {
Commitment []byte
}
func (n *VectorCommitmentLeafNode) Commit() []byte {
func (n *VectorCommitmentLeafNode) Commit(prover InclusionProver) []byte {
if n.Commitment == nil {
h := sha512.New()
h.Write([]byte{0})
@ -49,42 +50,55 @@ func (n *VectorCommitmentLeafNode) Commit() []byte {
return n.Commitment
}
func (n *VectorCommitmentBranchNode) Commit() []byte {
func (n *VectorCommitmentBranchNode) Commit(prover InclusionProver) []byte {
if n.Commitment == nil {
data := []byte{}
for _, child := range n.Children {
children := make([][]byte, len(n.Children))
wg := sync.WaitGroup{}
for i, child := range n.Children {
if child != nil {
out := child.Commit()
switch c := child.(type) {
case *VectorCommitmentBranchNode:
h := sha512.New()
h.Write([]byte{1})
for _, p := range c.Prefix {
h.Write(binary.BigEndian.AppendUint32([]byte{}, uint32(p)))
wg.Add(1)
child := child
go func() {
defer wg.Done()
out := child.Commit(prover)
switch c := child.(type) {
case *VectorCommitmentBranchNode:
h := sha512.New()
h.Write([]byte{1})
for _, p := range c.Prefix {
h.Write(binary.BigEndian.AppendUint32([]byte{}, uint32(p)))
}
h.Write(out)
out = h.Sum(nil)
case *VectorCommitmentLeafNode:
// do nothing
}
h.Write(out)
out = h.Sum(nil)
case *VectorCommitmentLeafNode:
// do nothing
}
data = append(data, out...)
children[i] = out
}()
} else {
data = append(data, make([]byte, 64)...)
children[i] = make([]byte, 64)
}
}
n.Commitment = rbls48581.CommitRaw(data, 1024)
wg.Wait()
data := []byte{}
for _, c := range children {
data = append(data, c...)
}
n.Commitment, _ = prover.Commit(data, 256)
}
return n.Commitment
}
func (n *VectorCommitmentBranchNode) Verify(index int, proof []byte) bool {
func (n *VectorCommitmentBranchNode) Verify(prover InclusionProver, index int, proof []byte) bool {
data := []byte{}
if n.Commitment == nil {
for _, child := range n.Children {
if child != nil {
out := child.Commit()
out := child.Commit(prover)
switch c := child.(type) {
case *VectorCommitmentBranchNode:
h := sha512.New()
@ -103,12 +117,12 @@ func (n *VectorCommitmentBranchNode) Verify(index int, proof []byte) bool {
}
}
n.Commitment = rbls48581.CommitRaw(data, 1024)
n.Commitment = rbls48581.CommitRaw(data, 256)
data = data[64*index : 64*(index+1)]
} else {
child := n.Children[index]
if child != nil {
out := child.Commit()
out := child.Commit(prover)
switch c := child.(type) {
case *VectorCommitmentBranchNode:
h := sha512.New()
@ -127,14 +141,14 @@ func (n *VectorCommitmentBranchNode) Verify(index int, proof []byte) bool {
}
}
return rbls48581.VerifyRaw(data, n.Commitment, uint64(index), proof, 1024)
return rbls48581.VerifyRaw(data, n.Commitment, uint64(index), proof, 256)
}
func (n *VectorCommitmentBranchNode) Prove(index int) []byte {
func (n *VectorCommitmentBranchNode) Prove(prover InclusionProver, index int) []byte {
data := []byte{}
for _, child := range n.Children {
if child != nil {
out := child.Commit()
out := child.Commit(prover)
switch c := child.(type) {
case *VectorCommitmentBranchNode:
h := sha512.New()
@ -153,7 +167,7 @@ func (n *VectorCommitmentBranchNode) Prove(index int) []byte {
}
}
return rbls48581.ProveRaw(data, uint64(index), 1024)
return rbls48581.ProveRaw(data, uint64(index), 256)
}
type VectorCommitmentTree struct {
@ -283,7 +297,7 @@ func (t *VectorCommitmentTree) Insert(key, value []byte) error {
return nil
}
func (t *VectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool {
func (t *VectorCommitmentTree) Verify(prover InclusionProver, key []byte, proofs [][]byte) bool {
if len(key) == 0 {
return false
}
@ -316,7 +330,7 @@ func (t *VectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool {
// Get final nibble after prefix
finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits)
if !n.Verify(finalNibble, proofs[0]) {
if !n.Verify(prover, finalNibble, proofs[0]) {
return false
}
@ -329,7 +343,7 @@ func (t *VectorCommitmentTree) Verify(key []byte, proofs [][]byte) bool {
return verify(t.Root, proofs, 0)
}
func (t *VectorCommitmentTree) Prove(key []byte) [][]byte {
func (t *VectorCommitmentTree) Prove(prover InclusionProver, key []byte) [][]byte {
if len(key) == 0 {
return nil
}
@ -358,7 +372,7 @@ func (t *VectorCommitmentTree) Prove(key []byte) [][]byte {
// Get final nibble after prefix
finalNibble := getNextNibble(key, depth+len(n.Prefix)*BranchBits)
proofs := [][]byte{n.Prove(finalNibble)}
proofs := [][]byte{n.Prove(prover, finalNibble)}
return append(proofs, prove(n.Children[finalNibble], depth+len(n.Prefix)*BranchBits+BranchBits)...)
}
@ -482,11 +496,11 @@ func (t *VectorCommitmentTree) Delete(key []byte) error {
}
// Commit returns the root of the tree
func (t *VectorCommitmentTree) Commit() []byte {
func (t *VectorCommitmentTree) Commit(prover InclusionProver) []byte {
if t.Root == nil {
return make([]byte, 64)
}
return t.Root.Commit()
return t.Root.Commit(prover)
}
func debugNode(node VectorCommitmentNode, depth int, prefix string) {

View File

@ -6,7 +6,9 @@ import (
"fmt"
"testing"
"go.uber.org/zap"
"source.quilibrium.com/quilibrium/monorepo/bls48581/generated/bls48581"
"source.quilibrium.com/quilibrium/monorepo/node/config"
)
func BenchmarkVectorCommitmentTreeInsert(b *testing.B) {
@ -27,6 +29,8 @@ func BenchmarkVectorCommitmentTreeInsert(b *testing.B) {
func BenchmarkVectorCommitmentTreeCommit(b *testing.B) {
tree := &VectorCommitmentTree{}
addresses := [][]byte{}
log, _ := zap.NewProduction()
prover := NewKZGInclusionProver(log, &config.EngineConfig{PendingCommitWorkers: 1})
for i := range b.N {
d := make([]byte, 32)
@ -36,13 +40,15 @@ func BenchmarkVectorCommitmentTreeCommit(b *testing.B) {
if err != nil {
b.Errorf("Failed to insert item %d: %v", i, err)
}
tree.Commit()
tree.Commit(prover)
}
}
func BenchmarkVectorCommitmentTreeProve(b *testing.B) {
tree := &VectorCommitmentTree{}
addresses := [][]byte{}
log, _ := zap.NewProduction()
prover := NewKZGInclusionProver(log, &config.EngineConfig{PendingCommitWorkers: 1})
for i := range b.N {
d := make([]byte, 32)
@ -52,13 +58,15 @@ func BenchmarkVectorCommitmentTreeProve(b *testing.B) {
if err != nil {
b.Errorf("Failed to insert item %d: %v", i, err)
}
tree.Prove(d)
tree.Prove(prover, d)
}
}
func BenchmarkVectorCommitmentTreeVerify(b *testing.B) {
tree := &VectorCommitmentTree{}
addresses := [][]byte{}
log, _ := zap.NewProduction()
prover := NewKZGInclusionProver(log, &config.EngineConfig{PendingCommitWorkers: 1})
for i := range b.N {
d := make([]byte, 32)
@ -68,8 +76,8 @@ func BenchmarkVectorCommitmentTreeVerify(b *testing.B) {
if err != nil {
b.Errorf("Failed to insert item %d: %v", i, err)
}
p := tree.Prove(d)
if !tree.Verify(d, p) {
p := tree.Prove(prover, d)
if !tree.Verify(prover, d, p) {
b.Errorf("bad proof")
}
}
@ -78,6 +86,8 @@ func BenchmarkVectorCommitmentTreeVerify(b *testing.B) {
func TestVectorCommitmentTrees(t *testing.T) {
bls48581.Init()
tree := &VectorCommitmentTree{}
log, _ := zap.NewProduction()
prover := NewKZGInclusionProver(log, &config.EngineConfig{PendingCommitWorkers: 1})
// Test single insert
err := tree.Insert([]byte("key1"), []byte("value1"))
@ -221,7 +231,7 @@ func TestVectorCommitmentTrees(t *testing.T) {
// Root should change after insert
tree.Insert([]byte("key1"), []byte("value1"))
firstRoot := tree.Root.Commit()
firstRoot := tree.Root.Commit(prover)
if bytes.Equal(firstRoot, bytes.Repeat([]byte{0x00}, 64)) {
t.Error("Root hash should change after insert")
@ -229,7 +239,7 @@ func TestVectorCommitmentTrees(t *testing.T) {
// Root should change after update
tree.Insert([]byte("key1"), []byte("value2"))
secondRoot := tree.Root.Commit()
secondRoot := tree.Root.Commit(prover)
if bytes.Equal(secondRoot, firstRoot) {
t.Error("Root hash should change after update")
@ -294,15 +304,15 @@ func TestVectorCommitmentTrees(t *testing.T) {
}
}
tcommit := tree.Root.Commit()
cmptcommit := cmptree.Root.Commit()
tcommit := tree.Root.Commit(prover)
cmptcommit := cmptree.Root.Commit(prover)
if !bytes.Equal(tcommit, cmptcommit) {
t.Errorf("tree mismatch, %x, %x", tcommit, cmptcommit)
}
proofs := tree.Prove(addresses[500])
if !tree.Verify(addresses[500], proofs) {
proofs := tree.Prove(prover, addresses[500])
if !tree.Verify(prover, addresses[500], proofs) {
t.Errorf("proof failed")
}

View File

@ -43,6 +43,7 @@ require (
require (
filippo.io/edwards25519 v1.0.0-rc.1 // indirect
github.com/deiu/gon3 v0.0.0-20230411081920-f0f8f879f597 // indirect
github.com/google/subcommands v1.0.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.7.2 // indirect
github.com/linkeddata/gojsonld v0.0.0-20170418210642-4f5db6791326 // indirect

View File

@ -183,6 +183,7 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo=
github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/subcommands v1.0.1 h1:/eqq+otEXm5vhfBrbREPCSVQbvofip6kIz+mX5TUH7k=
github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

View File

@ -198,10 +198,7 @@ func main() {
fmt.Println("Performing proof tree tests...")
fmt.Println("\nTree Insertion")
sets := []int{1000, 10000, 100000, 1000000, 10000000}
if memory.TotalMemory() > 64*1024*1024*1024 {
sets = append(sets, 100000000)
}
sets := []int{1000, 10000, 100000, 1000000}
for _, set := range sets {
var total atomic.Int64
vecTree := &qcrypto.VectorCommitmentTree{}
@ -243,6 +240,12 @@ func main() {
fmt.Println("Size: ", set, "Op Speed: ", time.Duration(total.Load())/time.Duration(set))
}
log, _ := zap.NewProduction()
incProver := qcrypto.NewKZGInclusionProver(
log,
&config.EngineConfig{PendingCommitWorkers: int64(runtime.NumCPU())},
)
fmt.Println("\nTree Commit")
for _, set := range sets {
var total atomic.Int64
@ -259,7 +262,7 @@ func main() {
}
start := time.Now()
vecTree.Commit()
vecTree.Commit(incProver)
total.Add(int64(time.Since(start)))
fmt.Println("Size: ", set, "Op Speed: ", time.Duration(total.Load()))
}
@ -278,17 +281,16 @@ func main() {
panic(err)
}
}
vecTree.Commit()
vecTree.Commit(incProver)
for k := 0; k < set; k++ {
start := time.Now()
vecTree.Prove(data[k])
vecTree.Prove(incProver, data[k])
total.Add(int64(time.Since(start)))
}
fmt.Println("Size: ", set, "Op Speed: ", time.Duration(total.Load())/time.Duration(set))
}
fmt.Println("\nVDF Prove")
log, _ := zap.NewProduction()
prover := qcrypto.NewWesolowskiFrameProver(log)
sets = []int{100000, 200000, 500000, 1000000, 2000000, 5000000}
for _, set := range sets {