mirror of
https://github.com/ipfs/kubo.git
synced 2026-02-21 10:27:46 +08:00
feat(config): validate Import config at daemon startup (#10957)
Some checks failed
Docker Check / lint (push) Waiting to run
Docker Check / build (push) Waiting to run
Gateway Conformance / gateway-conformance (push) Waiting to run
Gateway Conformance / gateway-conformance-libp2p-experiment (push) Waiting to run
Go Build / go-build (push) Waiting to run
Go Check / go-check (push) Waiting to run
Go Lint / go-lint (push) Waiting to run
Go Test / go-test (push) Waiting to run
Interop / interop-prep (push) Waiting to run
Interop / helia-interop (push) Blocked by required conditions
Interop / ipfs-webui (push) Blocked by required conditions
Sharness / sharness-test (push) Waiting to run
Spell Check / spellcheck (push) Waiting to run
CodeQL / codeql (push) Has been cancelled
Some checks failed
Docker Check / lint (push) Waiting to run
Docker Check / build (push) Waiting to run
Gateway Conformance / gateway-conformance (push) Waiting to run
Gateway Conformance / gateway-conformance-libp2p-experiment (push) Waiting to run
Go Build / go-build (push) Waiting to run
Go Check / go-check (push) Waiting to run
Go Lint / go-lint (push) Waiting to run
Go Test / go-test (push) Waiting to run
Interop / interop-prep (push) Waiting to run
Interop / helia-interop (push) Blocked by required conditions
Interop / ipfs-webui (push) Blocked by required conditions
Sharness / sharness-test (push) Waiting to run
Spell Check / spellcheck (push) Waiting to run
CodeQL / codeql (push) Has been cancelled
validates Import configuration fields to prevent invalid values: - CidVersion: must be 0 or 1 - UnixFSFileMaxLinks: must be positive - UnixFSDirectoryMaxLinks: must be non-negative - UnixFSHAMTDirectoryMaxFanout: power of 2, multiple of 8, ≤ 1024 - BatchMaxNodes/BatchMaxSize: must be positive - UnixFSChunker: validates format patterns - HashFunction: must be allowed by verifcid
This commit is contained in:
parent
049256c22f
commit
3e1e7d17fb
135
config/import.go
135
config/import.go
@ -1,8 +1,14 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
|
||||
"github.com/ipfs/boxo/ipld/unixfs/io"
|
||||
"github.com/ipfs/boxo/verifcid"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -43,3 +49,132 @@ type Import struct {
|
||||
BatchMaxNodes OptionalInteger
|
||||
BatchMaxSize OptionalInteger
|
||||
}
|
||||
|
||||
// ValidateImportConfig validates the Import configuration according to UnixFS spec requirements.
|
||||
// See: https://specs.ipfs.tech/unixfs/#hamt-structure-and-parameters
|
||||
func ValidateImportConfig(cfg *Import) error {
|
||||
// Validate CidVersion
|
||||
if !cfg.CidVersion.IsDefault() {
|
||||
cidVer := cfg.CidVersion.WithDefault(DefaultCidVersion)
|
||||
if cidVer != 0 && cidVer != 1 {
|
||||
return fmt.Errorf("Import.CidVersion must be 0 or 1, got %d", cidVer)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate UnixFSFileMaxLinks
|
||||
if !cfg.UnixFSFileMaxLinks.IsDefault() {
|
||||
maxLinks := cfg.UnixFSFileMaxLinks.WithDefault(DefaultUnixFSFileMaxLinks)
|
||||
if maxLinks <= 0 {
|
||||
return fmt.Errorf("Import.UnixFSFileMaxLinks must be positive, got %d", maxLinks)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate UnixFSDirectoryMaxLinks
|
||||
if !cfg.UnixFSDirectoryMaxLinks.IsDefault() {
|
||||
maxLinks := cfg.UnixFSDirectoryMaxLinks.WithDefault(DefaultUnixFSDirectoryMaxLinks)
|
||||
if maxLinks < 0 {
|
||||
return fmt.Errorf("Import.UnixFSDirectoryMaxLinks must be non-negative, got %d", maxLinks)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate UnixFSHAMTDirectoryMaxFanout if set
|
||||
if !cfg.UnixFSHAMTDirectoryMaxFanout.IsDefault() {
|
||||
fanout := cfg.UnixFSHAMTDirectoryMaxFanout.WithDefault(DefaultUnixFSHAMTDirectoryMaxFanout)
|
||||
|
||||
// Check all requirements: fanout < 8 covers both non-positive and non-multiple of 8
|
||||
// Combined with power of 2 check and max limit, this ensures valid values: 8, 16, 32, 64, 128, 256, 512, 1024
|
||||
if fanout < 8 || !isPowerOfTwo(fanout) || fanout > 1024 {
|
||||
return fmt.Errorf("Import.UnixFSHAMTDirectoryMaxFanout must be a positive power of 2, multiple of 8, and not exceed 1024 (got %d)", fanout)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate BatchMaxNodes
|
||||
if !cfg.BatchMaxNodes.IsDefault() {
|
||||
maxNodes := cfg.BatchMaxNodes.WithDefault(DefaultBatchMaxNodes)
|
||||
if maxNodes <= 0 {
|
||||
return fmt.Errorf("Import.BatchMaxNodes must be positive, got %d", maxNodes)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate BatchMaxSize
|
||||
if !cfg.BatchMaxSize.IsDefault() {
|
||||
maxSize := cfg.BatchMaxSize.WithDefault(DefaultBatchMaxSize)
|
||||
if maxSize <= 0 {
|
||||
return fmt.Errorf("Import.BatchMaxSize must be positive, got %d", maxSize)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate UnixFSChunker format
|
||||
if !cfg.UnixFSChunker.IsDefault() {
|
||||
chunker := cfg.UnixFSChunker.WithDefault(DefaultUnixFSChunker)
|
||||
if !isValidChunker(chunker) {
|
||||
return fmt.Errorf("Import.UnixFSChunker invalid format: %q (expected \"size-<bytes>\", \"rabin-<min>-<avg>-<max>\", or \"buzhash\")", chunker)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate HashFunction
|
||||
if !cfg.HashFunction.IsDefault() {
|
||||
hashFunc := cfg.HashFunction.WithDefault(DefaultHashFunction)
|
||||
hashCode, ok := mh.Names[strings.ToLower(hashFunc)]
|
||||
if !ok {
|
||||
return fmt.Errorf("Import.HashFunction unrecognized: %q", hashFunc)
|
||||
}
|
||||
// Check if the hash is allowed by verifcid
|
||||
if !verifcid.DefaultAllowlist.IsAllowed(hashCode) {
|
||||
return fmt.Errorf("Import.HashFunction %q is not allowed for use in IPFS", hashFunc)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isPowerOfTwo checks if a number is a power of 2
|
||||
func isPowerOfTwo(n int64) bool {
|
||||
return n > 0 && (n&(n-1)) == 0
|
||||
}
|
||||
|
||||
// isValidChunker validates chunker format
|
||||
func isValidChunker(chunker string) bool {
|
||||
if chunker == "buzhash" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check for size-<bytes> format
|
||||
if strings.HasPrefix(chunker, "size-") {
|
||||
sizeStr := strings.TrimPrefix(chunker, "size-")
|
||||
if sizeStr == "" {
|
||||
return false
|
||||
}
|
||||
// Check if it's a valid positive integer (no negative sign allowed)
|
||||
if sizeStr[0] == '-' {
|
||||
return false
|
||||
}
|
||||
size, err := strconv.Atoi(sizeStr)
|
||||
// Size must be positive (not zero)
|
||||
return err == nil && size > 0
|
||||
}
|
||||
|
||||
// Check for rabin-<min>-<avg>-<max> format
|
||||
if strings.HasPrefix(chunker, "rabin-") {
|
||||
parts := strings.Split(chunker, "-")
|
||||
if len(parts) != 4 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Parse and validate min, avg, max values
|
||||
values := make([]int, 3)
|
||||
for i := 0; i < 3; i++ {
|
||||
val, err := strconv.Atoi(parts[i+1])
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
values[i] = val
|
||||
}
|
||||
|
||||
// Validate ordering: min <= avg <= max
|
||||
min, avg, max := values[0], values[1], values[2]
|
||||
return min <= avg && avg <= max
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
408
config/import_test.go
Normal file
408
config/import_test.go
Normal file
@ -0,0 +1,408 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
func TestValidateImportConfig_HAMTFanout(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fanout int64
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{
|
||||
// Valid values - powers of 2, multiples of 8, and <= 1024
|
||||
{name: "valid 8", fanout: 8, wantErr: false},
|
||||
{name: "valid 16", fanout: 16, wantErr: false},
|
||||
{name: "valid 32", fanout: 32, wantErr: false},
|
||||
{name: "valid 64", fanout: 64, wantErr: false},
|
||||
{name: "valid 128", fanout: 128, wantErr: false},
|
||||
{name: "valid 256", fanout: 256, wantErr: false},
|
||||
{name: "valid 512", fanout: 512, wantErr: false},
|
||||
{name: "valid 1024", fanout: 1024, wantErr: false},
|
||||
|
||||
// Invalid values - not powers of 2
|
||||
{name: "invalid 7", fanout: 7, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
{name: "invalid 15", fanout: 15, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
{name: "invalid 100", fanout: 100, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
{name: "invalid 257", fanout: 257, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
{name: "invalid 1000", fanout: 1000, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
|
||||
// Invalid values - powers of 2 but not multiples of 8
|
||||
{name: "invalid 1", fanout: 1, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
{name: "invalid 2", fanout: 2, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
{name: "invalid 4", fanout: 4, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
|
||||
// Invalid values - exceeds 1024
|
||||
{name: "invalid 2048", fanout: 2048, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
{name: "invalid 4096", fanout: 4096, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
|
||||
// Invalid values - negative or zero
|
||||
{name: "invalid 0", fanout: 0, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
{name: "invalid -8", fanout: -8, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
{name: "invalid -256", fanout: -256, wantErr: true, errMsg: "must be a positive power of 2, multiple of 8, and not exceed 1024"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &Import{
|
||||
UnixFSHAMTDirectoryMaxFanout: *NewOptionalInteger(tt.fanout),
|
||||
}
|
||||
|
||||
err := ValidateImportConfig(cfg)
|
||||
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("ValidateImportConfig() expected error for fanout=%d, got nil", tt.fanout)
|
||||
} else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
|
||||
t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("ValidateImportConfig() unexpected error for fanout=%d: %v", tt.fanout, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateImportConfig_CidVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cidVer int64
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{
|
||||
{name: "valid 0", cidVer: 0, wantErr: false},
|
||||
{name: "valid 1", cidVer: 1, wantErr: false},
|
||||
{name: "invalid 2", cidVer: 2, wantErr: true, errMsg: "must be 0 or 1"},
|
||||
{name: "invalid -1", cidVer: -1, wantErr: true, errMsg: "must be 0 or 1"},
|
||||
{name: "invalid 100", cidVer: 100, wantErr: true, errMsg: "must be 0 or 1"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &Import{
|
||||
CidVersion: *NewOptionalInteger(tt.cidVer),
|
||||
}
|
||||
|
||||
err := ValidateImportConfig(cfg)
|
||||
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("ValidateImportConfig() expected error for cidVer=%d, got nil", tt.cidVer)
|
||||
} else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
|
||||
t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("ValidateImportConfig() unexpected error for cidVer=%d: %v", tt.cidVer, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateImportConfig_UnixFSFileMaxLinks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
maxLinks int64
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{
|
||||
{name: "valid 1", maxLinks: 1, wantErr: false},
|
||||
{name: "valid 174", maxLinks: 174, wantErr: false},
|
||||
{name: "valid 1000", maxLinks: 1000, wantErr: false},
|
||||
{name: "invalid 0", maxLinks: 0, wantErr: true, errMsg: "must be positive"},
|
||||
{name: "invalid -1", maxLinks: -1, wantErr: true, errMsg: "must be positive"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &Import{
|
||||
UnixFSFileMaxLinks: *NewOptionalInteger(tt.maxLinks),
|
||||
}
|
||||
|
||||
err := ValidateImportConfig(cfg)
|
||||
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("ValidateImportConfig() expected error for maxLinks=%d, got nil", tt.maxLinks)
|
||||
} else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
|
||||
t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("ValidateImportConfig() unexpected error for maxLinks=%d: %v", tt.maxLinks, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateImportConfig_UnixFSDirectoryMaxLinks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
maxLinks int64
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{
|
||||
{name: "valid 0", maxLinks: 0, wantErr: false}, // 0 means no limit
|
||||
{name: "valid 1", maxLinks: 1, wantErr: false},
|
||||
{name: "valid 1000", maxLinks: 1000, wantErr: false},
|
||||
{name: "invalid -1", maxLinks: -1, wantErr: true, errMsg: "must be non-negative"},
|
||||
{name: "invalid -100", maxLinks: -100, wantErr: true, errMsg: "must be non-negative"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &Import{
|
||||
UnixFSDirectoryMaxLinks: *NewOptionalInteger(tt.maxLinks),
|
||||
}
|
||||
|
||||
err := ValidateImportConfig(cfg)
|
||||
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("ValidateImportConfig() expected error for maxLinks=%d, got nil", tt.maxLinks)
|
||||
} else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
|
||||
t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("ValidateImportConfig() unexpected error for maxLinks=%d: %v", tt.maxLinks, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateImportConfig_BatchMax(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
maxNodes int64
|
||||
maxSize int64
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{
|
||||
{name: "valid nodes 1", maxNodes: 1, maxSize: -999, wantErr: false},
|
||||
{name: "valid nodes 128", maxNodes: 128, maxSize: -999, wantErr: false},
|
||||
{name: "valid size 1", maxNodes: -999, maxSize: 1, wantErr: false},
|
||||
{name: "valid size 20MB", maxNodes: -999, maxSize: 20 << 20, wantErr: false},
|
||||
{name: "invalid nodes 0", maxNodes: 0, maxSize: -999, wantErr: true, errMsg: "BatchMaxNodes must be positive"},
|
||||
{name: "invalid nodes -1", maxNodes: -1, maxSize: -999, wantErr: true, errMsg: "BatchMaxNodes must be positive"},
|
||||
{name: "invalid size 0", maxNodes: -999, maxSize: 0, wantErr: true, errMsg: "BatchMaxSize must be positive"},
|
||||
{name: "invalid size -1", maxNodes: -999, maxSize: -1, wantErr: true, errMsg: "BatchMaxSize must be positive"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &Import{}
|
||||
if tt.maxNodes != -999 {
|
||||
cfg.BatchMaxNodes = *NewOptionalInteger(tt.maxNodes)
|
||||
}
|
||||
if tt.maxSize != -999 {
|
||||
cfg.BatchMaxSize = *NewOptionalInteger(tt.maxSize)
|
||||
}
|
||||
|
||||
err := ValidateImportConfig(cfg)
|
||||
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("ValidateImportConfig() expected error, got nil")
|
||||
} else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
|
||||
t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("ValidateImportConfig() unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateImportConfig_UnixFSChunker(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
chunker string
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{
|
||||
{name: "valid size-262144", chunker: "size-262144", wantErr: false},
|
||||
{name: "valid size-1", chunker: "size-1", wantErr: false},
|
||||
{name: "valid size-1048576", chunker: "size-1048576", wantErr: false},
|
||||
{name: "valid rabin", chunker: "rabin-128-256-512", wantErr: false},
|
||||
{name: "valid rabin min", chunker: "rabin-16-32-64", wantErr: false},
|
||||
{name: "valid buzhash", chunker: "buzhash", wantErr: false},
|
||||
{name: "invalid size-", chunker: "size-", wantErr: true, errMsg: "invalid format"},
|
||||
{name: "invalid size-abc", chunker: "size-abc", wantErr: true, errMsg: "invalid format"},
|
||||
{name: "invalid rabin-", chunker: "rabin-", wantErr: true, errMsg: "invalid format"},
|
||||
{name: "invalid rabin-128", chunker: "rabin-128", wantErr: true, errMsg: "invalid format"},
|
||||
{name: "invalid rabin-128-256", chunker: "rabin-128-256", wantErr: true, errMsg: "invalid format"},
|
||||
{name: "invalid rabin-a-b-c", chunker: "rabin-a-b-c", wantErr: true, errMsg: "invalid format"},
|
||||
{name: "invalid unknown", chunker: "unknown", wantErr: true, errMsg: "invalid format"},
|
||||
{name: "invalid empty", chunker: "", wantErr: true, errMsg: "invalid format"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &Import{
|
||||
UnixFSChunker: *NewOptionalString(tt.chunker),
|
||||
}
|
||||
|
||||
err := ValidateImportConfig(cfg)
|
||||
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("ValidateImportConfig() expected error for chunker=%s, got nil", tt.chunker)
|
||||
} else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
|
||||
t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("ValidateImportConfig() unexpected error for chunker=%s: %v", tt.chunker, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateImportConfig_HashFunction(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
hashFunc string
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{
|
||||
{name: "valid sha2-256", hashFunc: "sha2-256", wantErr: false},
|
||||
{name: "valid sha2-512", hashFunc: "sha2-512", wantErr: false},
|
||||
{name: "valid sha3-256", hashFunc: "sha3-256", wantErr: false},
|
||||
{name: "valid blake2b-256", hashFunc: "blake2b-256", wantErr: false},
|
||||
{name: "valid blake3", hashFunc: "blake3", wantErr: false},
|
||||
{name: "invalid unknown", hashFunc: "unknown-hash", wantErr: true, errMsg: "unrecognized"},
|
||||
{name: "invalid empty", hashFunc: "", wantErr: true, errMsg: "unrecognized"},
|
||||
}
|
||||
|
||||
// Check for hashes that exist but are not allowed
|
||||
// MD5 should exist but not be allowed
|
||||
if code, ok := mh.Names["md5"]; ok {
|
||||
tests = append(tests, struct {
|
||||
name string
|
||||
hashFunc string
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{name: "md5 not allowed", hashFunc: "md5", wantErr: true, errMsg: "not allowed"})
|
||||
_ = code // use the variable
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &Import{
|
||||
HashFunction: *NewOptionalString(tt.hashFunc),
|
||||
}
|
||||
|
||||
err := ValidateImportConfig(cfg)
|
||||
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("ValidateImportConfig() expected error for hashFunc=%s, got nil", tt.hashFunc)
|
||||
} else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
|
||||
t.Errorf("ValidateImportConfig() error = %v, want error containing %q", err, tt.errMsg)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("ValidateImportConfig() unexpected error for hashFunc=%s: %v", tt.hashFunc, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateImportConfig_DefaultValue(t *testing.T) {
|
||||
// Test that default (unset) value doesn't trigger validation
|
||||
cfg := &Import{}
|
||||
|
||||
err := ValidateImportConfig(cfg)
|
||||
if err != nil {
|
||||
t.Errorf("ValidateImportConfig() unexpected error for default config: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsValidChunker(t *testing.T) {
|
||||
tests := []struct {
|
||||
chunker string
|
||||
want bool
|
||||
}{
|
||||
{"buzhash", true},
|
||||
{"size-262144", true},
|
||||
{"size-1", true},
|
||||
{"size-0", false}, // 0 is not valid - must be positive
|
||||
{"size-9999999", true},
|
||||
{"rabin-128-256-512", true},
|
||||
{"rabin-16-32-64", true},
|
||||
{"rabin-1-2-3", true},
|
||||
{"rabin-512-256-128", false}, // Invalid ordering: min > avg > max
|
||||
{"rabin-256-128-512", false}, // Invalid ordering: min > avg
|
||||
{"rabin-128-512-256", false}, // Invalid ordering: avg > max
|
||||
|
||||
{"", false},
|
||||
{"size-", false},
|
||||
{"size-abc", false},
|
||||
{"size--1", false},
|
||||
{"rabin-", false},
|
||||
{"rabin-128", false},
|
||||
{"rabin-128-256", false},
|
||||
{"rabin-128-256-512-1024", false},
|
||||
{"rabin-a-b-c", false},
|
||||
{"unknown", false},
|
||||
{"buzzhash", false}, // typo
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.chunker, func(t *testing.T) {
|
||||
if got := isValidChunker(tt.chunker); got != tt.want {
|
||||
t.Errorf("isValidChunker(%q) = %v, want %v", tt.chunker, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsPowerOfTwo(t *testing.T) {
|
||||
tests := []struct {
|
||||
n int64
|
||||
want bool
|
||||
}{
|
||||
{0, false},
|
||||
{1, true},
|
||||
{2, true},
|
||||
{3, false},
|
||||
{4, true},
|
||||
{5, false},
|
||||
{6, false},
|
||||
{7, false},
|
||||
{8, true},
|
||||
{16, true},
|
||||
{32, true},
|
||||
{64, true},
|
||||
{100, false},
|
||||
{128, true},
|
||||
{256, true},
|
||||
{512, true},
|
||||
{1024, true},
|
||||
{2048, true},
|
||||
{-1, false},
|
||||
{-8, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
if got := isPowerOfTwo(tt.n); got != tt.want {
|
||||
t.Errorf("isPowerOfTwo(%d) = %v, want %v", tt.n, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -432,6 +432,11 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option {
|
||||
cfg.Import.UnixFSHAMTDirectorySizeThreshold = *cfg.Internal.UnixFSShardingSizeThreshold
|
||||
}
|
||||
|
||||
// Validate Import configuration
|
||||
if err := config.ValidateImportConfig(&cfg.Import); err != nil {
|
||||
return fx.Error(err)
|
||||
}
|
||||
|
||||
// Auto-sharding settings
|
||||
shardingThresholdString := cfg.Import.UnixFSHAMTDirectorySizeThreshold.WithDefault(config.DefaultUnixFSHAMTDirectorySizeThreshold)
|
||||
shardSingThresholdInt, err := humanize.ParseBytes(shardingThresholdString)
|
||||
|
||||
@ -3133,6 +3133,8 @@ Note that using flags will override the options defined here.
|
||||
|
||||
The default CID version. Commands affected: `ipfs add`.
|
||||
|
||||
Must be either 0 or 1. CIDv0 uses SHA2-256 only, while CIDv1 supports multiple hash functions.
|
||||
|
||||
Default: `0`
|
||||
|
||||
Type: `optionalInteger`
|
||||
@ -3149,6 +3151,11 @@ Type: `flag`
|
||||
|
||||
The default UnixFS chunker. Commands affected: `ipfs add`.
|
||||
|
||||
Valid formats:
|
||||
- `size-<bytes>` - fixed size chunker
|
||||
- `rabin-<min>-<avg>-<max>` - rabin fingerprint chunker
|
||||
- `buzhash` - buzhash chunker
|
||||
|
||||
Default: `size-262144`
|
||||
|
||||
Type: `optionalString`
|
||||
@ -3157,6 +3164,10 @@ Type: `optionalString`
|
||||
|
||||
The default hash function. Commands affected: `ipfs add`, `ipfs block put`, `ipfs dag put`.
|
||||
|
||||
Must be a valid multihash name (e.g., `sha2-256`, `blake3`) and must be allowed for use in IPFS according to security constraints.
|
||||
|
||||
Run `ipfs cid hashes --supported` to see the full list of allowed hash functions.
|
||||
|
||||
Default: `sha2-256`
|
||||
|
||||
Type: `optionalString`
|
||||
@ -3167,6 +3178,8 @@ The maximum number of nodes in a write-batch. The total size of the batch is lim
|
||||
|
||||
Increasing this will batch more items together when importing data with `ipfs dag import`, which can speed things up.
|
||||
|
||||
Must be positive (> 0). Setting to 0 would cause immediate batching after each node, which is inefficient.
|
||||
|
||||
Default: `128`
|
||||
|
||||
Type: `optionalInteger`
|
||||
@ -3177,6 +3190,8 @@ The maximum size of a single write-batch (computed as the sum of the sizes of th
|
||||
|
||||
Increasing this will batch more items together when importing data with `ipfs dag import`, which can speed things up.
|
||||
|
||||
Must be positive (> 0). Setting to 0 would cause immediate batching after any data, which is inefficient.
|
||||
|
||||
Default: `20971520` (20MiB)
|
||||
|
||||
Type: `optionalInteger`
|
||||
@ -3189,6 +3204,8 @@ when building the DAG while importing.
|
||||
This setting controls both the fanout in files that are chunked into several
|
||||
blocks and grouped as a Unixfs (dag-pb) DAG.
|
||||
|
||||
Must be positive (> 0). Zero or negative values would break file DAG construction.
|
||||
|
||||
Default: `174`
|
||||
|
||||
Type: `optionalInteger`
|
||||
@ -3208,6 +3225,8 @@ This setting will cause basic directories to be converted to HAMTs when they
|
||||
exceed the maximum number of children. This happens transparently during the
|
||||
add process. The fanout of HAMT nodes is controlled by `MaxHAMTFanout`.
|
||||
|
||||
Must be non-negative (>= 0). Zero means no limit, negative values are invalid.
|
||||
|
||||
Commands affected: `ipfs add`
|
||||
|
||||
Default: `0` (no limit, because [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold) triggers controls when to switch to HAMT sharding when a directory grows too big)
|
||||
@ -3216,15 +3235,15 @@ Type: `optionalInteger`
|
||||
|
||||
### `Import.UnixFSHAMTDirectoryMaxFanout`
|
||||
|
||||
The maximum number of children that a node part of a Unixfs HAMT directory
|
||||
The maximum number of children that a node part of a UnixFS HAMT directory
|
||||
(aka sharded directory) can have.
|
||||
|
||||
HAMT directories have unlimited children and are used when basic directories
|
||||
become too big or reach `MaxLinks`. A HAMT is a structure made of unixfs
|
||||
become too big or reach `MaxLinks`. A HAMT is a structure made of UnixFS
|
||||
nodes that store the list of elements in the folder. This option controls the
|
||||
maximum number of children that the HAMT nodes can have.
|
||||
|
||||
Needs to be a power of two (shard entry size) and multiple of 8 (bitfield size).
|
||||
According to the [UnixFS specification](https://specs.ipfs.tech/unixfs/#hamt-structure-and-parameters), this value must be a power of 2, a multiple of 8 (for byte-aligned bitfields), and not exceed 1024 (to prevent denial-of-service attacks).
|
||||
|
||||
Commands affected: `ipfs add`, `ipfs daemon` (globally overrides [`boxo/ipld/unixfs/io.DefaultShardWidth`](https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L30C5-L30C22))
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user