mirror of
https://github.com/ipfs/kubo.git
synced 2026-03-10 02:40:11 +08:00
01b1ce0cca
add CLI flags for controlling file collection behavior during ipfs add: - `--dereference-symlinks`: recursively resolve symlinks to their target content (replaces deprecated --dereference-args which only worked on CLI arguments). wired through go-ipfs-cmds to boxo's SerialFileOptions. - `--empty-dirs` / `-E`: include empty directories (default: true) - `--hidden` / `-H`: include hidden files (default: false) these flags are CLI-only and not wired to Import.* config options because go-ipfs-cmds library handles input file filtering before the directory tree is passed to kubo. removed unused Import.UnixFSSymlinkMode config option that was defined but never actually read by the CLI. also: - wire --trickle to Import.UnixFSDAGLayout config default - update go-ipfs-cmds to v0.15.1-0.20260117043932-17687e216294 - add SYMLINK HANDLING section to ipfs add help text - add CLI tests for all three flags ref: https://github.com/ipfs/specs/pull/499
859 lines
30 KiB
Go
859 lines
30 KiB
Go
package cli
|
|
|
|
import (
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/dustin/go-humanize"
|
|
"github.com/ipfs/kubo/config"
|
|
"github.com/ipfs/kubo/test/cli/harness"
|
|
"github.com/ipfs/kubo/test/cli/testutils"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// waitForLogMessage polls a buffer for a log message, waiting up to timeout duration.
|
|
// Returns true if message found, false if timeout reached.
|
|
func waitForLogMessage(buffer *harness.Buffer, message string, timeout time.Duration) bool {
|
|
deadline := time.Now().Add(timeout)
|
|
for time.Now().Before(deadline) {
|
|
if strings.Contains(buffer.String(), message) {
|
|
return true
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
return false
|
|
}
|
|
|
|
func TestAdd(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var (
|
|
shortString = "hello world"
|
|
shortStringCidV0 = "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD" // cidv0 - dag-pb - sha2-256
|
|
shortStringCidV1 = "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e" // cidv1 - raw - sha2-256
|
|
shortStringCidV1NoRawLeaves = "bafybeihykld7uyxzogax6vgyvag42y7464eywpf55gxi5qpoisibh3c5wa" // cidv1 - dag-pb - sha2-256
|
|
shortStringCidV1Sha512 = "bafkrgqbqt3gerhas23vuzrapkdeqf4vu2dwxp3srdj6hvg6nhsug2tgyn6mj3u23yx7utftq3i2ckw2fwdh5qmhid5qf3t35yvkc5e5ottlw6"
|
|
)
|
|
|
|
const (
|
|
cidV0Length = 34 // cidv0 sha2-256
|
|
cidV1Length = 36 // cidv1 sha2-256
|
|
)
|
|
|
|
t.Run("produced cid version: implicit default (CIDv0)", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString)
|
|
require.Equal(t, shortStringCidV0, cidStr)
|
|
})
|
|
|
|
t.Run("produced cid version: follows user-set configuration Import.CidVersion=0", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init()
|
|
node.UpdateConfig(func(cfg *config.Config) {
|
|
cfg.Import.CidVersion = *config.NewOptionalInteger(0)
|
|
})
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString)
|
|
require.Equal(t, shortStringCidV0, cidStr)
|
|
})
|
|
|
|
t.Run("produced cid multihash: follows user-set configuration in Import.HashFunction", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init()
|
|
node.UpdateConfig(func(cfg *config.Config) {
|
|
cfg.Import.HashFunction = *config.NewOptionalString("sha2-512")
|
|
})
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString)
|
|
require.Equal(t, shortStringCidV1Sha512, cidStr)
|
|
})
|
|
|
|
t.Run("produced cid version: follows user-set configuration Import.CidVersion=1", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init()
|
|
node.UpdateConfig(func(cfg *config.Config) {
|
|
cfg.Import.CidVersion = *config.NewOptionalInteger(1)
|
|
})
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString)
|
|
require.Equal(t, shortStringCidV1, cidStr)
|
|
})
|
|
|
|
t.Run("produced cid version: command flag overrides configuration in Import.CidVersion", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init()
|
|
node.UpdateConfig(func(cfg *config.Config) {
|
|
cfg.Import.CidVersion = *config.NewOptionalInteger(1)
|
|
})
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString, "--cid-version", "0")
|
|
require.Equal(t, shortStringCidV0, cidStr)
|
|
})
|
|
|
|
t.Run("produced unixfs raw leaves: follows user-set configuration Import.UnixFSRawLeaves", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init()
|
|
node.UpdateConfig(func(cfg *config.Config) {
|
|
// CIDv1 defaults to raw-leaves=true
|
|
cfg.Import.CidVersion = *config.NewOptionalInteger(1)
|
|
// disable manually
|
|
cfg.Import.UnixFSRawLeaves = config.False
|
|
})
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString)
|
|
require.Equal(t, shortStringCidV1NoRawLeaves, cidStr)
|
|
})
|
|
|
|
t.Run("ipfs add --pin-name=foo", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
pinName := "test-pin-name"
|
|
cidStr := node.IPFSAddStr(shortString, "--pin-name", pinName)
|
|
require.Equal(t, shortStringCidV0, cidStr)
|
|
|
|
pinList := node.IPFS("pin", "ls", "--names").Stdout.Trimmed()
|
|
require.Contains(t, pinList, shortStringCidV0)
|
|
require.Contains(t, pinList, pinName)
|
|
})
|
|
|
|
t.Run("ipfs add --pin=false --pin-name=foo returns an error", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
// Use RunIPFS to allow for errors without assertion
|
|
result := node.RunIPFS("add", "--pin=false", "--pin-name=foo")
|
|
require.Error(t, result.Err, "Expected an error due to incompatible --pin and --pin-name")
|
|
require.Contains(t, result.Stderr.String(), "pin-name option requires pin to be set")
|
|
})
|
|
|
|
t.Run("ipfs add --pin-name without value should fail", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
// When --pin-name is passed without any value, it should fail
|
|
result := node.RunIPFS("add", "--pin-name")
|
|
require.Error(t, result.Err, "Expected an error when --pin-name has no value")
|
|
require.Contains(t, result.Stderr.String(), "missing argument for option \"pin-name\"")
|
|
})
|
|
|
|
t.Run("produced unixfs max file links: command flag --max-file-links overrides configuration in Import.UnixFSFileMaxLinks", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
//
|
|
// UnixFSChunker=size-262144 (256KiB)
|
|
// Import.UnixFSFileMaxLinks=174
|
|
node := harness.NewT(t).NewNode().Init("--profile=legacy-cid-v0") // legacy-cid-v0 for determinism across all params
|
|
node.UpdateConfig(func(cfg *config.Config) {
|
|
cfg.Import.UnixFSChunker = *config.NewOptionalString("size-262144") // 256 KiB chunks
|
|
cfg.Import.UnixFSFileMaxLinks = *config.NewOptionalInteger(174) // max 174 per level
|
|
})
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
// Add 174MiB file:
|
|
// 1024 * 256KiB should fit in single layer
|
|
seed := shortString
|
|
cidStr := node.IPFSAddDeterministic("262144KiB", seed, "--max-file-links", "1024")
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
|
|
// Expect 1024 links due to cli parameter raising link limit from 174 to 1024
|
|
require.Equal(t, 1024, len(root.Links))
|
|
// expect same CID every time
|
|
require.Equal(t, "QmbBftNHWmjSWKLC49dMVrfnY8pjrJYntiAXirFJ7oJrNk", cidStr)
|
|
})
|
|
|
|
t.Run("ipfs init --profile=legacy-cid-v0 sets config that produces legacy CIDv0", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init("--profile=legacy-cid-v0")
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString)
|
|
require.Equal(t, shortStringCidV0, cidStr)
|
|
})
|
|
|
|
t.Run("ipfs init --profile=legacy-cid-v0 applies UnixFSChunker=size-262144 and UnixFSFileMaxLinks", func(t *testing.T) {
|
|
t.Parallel()
|
|
seed := "v0-seed"
|
|
profile := "--profile=legacy-cid-v0"
|
|
|
|
t.Run("under UnixFSFileMaxLinks=174", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init(profile)
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
// Add 44544KiB file:
|
|
// 174 * 256KiB should fit in single DAG layer
|
|
cidStr := node.IPFSAddDeterministic("44544KiB", seed)
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
require.Equal(t, 174, len(root.Links))
|
|
// expect same CID every time
|
|
require.Equal(t, "QmUbBALi174SnogsUzLpYbD4xPiBSFANF4iztWCsHbMKh2", cidStr)
|
|
})
|
|
|
|
t.Run("above UnixFSFileMaxLinks=174", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init(profile)
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
// add 256KiB (one more block), it should force rebalancing DAG and moving most to second layer
|
|
cidStr := node.IPFSAddDeterministic("44800KiB", seed)
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
require.Equal(t, 2, len(root.Links))
|
|
// expect same CID every time
|
|
require.Equal(t, "QmepeWtdmS1hHXx1oZXsPUv6bMrfRRKfZcoPPU4eEfjnbf", cidStr)
|
|
})
|
|
})
|
|
|
|
t.Run("ipfs init --profile=legacy-cid-v0 applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) {
|
|
t.Parallel()
|
|
seed := "hamt-legacy-cid-v0"
|
|
profile := "--profile=legacy-cid-v0"
|
|
|
|
t.Run("under UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init(profile)
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
randDir, err := os.MkdirTemp(node.Dir, seed)
|
|
require.NoError(t, err)
|
|
|
|
// Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total
|
|
err = createDirectoryForHAMT(randDir, cidV0Length, "255KiB", seed)
|
|
require.NoError(t, err)
|
|
cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
|
|
|
|
// Confirm the number of links is more than UnixFSHAMTDirectorySizeThreshold (indicating regular "basic" directory"
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
require.Equal(t, 903, len(root.Links))
|
|
})
|
|
|
|
t.Run("above UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init(profile)
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
randDir, err := os.MkdirTemp(node.Dir, seed)
|
|
require.NoError(t, err)
|
|
|
|
// Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total
|
|
err = createDirectoryForHAMT(randDir, cidV0Length, "257KiB", seed)
|
|
require.NoError(t, err)
|
|
cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
|
|
|
|
// Confirm this time, the number of links is less than UnixFSHAMTDirectorySizeThreshold
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
require.Equal(t, 252, len(root.Links))
|
|
})
|
|
})
|
|
|
|
t.Run("ipfs init --profile=test-cid-v1 produces CIDv1 with raw leaves", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init("--profile=test-cid-v1")
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString)
|
|
require.Equal(t, shortStringCidV1, cidStr) // raw leaf
|
|
})
|
|
|
|
t.Run("ipfs init --profile=test-cid-v1 applies UnixFSChunker=size-1048576", func(t *testing.T) {
|
|
t.Parallel()
|
|
seed := "v1-seed"
|
|
profile := "--profile=test-cid-v1"
|
|
|
|
t.Run("under UnixFSFileMaxLinks=174", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init(profile)
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
// Add 174MiB file:
|
|
// 174 * 1MiB should fit in single layer
|
|
cidStr := node.IPFSAddDeterministic("174MiB", seed)
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
require.Equal(t, 174, len(root.Links))
|
|
// expect same CID every time
|
|
require.Equal(t, "bafybeigwduxcf2aawppv3isnfeshnimkyplvw3hthxjhr2bdeje4tdaicu", cidStr)
|
|
})
|
|
|
|
t.Run("above UnixFSFileMaxLinks=174", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init(profile)
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
// add +1MiB (one more block), it should force rebalancing DAG and moving most to second layer
|
|
cidStr := node.IPFSAddDeterministic("175MiB", seed)
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
require.Equal(t, 2, len(root.Links))
|
|
// expect same CID every time
|
|
require.Equal(t, "bafybeidhd7lo2n2v7lta5yamob3xwhbxcczmmtmhquwhjesi35jntf7mpu", cidStr)
|
|
})
|
|
})
|
|
|
|
t.Run("ipfs init --profile=test-cid-v1 applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) {
|
|
t.Parallel()
|
|
seed := "hamt-cid-v1"
|
|
profile := "--profile=test-cid-v1"
|
|
|
|
t.Run("under UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init(profile)
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
randDir, err := os.MkdirTemp(node.Dir, seed)
|
|
require.NoError(t, err)
|
|
|
|
// Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total
|
|
err = createDirectoryForHAMT(randDir, cidV1Length, "255KiB", seed)
|
|
require.NoError(t, err)
|
|
cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
|
|
|
|
// Confirm the number of links is more than UnixFSHAMTDirectoryMaxFanout (indicating regular "basic" directory"
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
require.Equal(t, 897, len(root.Links))
|
|
})
|
|
|
|
t.Run("above UnixFSHAMTDirectorySizeThreshold=256KiB", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init(profile)
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
randDir, err := os.MkdirTemp(node.Dir, seed)
|
|
require.NoError(t, err)
|
|
|
|
// Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total
|
|
err = createDirectoryForHAMT(randDir, cidV1Length, "257KiB", seed)
|
|
require.NoError(t, err)
|
|
cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
|
|
|
|
// Confirm this time, the number of links is less than UnixFSHAMTDirectoryMaxFanout
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
require.Equal(t, 252, len(root.Links))
|
|
})
|
|
})
|
|
|
|
t.Run("ipfs init --profile=test-cid-v1-wide applies UnixFSChunker=size-1048576 and UnixFSFileMaxLinks=1024", func(t *testing.T) {
|
|
t.Parallel()
|
|
seed := "v1-seed-1024"
|
|
profile := "--profile=test-cid-v1-wide"
|
|
|
|
t.Run("under UnixFSFileMaxLinks=1024", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init(profile)
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
// Add 174MiB file:
|
|
// 1024 * 1MiB should fit in single layer
|
|
cidStr := node.IPFSAddDeterministic("1024MiB", seed)
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
require.Equal(t, 1024, len(root.Links))
|
|
// expect same CID every time
|
|
require.Equal(t, "bafybeiej5w63ir64oxgkr5htqmlerh5k2rqflurn2howimexrlkae64xru", cidStr)
|
|
})
|
|
|
|
t.Run("above UnixFSFileMaxLinks=1024", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init(profile)
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
// add +1MiB (one more block), it should force rebalancing DAG and moving most to second layer
|
|
cidStr := node.IPFSAddDeterministic("1025MiB", seed)
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
require.Equal(t, 2, len(root.Links))
|
|
// expect same CID every time
|
|
require.Equal(t, "bafybeieilp2qx24pe76hxrxe6bpef5meuxto3kj5dd6mhb5kplfeglskdm", cidStr)
|
|
})
|
|
})
|
|
|
|
t.Run("ipfs init --profile=test-cid-v1-wide applies UnixFSHAMTDirectoryMaxFanout=256 and UnixFSHAMTDirectorySizeThreshold=1MiB", func(t *testing.T) {
|
|
t.Parallel()
|
|
seed := "hamt-cid-v1"
|
|
profile := "--profile=test-cid-v1-wide"
|
|
|
|
t.Run("under UnixFSHAMTDirectorySizeThreshold=1MiB", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init(profile)
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
randDir, err := os.MkdirTemp(node.Dir, seed)
|
|
require.NoError(t, err)
|
|
|
|
// Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total
|
|
err = createDirectoryForHAMT(randDir, cidV1Length, "1023KiB", seed)
|
|
require.NoError(t, err)
|
|
cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
|
|
|
|
// Confirm the number of links is more than UnixFSHAMTDirectoryMaxFanout (indicating regular "basic" directory"
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
require.Equal(t, 3599, len(root.Links))
|
|
})
|
|
|
|
t.Run("above UnixFSHAMTDirectorySizeThreshold=1MiB", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init(profile)
|
|
node.StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
randDir, err := os.MkdirTemp(node.Dir, seed)
|
|
require.NoError(t, err)
|
|
|
|
// Create directory with a lot of files that have filenames which together take close to UnixFSHAMTDirectorySizeThreshold in total
|
|
err = createDirectoryForHAMT(randDir, cidV1Length, "1025KiB", seed)
|
|
require.NoError(t, err)
|
|
cidStr := node.IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
|
|
|
|
// Confirm this time, the number of links is less than UnixFSHAMTDirectoryMaxFanout
|
|
root, err := node.InspectPBNode(cidStr)
|
|
assert.NoError(t, err)
|
|
require.Equal(t, 992, len(root.Links))
|
|
})
|
|
})
|
|
|
|
t.Run("ipfs add --hidden", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Helper to create test directory with hidden file
|
|
setupTestDir := func(t *testing.T, node *harness.Node) string {
|
|
testDir, err := os.MkdirTemp(node.Dir, "hidden-test")
|
|
require.NoError(t, err)
|
|
require.NoError(t, os.WriteFile(filepath.Join(testDir, "visible.txt"), []byte("visible"), 0644))
|
|
require.NoError(t, os.WriteFile(filepath.Join(testDir, ".hidden"), []byte("hidden"), 0644))
|
|
return testDir
|
|
}
|
|
|
|
t.Run("default excludes hidden files", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
testDir := setupTestDir(t, node)
|
|
cidStr := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed()
|
|
lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed()
|
|
require.Contains(t, lsOutput, "visible.txt")
|
|
require.NotContains(t, lsOutput, ".hidden")
|
|
})
|
|
|
|
t.Run("--hidden includes hidden files", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
testDir := setupTestDir(t, node)
|
|
cidStr := node.IPFS("add", "-r", "-Q", "--hidden", testDir).Stdout.Trimmed()
|
|
lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed()
|
|
require.Contains(t, lsOutput, "visible.txt")
|
|
require.Contains(t, lsOutput, ".hidden")
|
|
})
|
|
|
|
t.Run("-H includes hidden files", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
testDir := setupTestDir(t, node)
|
|
cidStr := node.IPFS("add", "-r", "-Q", "-H", testDir).Stdout.Trimmed()
|
|
lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed()
|
|
require.Contains(t, lsOutput, "visible.txt")
|
|
require.Contains(t, lsOutput, ".hidden")
|
|
})
|
|
})
|
|
|
|
t.Run("ipfs add --empty-dirs", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Helper to create test directory with empty subdirectory
|
|
setupTestDir := func(t *testing.T, node *harness.Node) string {
|
|
testDir, err := os.MkdirTemp(node.Dir, "empty-dirs-test")
|
|
require.NoError(t, err)
|
|
require.NoError(t, os.Mkdir(filepath.Join(testDir, "empty-subdir"), 0755))
|
|
require.NoError(t, os.WriteFile(filepath.Join(testDir, "file.txt"), []byte("content"), 0644))
|
|
return testDir
|
|
}
|
|
|
|
t.Run("default includes empty directories", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
testDir := setupTestDir(t, node)
|
|
cidStr := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed()
|
|
require.Contains(t, node.IPFS("ls", cidStr).Stdout.Trimmed(), "empty-subdir")
|
|
})
|
|
|
|
t.Run("--empty-dirs=true includes empty directories", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
testDir := setupTestDir(t, node)
|
|
cidStr := node.IPFS("add", "-r", "-Q", "--empty-dirs=true", testDir).Stdout.Trimmed()
|
|
require.Contains(t, node.IPFS("ls", cidStr).Stdout.Trimmed(), "empty-subdir")
|
|
})
|
|
|
|
t.Run("--empty-dirs=false excludes empty directories", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
testDir := setupTestDir(t, node)
|
|
cidStr := node.IPFS("add", "-r", "-Q", "--empty-dirs=false", testDir).Stdout.Trimmed()
|
|
lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed()
|
|
require.NotContains(t, lsOutput, "empty-subdir")
|
|
require.Contains(t, lsOutput, "file.txt")
|
|
})
|
|
})
|
|
|
|
t.Run("ipfs add --dereference-symlinks", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Helper to create test directory with a file and symlink to it
|
|
setupTestDir := func(t *testing.T, node *harness.Node) string {
|
|
testDir, err := os.MkdirTemp(node.Dir, "deref-symlinks-test")
|
|
require.NoError(t, err)
|
|
|
|
targetFile := filepath.Join(testDir, "target.txt")
|
|
require.NoError(t, os.WriteFile(targetFile, []byte("target content"), 0644))
|
|
|
|
// Create symlink pointing to target
|
|
require.NoError(t, os.Symlink("target.txt", filepath.Join(testDir, "link.txt")))
|
|
|
|
return testDir
|
|
}
|
|
|
|
t.Run("default preserves symlinks", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
testDir := setupTestDir(t, node)
|
|
|
|
// Add directory with symlink (default: preserve)
|
|
dirCID := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed()
|
|
|
|
// Get to a new directory and verify symlink is preserved
|
|
outDir, err := os.MkdirTemp(node.Dir, "symlink-get-out")
|
|
require.NoError(t, err)
|
|
node.IPFS("get", "-o", outDir, dirCID)
|
|
|
|
// Check that link.txt is a symlink (ipfs get -o puts files directly in outDir)
|
|
linkPath := filepath.Join(outDir, "link.txt")
|
|
fi, err := os.Lstat(linkPath)
|
|
require.NoError(t, err)
|
|
require.True(t, fi.Mode()&os.ModeSymlink != 0, "link.txt should be a symlink")
|
|
|
|
// Verify symlink target
|
|
target, err := os.Readlink(linkPath)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "target.txt", target)
|
|
})
|
|
|
|
t.Run("--dereference-symlinks resolves nested symlinks", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
testDir := setupTestDir(t, node)
|
|
|
|
// Add directory with dereference flag - nested symlinks should be resolved
|
|
dirCID := node.IPFS("add", "-r", "-Q", "--dereference-symlinks", testDir).Stdout.Trimmed()
|
|
|
|
// Get and verify symlink was dereferenced to regular file
|
|
outDir, err := os.MkdirTemp(node.Dir, "symlink-get-out")
|
|
require.NoError(t, err)
|
|
node.IPFS("get", "-o", outDir, dirCID)
|
|
|
|
linkPath := filepath.Join(outDir, "link.txt")
|
|
fi, err := os.Lstat(linkPath)
|
|
require.NoError(t, err)
|
|
|
|
// Should be a regular file, not a symlink
|
|
require.False(t, fi.Mode()&os.ModeSymlink != 0,
|
|
"link.txt should be dereferenced to regular file, not preserved as symlink")
|
|
|
|
// Content should match the target file
|
|
content, err := os.ReadFile(linkPath)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "target content", string(content))
|
|
})
|
|
|
|
t.Run("--dereference-args is deprecated", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init().StartDaemon()
|
|
defer node.StopDaemon()
|
|
|
|
testDir := setupTestDir(t, node)
|
|
|
|
res := node.RunIPFS("add", "-Q", "--dereference-args", filepath.Join(testDir, "link.txt"))
|
|
require.Error(t, res.Err)
|
|
require.Contains(t, res.Stderr.String(), "--dereference-args is deprecated")
|
|
})
|
|
})
|
|
}
|
|
|
|
func TestAddFastProvide(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const (
|
|
shortString = "hello world"
|
|
shortStringCidV0 = "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD" // cidv0 - dag-pb - sha2-256
|
|
)
|
|
|
|
t.Run("fast-provide-root disabled via config: verify skipped in logs", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init()
|
|
node.UpdateConfig(func(cfg *config.Config) {
|
|
cfg.Import.FastProvideRoot = config.False
|
|
})
|
|
|
|
// Start daemon with debug logging
|
|
node.StartDaemonWithReq(harness.RunRequest{
|
|
CmdOpts: []harness.CmdOpt{
|
|
harness.RunWithEnv(map[string]string{
|
|
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
|
}),
|
|
},
|
|
}, "")
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString)
|
|
require.Equal(t, shortStringCidV0, cidStr)
|
|
|
|
// Verify fast-provide-root was disabled
|
|
daemonLog := node.Daemon.Stderr.String()
|
|
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
|
})
|
|
|
|
t.Run("fast-provide-root enabled with wait=false: verify async provide", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init()
|
|
// Use default config (FastProvideRoot=true, FastProvideWait=false)
|
|
|
|
node.StartDaemonWithReq(harness.RunRequest{
|
|
CmdOpts: []harness.CmdOpt{
|
|
harness.RunWithEnv(map[string]string{
|
|
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
|
}),
|
|
},
|
|
}, "")
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString)
|
|
require.Equal(t, shortStringCidV0, cidStr)
|
|
|
|
daemonLog := node.Daemon.Stderr
|
|
// Should see async mode started
|
|
require.Contains(t, daemonLog.String(), "fast-provide-root: enabled")
|
|
require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously")
|
|
|
|
// Wait for async completion or failure (up to 11 seconds - slightly more than fastProvideTimeout)
|
|
// In test environment with no DHT peers, this will fail with "failed to find any peer in table"
|
|
completedOrFailed := waitForLogMessage(daemonLog, "async provide completed", 11*time.Second) ||
|
|
waitForLogMessage(daemonLog, "async provide failed", 11*time.Second)
|
|
require.True(t, completedOrFailed, "async provide should complete or fail within timeout")
|
|
})
|
|
|
|
t.Run("fast-provide-root enabled with wait=true: verify sync provide", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init()
|
|
node.UpdateConfig(func(cfg *config.Config) {
|
|
cfg.Import.FastProvideWait = config.True
|
|
})
|
|
|
|
node.StartDaemonWithReq(harness.RunRequest{
|
|
CmdOpts: []harness.CmdOpt{
|
|
harness.RunWithEnv(map[string]string{
|
|
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
|
}),
|
|
},
|
|
}, "")
|
|
defer node.StopDaemon()
|
|
|
|
// Use Runner.Run with stdin to allow for expected errors
|
|
res := node.Runner.Run(harness.RunRequest{
|
|
Path: node.IPFSBin,
|
|
Args: []string{"add", "-q"},
|
|
CmdOpts: []harness.CmdOpt{
|
|
harness.RunWithStdin(strings.NewReader(shortString)),
|
|
},
|
|
})
|
|
|
|
// In sync mode (wait=true), provide errors propagate and fail the command.
|
|
// Test environment uses 'test' profile with no bootstrappers, and CI has
|
|
// insufficient peers for proper DHT puts, so we expect this to fail with
|
|
// "failed to find any peer in table" error from the DHT.
|
|
require.Equal(t, 1, res.ExitCode())
|
|
require.Contains(t, res.Stderr.String(), "Error: fast-provide: failed to find any peer in table")
|
|
|
|
daemonLog := node.Daemon.Stderr.String()
|
|
// Should see sync mode started
|
|
require.Contains(t, daemonLog, "fast-provide-root: enabled")
|
|
require.Contains(t, daemonLog, "fast-provide-root: providing synchronously")
|
|
require.Contains(t, daemonLog, "sync provide failed") // Verify the failure was logged
|
|
})
|
|
|
|
t.Run("fast-provide-wait ignored when root disabled", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init()
|
|
node.UpdateConfig(func(cfg *config.Config) {
|
|
cfg.Import.FastProvideRoot = config.False
|
|
cfg.Import.FastProvideWait = config.True
|
|
})
|
|
|
|
node.StartDaemonWithReq(harness.RunRequest{
|
|
CmdOpts: []harness.CmdOpt{
|
|
harness.RunWithEnv(map[string]string{
|
|
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
|
}),
|
|
},
|
|
}, "")
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString)
|
|
require.Equal(t, shortStringCidV0, cidStr)
|
|
|
|
daemonLog := node.Daemon.Stderr.String()
|
|
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
|
require.Contains(t, daemonLog, "wait-flag-ignored")
|
|
})
|
|
|
|
t.Run("CLI flag overrides config: flag=true overrides config=false", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init()
|
|
node.UpdateConfig(func(cfg *config.Config) {
|
|
cfg.Import.FastProvideRoot = config.False
|
|
})
|
|
|
|
node.StartDaemonWithReq(harness.RunRequest{
|
|
CmdOpts: []harness.CmdOpt{
|
|
harness.RunWithEnv(map[string]string{
|
|
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
|
}),
|
|
},
|
|
}, "")
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString, "--fast-provide-root=true")
|
|
require.Equal(t, shortStringCidV0, cidStr)
|
|
|
|
daemonLog := node.Daemon.Stderr
|
|
// Flag should enable it despite config saying false
|
|
require.Contains(t, daemonLog.String(), "fast-provide-root: enabled")
|
|
require.Contains(t, daemonLog.String(), "fast-provide-root: providing asynchronously")
|
|
})
|
|
|
|
t.Run("CLI flag overrides config: flag=false overrides config=true", func(t *testing.T) {
|
|
t.Parallel()
|
|
node := harness.NewT(t).NewNode().Init()
|
|
node.UpdateConfig(func(cfg *config.Config) {
|
|
cfg.Import.FastProvideRoot = config.True
|
|
})
|
|
|
|
node.StartDaemonWithReq(harness.RunRequest{
|
|
CmdOpts: []harness.CmdOpt{
|
|
harness.RunWithEnv(map[string]string{
|
|
"GOLOG_LOG_LEVEL": "error,core/commands=debug,core/commands/cmdenv=debug",
|
|
}),
|
|
},
|
|
}, "")
|
|
defer node.StopDaemon()
|
|
|
|
cidStr := node.IPFSAddStr(shortString, "--fast-provide-root=false")
|
|
require.Equal(t, shortStringCidV0, cidStr)
|
|
|
|
daemonLog := node.Daemon.Stderr.String()
|
|
// Flag should disable it despite config saying true
|
|
require.Contains(t, daemonLog, "fast-provide-root: skipped")
|
|
})
|
|
}
|
|
|
|
// createDirectoryForHAMT aims to create enough files with long names for the directory block to be close to the UnixFSHAMTDirectorySizeThreshold.
|
|
// The calculation is based on boxo's HAMTShardingSize and sizeBelowThreshold which calculates ballpark size of the block
|
|
// by adding length of link names and the binary cid length.
|
|
// See https://github.com/ipfs/boxo/blob/6c5a07602aed248acc86598f30ab61923a54a83e/ipld/unixfs/io/directory.go#L491
|
|
func createDirectoryForHAMT(dirPath string, cidLength int, unixfsNodeSizeTarget, seed string) error {
|
|
hamtThreshold, err := humanize.ParseBytes(unixfsNodeSizeTarget)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Calculate how many files with long filenames are needed to hit UnixFSHAMTDirectorySizeThreshold
|
|
nameLen := 255 // max that works across windows/macos/linux
|
|
alphabetLen := len(testutils.AlphabetEasy)
|
|
numFiles := int(hamtThreshold) / (nameLen + cidLength)
|
|
|
|
// Deterministic pseudo-random bytes for static CID
|
|
drand, err := testutils.DeterministicRandomReader(unixfsNodeSizeTarget, seed)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Create necessary files in a single, flat directory
|
|
for i := 0; i < numFiles; i++ {
|
|
buf := make([]byte, nameLen)
|
|
_, err := io.ReadFull(drand, buf)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Convert deterministic pseudo-random bytes to ASCII
|
|
var sb strings.Builder
|
|
|
|
for _, b := range buf {
|
|
// Map byte to printable ASCII range (33-126)
|
|
char := testutils.AlphabetEasy[int(b)%alphabetLen]
|
|
sb.WriteRune(char)
|
|
}
|
|
filename := sb.String()[:nameLen]
|
|
filePath := filepath.Join(dirPath, filename)
|
|
|
|
// Create empty file
|
|
f, err := os.Create(filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
f.Close()
|
|
}
|
|
return nil
|
|
}
|