diff --git a/config/import.go b/config/import.go index 56511a079..995c371d2 100644 --- a/config/import.go +++ b/config/import.go @@ -35,17 +35,13 @@ const ( HAMTSizeEstimationBlock = "block" // full serialized dag-pb block size HAMTSizeEstimationDisabled = "disabled" // disable HAMT sharding entirely - // SymlinkMode values for Import.UnixFSSymlinkMode - SymlinkModePreserve = "preserve" // preserve symlinks as UnixFS symlink nodes - SymlinkModeDereference = "dereference" // dereference symlinks, import target content - // DAGLayout values for Import.UnixFSDAGLayout DAGLayoutBalanced = "balanced" // balanced DAG layout (default) DAGLayoutTrickle = "trickle" // trickle DAG layout DefaultUnixFSHAMTDirectorySizeEstimation = HAMTSizeEstimationLinks // legacy behavior - DefaultUnixFSSymlinkMode = SymlinkModePreserve // preserve symlinks as UnixFS symlink nodes DefaultUnixFSDAGLayout = DAGLayoutBalanced // balanced DAG layout + DefaultUnixFSIncludeEmptyDirs = true // include empty directories ) var ( @@ -66,7 +62,6 @@ type Import struct { UnixFSHAMTDirectoryMaxFanout OptionalInteger UnixFSHAMTDirectorySizeThreshold OptionalBytes UnixFSHAMTDirectorySizeEstimation OptionalString // "links", "block", or "disabled" - UnixFSSymlinkMode OptionalString // "preserve" or "dereference" UnixFSDAGLayout OptionalString // "balanced" or "trickle" BatchMaxNodes OptionalInteger BatchMaxSize OptionalInteger @@ -161,18 +156,6 @@ func ValidateImportConfig(cfg *Import) error { } } - // Validate UnixFSSymlinkMode - if !cfg.UnixFSSymlinkMode.IsDefault() { - mode := cfg.UnixFSSymlinkMode.WithDefault(DefaultUnixFSSymlinkMode) - switch mode { - case SymlinkModePreserve, SymlinkModeDereference: - // valid - default: - return fmt.Errorf("Import.UnixFSSymlinkMode must be %q or %q, got %q", - SymlinkModePreserve, SymlinkModeDereference, mode) - } - } - // Validate UnixFSDAGLayout if !cfg.UnixFSDAGLayout.IsDefault() { layout := cfg.UnixFSDAGLayout.WithDefault(DefaultUnixFSDAGLayout) diff --git a/config/import_test.go b/config/import_test.go index 31e6dcf56..5d9605c1d 100644 --- a/config/import_test.go +++ b/config/import_test.go @@ -446,43 +446,6 @@ func TestValidateImportConfig_HAMTSizeEstimation(t *testing.T) { } } -func TestValidateImportConfig_SymlinkMode(t *testing.T) { - tests := []struct { - name string - value string - wantErr bool - errMsg string - }{ - {name: "valid preserve", value: SymlinkModePreserve, wantErr: false}, - {name: "valid dereference", value: SymlinkModeDereference, wantErr: false}, - {name: "invalid unknown", value: "unknown", wantErr: true, errMsg: "must be"}, - {name: "invalid empty", value: "", wantErr: true, errMsg: "must be"}, - {name: "invalid follow", value: "follow", wantErr: true, errMsg: "must be"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg := &Import{ - UnixFSSymlinkMode: *NewOptionalString(tt.value), - } - - err := ValidateImportConfig(cfg) - - if tt.wantErr { - if err == nil { - t.Errorf("expected error for value=%q, got nil", tt.value) - } else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) { - t.Errorf("error = %v, want error containing %q", err, tt.errMsg) - } - } else { - if err != nil { - t.Errorf("unexpected error for value=%q: %v", tt.value, err) - } - } - }) - } -} - func TestValidateImportConfig_DAGLayout(t *testing.T) { tests := []struct { name string @@ -528,7 +491,7 @@ func TestImport_HAMTSizeEstimationMode(t *testing.T) { {HAMTSizeEstimationLinks, io.SizeEstimationLinks}, {HAMTSizeEstimationBlock, io.SizeEstimationBlock}, {HAMTSizeEstimationDisabled, io.SizeEstimationDisabled}, - {"", io.SizeEstimationLinks}, // default (unset returns default) + {"", io.SizeEstimationLinks}, // default (unset returns default) {"unknown", io.SizeEstimationLinks}, // fallback to default } diff --git a/config/profile.go b/config/profile.go index 904a32497..7832c7599 100644 --- a/config/profile.go +++ b/config/profile.go @@ -338,7 +338,6 @@ See https://github.com/ipfs/specs/pull/499`, c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256) c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB") c.Import.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(HAMTSizeEstimationBlock) - c.Import.UnixFSSymlinkMode = *NewOptionalString(SymlinkModePreserve) c.Import.UnixFSDAGLayout = *NewOptionalString(DAGLayoutBalanced) return nil }, @@ -436,7 +435,6 @@ func applyUnixFSv02015(c *Config) error { c.Import.UnixFSHAMTDirectoryMaxFanout = *NewOptionalInteger(256) c.Import.UnixFSHAMTDirectorySizeThreshold = *NewOptionalBytes("256KiB") c.Import.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(HAMTSizeEstimationLinks) - c.Import.UnixFSSymlinkMode = *NewOptionalString(SymlinkModePreserve) c.Import.UnixFSDAGLayout = *NewOptionalString(DAGLayoutBalanced) return nil } diff --git a/core/commands/add.go b/core/commands/add.go index cb4bcb312..a56b69434 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -68,6 +68,7 @@ const ( mtimeNsecsOptionName = "mtime-nsecs" fastProvideRootOptionName = "fast-provide-root" fastProvideWaitOptionName = "fast-provide-wait" + emptyDirsOptionName = "empty-dirs" ) const ( @@ -147,6 +148,18 @@ to find it in the future: See 'ipfs files --help' to learn more about using MFS for keeping track of added files and directories. +SYMLINK HANDLING: + +By default, symbolic links are preserved as UnixFS symlink nodes that store +the target path. Use --dereference-symlinks to resolve symlinks to their +target content instead: + + > ipfs add -r --dereference-symlinks ./mydir + +This recursively resolves all symlinks encountered during directory traversal. +Symlinks to files become regular file content, symlinks to directories are +traversed and their contents are added. + CHUNKING EXAMPLES: The chunker option, '-s', specifies the chunking strategy that dictates @@ -200,11 +213,13 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import Options: []cmds.Option{ // Input Processing cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive) - cmds.OptionDerefArgs, // a builtin option that resolves passed in filesystem links (--dereference-args) + cmds.OptionDerefArgs, // DEPRECATED: use --dereference-symlinks instead cmds.OptionStdinName, // a builtin option that optionally allows wrapping stdin into a named file cmds.OptionHidden, cmds.OptionIgnore, cmds.OptionIgnoreRules, + cmds.BoolOption(emptyDirsOptionName, "E", "Include empty directories in the import.").WithDefault(config.DefaultUnixFSIncludeEmptyDirs), + cmds.OptionDerefSymlinks, // resolve symlinks to their target content // Output Control cmds.BoolOption(quietOptionName, "q", "Write minimal output."), cmds.BoolOption(quieterOptionName, "Q", "Write only final hash."), @@ -274,7 +289,7 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import } progress, _ := req.Options[progressOptionName].(bool) - trickle, _ := req.Options[trickleOptionName].(bool) + trickle, trickleSet := req.Options[trickleOptionName].(bool) wrap, _ := req.Options[wrapOptionName].(bool) onlyHash, _ := req.Options[onlyHashOptionName].(bool) silent, _ := req.Options[silentOptionName].(bool) @@ -312,6 +327,19 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import mtimeNsecs, _ := req.Options[mtimeNsecsOptionName].(uint) fastProvideRoot, fastProvideRootSet := req.Options[fastProvideRootOptionName].(bool) fastProvideWait, fastProvideWaitSet := req.Options[fastProvideWaitOptionName].(bool) + emptyDirs, _ := req.Options[emptyDirsOptionName].(bool) + + // Handle --dereference-args deprecation + derefArgs, derefArgsSet := req.Options[cmds.DerefLong].(bool) + if derefArgsSet && derefArgs { + return fmt.Errorf("--dereference-args is deprecated: use --dereference-symlinks instead") + } + + // Wire --trickle from config + if !trickleSet && !cfg.Import.UnixFSDAGLayout.IsDefault() { + layout := cfg.Import.UnixFSDAGLayout.WithDefault(config.DefaultUnixFSDAGLayout) + trickle = layout == config.DAGLayoutTrickle + } if chunker == "" { chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker) @@ -409,6 +437,8 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#import options.Unixfs.PreserveMode(preserveMode), options.Unixfs.PreserveMtime(preserveMtime), + + options.Unixfs.IncludeEmptyDirs(emptyDirs), } if mode != 0 { diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 7f068a227..a5507f557 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -183,6 +183,9 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options fileAdder.PreserveMtime = settings.PreserveMtime fileAdder.FileMode = settings.Mode fileAdder.FileMtime = settings.Mtime + if settings.IncludeEmptyDirsSet { + fileAdder.IncludeEmptyDirs = settings.IncludeEmptyDirs + } switch settings.Layout { case options.BalancedLayout: diff --git a/core/coreiface/options/unixfs.go b/core/coreiface/options/unixfs.go index 45e880ed1..32303ffc2 100644 --- a/core/coreiface/options/unixfs.go +++ b/core/coreiface/options/unixfs.go @@ -48,10 +48,12 @@ type UnixfsAddSettings struct { Silent bool Progress bool - PreserveMode bool - PreserveMtime bool - Mode os.FileMode - Mtime time.Time + PreserveMode bool + PreserveMtime bool + Mode os.FileMode + Mtime time.Time + IncludeEmptyDirs bool + IncludeEmptyDirsSet bool } type UnixfsLsSettings struct { @@ -93,10 +95,12 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix, Silent: false, Progress: false, - PreserveMode: false, - PreserveMtime: false, - Mode: 0, - Mtime: time.Time{}, + PreserveMode: false, + PreserveMtime: false, + Mode: 0, + Mtime: time.Time{}, + IncludeEmptyDirs: true, // default: include empty directories + IncludeEmptyDirsSet: false, } for _, opt := range opts { @@ -396,3 +400,12 @@ func (unixfsOpts) Mtime(seconds int64, nsecs uint32) UnixfsAddOption { return nil } } + +// IncludeEmptyDirs tells the adder to include empty directories in the DAG +func (unixfsOpts) IncludeEmptyDirs(include bool) UnixfsAddOption { + return func(settings *UnixfsAddSettings) error { + settings.IncludeEmptyDirs = include + settings.IncludeEmptyDirsSet = true + return nil + } +} diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 55a9d5bec..3b0bee057 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -26,6 +26,7 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" + "github.com/ipfs/kubo/config" coreiface "github.com/ipfs/kubo/core/coreiface" "github.com/ipfs/kubo/tracing" @@ -52,17 +53,18 @@ func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCLocker, ds ipld.DAG bufferedDS := ipld.NewBufferedDAG(ctx, ds) return &Adder{ - ctx: ctx, - pinning: p, - gcLocker: bs, - dagService: ds, - bufferedDS: bufferedDS, - Progress: false, - Pin: true, - Trickle: false, - MaxLinks: ihelper.DefaultLinksPerBlock, - MaxHAMTFanout: uio.DefaultShardWidth, - Chunker: "", + ctx: ctx, + pinning: p, + gcLocker: bs, + dagService: ds, + bufferedDS: bufferedDS, + Progress: false, + Pin: true, + Trickle: false, + MaxLinks: ihelper.DefaultLinksPerBlock, + MaxHAMTFanout: uio.DefaultShardWidth, + Chunker: "", + IncludeEmptyDirs: config.DefaultUnixFSIncludeEmptyDirs, }, nil } @@ -91,10 +93,11 @@ type Adder struct { CidBuilder cid.Builder liveNodes uint64 - PreserveMode bool - PreserveMtime bool - FileMode os.FileMode - FileMtime time.Time + PreserveMode bool + PreserveMtime bool + FileMode os.FileMode + FileMtime time.Time + IncludeEmptyDirs bool } func (adder *Adder) mfsRoot() (*mfs.Root, error) { @@ -480,6 +483,24 @@ func (adder *Adder) addFile(path string, file files.File) error { func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory, toplevel bool) error { log.Infof("adding directory: %s", path) + // Peek at first entry to check if directory is empty. + // We advance the iterator once here and continue from this position + // in the processing loop below. This avoids allocating a slice to + // collect all entries just to check for emptiness. + it := dir.Entries() + hasEntry := it.Next() + if !hasEntry { + if err := it.Err(); err != nil { + return err + } + // Directory is empty. Skip it unless IncludeEmptyDirs is set or + // this is the toplevel directory (we always include the root). + if !adder.IncludeEmptyDirs && !toplevel { + log.Debugf("skipping empty directory: %s", path) + return nil + } + } + // if we need to store mode or modification time then create a new root which includes that data if toplevel && (adder.FileMode != 0 || !adder.FileMtime.IsZero()) { mr, err := mfs.NewEmptyRoot(ctx, adder.dagService, nil, nil, @@ -515,13 +536,14 @@ func (adder *Adder) addDir(ctx context.Context, path string, dir files.Directory } } - it := dir.Entries() - for it.Next() { + // Process directory entries. The iterator was already advanced once above + // to peek for emptiness, so we start from that position. + for hasEntry { fpath := gopath.Join(path, it.Name()) - err := adder.addFileNode(ctx, fpath, it.Node(), false) - if err != nil { + if err := adder.addFileNode(ctx, fpath, it.Node(), false); err != nil { return err } + hasEntry = it.Next() } return it.Err() diff --git a/docs/changelogs/v0.40.md b/docs/changelogs/v0.40.md index 524852a30..ef8f9df76 100644 --- a/docs/changelogs/v0.40.md +++ b/docs/changelogs/v0.40.md @@ -10,6 +10,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) + - [๐Ÿ”ข UnixFS CID Profiles (IPIP-499)](#-unixfs-cid-profiles-ipip-499) - [๐Ÿงน Automatic cleanup of interrupted imports](#-automatic-cleanup-of-interrupted-imports) - [Routing V1 HTTP API now exposed by default](#routing-v1-http-api-now-exposed-by-default) - [Track total size when adding pins](#track-total-size-when-adding-pins) @@ -30,6 +31,43 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. ### ๐Ÿ”ฆ Highlights +#### ๐Ÿ”ข UnixFS CID Profiles (IPIP-499) + +This release introduces [IPIP-499](https://github.com/ipfs/specs/pull/499) UnixFS CID Profiles for reproducible CID generation across IPFS implementations. + +**New Profiles** + +- `unixfs-v1-2025`: the recommended profile for CIDv1 imports with deterministic HAMT directory sharding based on block size estimation +- `unixfs-v0-2015` (alias `legacy-cid-v0`): preserves legacy CIDv0 behavior for backward compatibility + +Apply a profile with: `ipfs config profile apply unixfs-v1-2025` + +**New `Import.*` Configuration Options** + +New [`Import.*`](https://github.com/ipfs/kubo/blob/master/docs/config.md#import) options allow fine-grained control over import parameters: + +- `Import.CidVersion`: CID version (0 or 1) +- `Import.HashFunction`: hash algorithm +- `Import.UnixFSChunker`: chunking strategy +- `Import.UnixFSRawLeaves`: raw leaf blocks +- `Import.UnixFSFileMaxLinks`: max children per file node +- `Import.UnixFSDirectoryMaxLinks`: max children per basic directory +- `Import.UnixFSHAMTDirectoryMaxFanout`: HAMT shard width +- `Import.UnixFSHAMTDirectorySizeThreshold`: threshold for HAMT sharding +- `Import.UnixFSHAMTDirectorySizeEstimation`: estimation mode (`links`, `block`, or `disabled`) +- `Import.UnixFSDAGLayout`: DAG layout (`balanced` or `trickle`) + +**Deprecated Profiles** + +The `test-cid-v1` and `test-cid-v1-wide` profiles have been removed. Use `unixfs-v1-2025` for CIDv1 imports. + +**CLI Changes** + +- New `--dereference-symlinks` flag for `ipfs add` recursively resolves symlinks to their target content (replaces deprecated `--dereference-args` which only worked on CLI arguments) +- New `--empty-dirs` / `-E` flag for `ipfs add` controls inclusion of empty directories (default: true) +- New `--hidden` / `-H` flag for `ipfs add` includes hidden files (default: false) +- The `--trickle` flag in `ipfs add` now respects `Import.UnixFSDAGLayout` config default + #### ๐Ÿงน Automatic cleanup of interrupted imports If you cancel `ipfs add` or `ipfs dag import` mid-operation, Kubo now automatically cleans up incomplete data on the next daemon start. Previously, interrupted imports would leave orphan blocks in your repository that were difficult to identify and remove without pins and running explicit garbage collection. diff --git a/docs/config.md b/docs/config.md index 16987b0ce..3b040c61e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -243,7 +243,6 @@ config file at runtime. - [`Import.UnixFSHAMTDirectoryMaxFanout`](#importunixfshamtdirectorymaxfanout) - [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold) - [`Import.UnixFSHAMTDirectorySizeEstimation`](#importunixfshamtdirectorysizeestimation) - - [`Import.UnixFSSymlinkMode`](#importunixfssymlinkmode) - [`Import.UnixFSDAGLayout`](#importunixfsdaglayout) - [`Version`](#version) - [`Version.AgentSuffix`](#versionagentsuffix) @@ -3642,9 +3641,11 @@ Type: `flag` ## `Import` -Options to configure the default options used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option. +Options to configure the default parameters used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option. -Note that using flags will override the options defined here. +These options implement [IPIP-499: UnixFS CID Profiles](https://github.com/ipfs/specs/pull/499) for reproducible CID generation across IPFS implementations. Instead of configuring individual options, you can apply a predefined profile with `ipfs config profile apply `. See [Profiles](#profiles) for available options like `unixfs-v1-2025`. + +Note that using CLI flags will override the options defined here. ### `Import.CidVersion` @@ -3845,21 +3846,6 @@ Default: `links` Type: `optionalString` -### `Import.UnixFSSymlinkMode` - -Controls how symbolic links are handled during import. - -Accepted values: - -- `preserve` (default): Store symlinks as UnixFS symlink nodes containing the target path. -- `dereference`: Follow symlinks and import the target content instead. - -Commands affected: `ipfs add` - -Default: `preserve` - -Type: `optionalString` - ### `Import.UnixFSDAGLayout` Controls the DAG layout used when chunking files. @@ -3918,7 +3904,7 @@ applied with the `--profile` flag to `ipfs init` or with the `ipfs config profil apply` command. When a profile is applied a backup of the configuration file will be created in `$IPFS_PATH`. -Configuration profiles can be applied additively. For example, both the `test-cid-v1` and `lowpower` profiles can be applied one after the other. +Configuration profiles can be applied additively. For example, both the `unixfs-v1-2025` and `lowpower` profiles can be applied one after the other. The available configuration profiles are listed below. You can also find them documented in `ipfs config profile --help`. diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 234df7d5d..7db9fd2a1 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -84,7 +84,7 @@ require ( github.com/ipfs/go-ds-pebble v0.5.8 // indirect github.com/ipfs/go-dsqueue v0.1.1 // indirect github.com/ipfs/go-fs-lock v0.1.1 // indirect - github.com/ipfs/go-ipfs-cmds v0.15.0 // indirect + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index f998634d5..18db9cc6d 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -301,8 +301,8 @@ github.com/ipfs/go-dsqueue v0.1.1 h1:6PQlHDyf9PSTN69NmwUir5+0is3tU0vRJj8zLlgK8Mc github.com/ipfs/go-dsqueue v0.1.1/go.mod h1:Xxg353WSwwzYn3FGSzZ+taSQII3pIZ+EJC8/oWRDM10= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0= -github.com/ipfs/go-ipfs-cmds v0.15.0 h1:nQDgKadrzyiFyYoZMARMIoVoSwe3gGTAfGvrWLeAQbQ= -github.com/ipfs/go-ipfs-cmds v0.15.0/go.mod h1:VABf/mv/wqvYX6hLG6Z+40eNAEw3FQO0bSm370Or3Wk= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 h1:pQ6LhtU+nEBajAgFz3uU7ta6JN4KY0W5T7JxuaRQJVE= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/go.mod b/go.mod index 37ceedbf7..9a7b844a8 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/ipfs/go-ds-measure v0.2.2 github.com/ipfs/go-ds-pebble v0.5.8 github.com/ipfs/go-fs-lock v0.1.1 - github.com/ipfs/go-ipfs-cmds v0.15.0 + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 github.com/ipfs/go-ipld-cbor v0.2.1 github.com/ipfs/go-ipld-format v0.6.3 github.com/ipfs/go-ipld-git v0.1.1 diff --git a/go.sum b/go.sum index 4e1c346a1..ee5503567 100644 --- a/go.sum +++ b/go.sum @@ -372,8 +372,8 @@ github.com/ipfs/go-dsqueue v0.1.1 h1:6PQlHDyf9PSTN69NmwUir5+0is3tU0vRJj8zLlgK8Mc github.com/ipfs/go-dsqueue v0.1.1/go.mod h1:Xxg353WSwwzYn3FGSzZ+taSQII3pIZ+EJC8/oWRDM10= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= github.com/ipfs/go-fs-lock v0.1.1/go.mod h1:2goSXMCw7QfscHmSe09oXiR34DQeUdm+ei+dhonqly0= -github.com/ipfs/go-ipfs-cmds v0.15.0 h1:nQDgKadrzyiFyYoZMARMIoVoSwe3gGTAfGvrWLeAQbQ= -github.com/ipfs/go-ipfs-cmds v0.15.0/go.mod h1:VABf/mv/wqvYX6hLG6Z+40eNAEw3FQO0bSm370Or3Wk= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 h1:pQ6LhtU+nEBajAgFz3uU7ta6JN4KY0W5T7JxuaRQJVE= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= diff --git a/test/cli/add_test.go b/test/cli/add_test.go index cda0c977d..9a232112e 100644 --- a/test/cli/add_test.go +++ b/test/cli/add_test.go @@ -449,6 +449,186 @@ func TestAdd(t *testing.T) { require.Equal(t, 992, len(root.Links)) }) }) + + t.Run("ipfs add --hidden", func(t *testing.T) { + t.Parallel() + + // Helper to create test directory with hidden file + setupTestDir := func(t *testing.T, node *harness.Node) string { + testDir, err := os.MkdirTemp(node.Dir, "hidden-test") + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(testDir, "visible.txt"), []byte("visible"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(testDir, ".hidden"), []byte("hidden"), 0644)) + return testDir + } + + t.Run("default excludes hidden files", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + cidStr := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed() + lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed() + require.Contains(t, lsOutput, "visible.txt") + require.NotContains(t, lsOutput, ".hidden") + }) + + t.Run("--hidden includes hidden files", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + cidStr := node.IPFS("add", "-r", "-Q", "--hidden", testDir).Stdout.Trimmed() + lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed() + require.Contains(t, lsOutput, "visible.txt") + require.Contains(t, lsOutput, ".hidden") + }) + + t.Run("-H includes hidden files", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + cidStr := node.IPFS("add", "-r", "-Q", "-H", testDir).Stdout.Trimmed() + lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed() + require.Contains(t, lsOutput, "visible.txt") + require.Contains(t, lsOutput, ".hidden") + }) + }) + + t.Run("ipfs add --empty-dirs", func(t *testing.T) { + t.Parallel() + + // Helper to create test directory with empty subdirectory + setupTestDir := func(t *testing.T, node *harness.Node) string { + testDir, err := os.MkdirTemp(node.Dir, "empty-dirs-test") + require.NoError(t, err) + require.NoError(t, os.Mkdir(filepath.Join(testDir, "empty-subdir"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(testDir, "file.txt"), []byte("content"), 0644)) + return testDir + } + + t.Run("default includes empty directories", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + cidStr := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed() + require.Contains(t, node.IPFS("ls", cidStr).Stdout.Trimmed(), "empty-subdir") + }) + + t.Run("--empty-dirs=true includes empty directories", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + cidStr := node.IPFS("add", "-r", "-Q", "--empty-dirs=true", testDir).Stdout.Trimmed() + require.Contains(t, node.IPFS("ls", cidStr).Stdout.Trimmed(), "empty-subdir") + }) + + t.Run("--empty-dirs=false excludes empty directories", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + cidStr := node.IPFS("add", "-r", "-Q", "--empty-dirs=false", testDir).Stdout.Trimmed() + lsOutput := node.IPFS("ls", cidStr).Stdout.Trimmed() + require.NotContains(t, lsOutput, "empty-subdir") + require.Contains(t, lsOutput, "file.txt") + }) + }) + + t.Run("ipfs add --dereference-symlinks", func(t *testing.T) { + t.Parallel() + + // Helper to create test directory with a file and symlink to it + setupTestDir := func(t *testing.T, node *harness.Node) string { + testDir, err := os.MkdirTemp(node.Dir, "deref-symlinks-test") + require.NoError(t, err) + + targetFile := filepath.Join(testDir, "target.txt") + require.NoError(t, os.WriteFile(targetFile, []byte("target content"), 0644)) + + // Create symlink pointing to target + require.NoError(t, os.Symlink("target.txt", filepath.Join(testDir, "link.txt"))) + + return testDir + } + + t.Run("default preserves symlinks", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + + // Add directory with symlink (default: preserve) + dirCID := node.IPFS("add", "-r", "-Q", testDir).Stdout.Trimmed() + + // Get to a new directory and verify symlink is preserved + outDir, err := os.MkdirTemp(node.Dir, "symlink-get-out") + require.NoError(t, err) + node.IPFS("get", "-o", outDir, dirCID) + + // Check that link.txt is a symlink (ipfs get -o puts files directly in outDir) + linkPath := filepath.Join(outDir, "link.txt") + fi, err := os.Lstat(linkPath) + require.NoError(t, err) + require.True(t, fi.Mode()&os.ModeSymlink != 0, "link.txt should be a symlink") + + // Verify symlink target + target, err := os.Readlink(linkPath) + require.NoError(t, err) + require.Equal(t, "target.txt", target) + }) + + t.Run("--dereference-symlinks resolves nested symlinks", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + + // Add directory with dereference flag - nested symlinks should be resolved + dirCID := node.IPFS("add", "-r", "-Q", "--dereference-symlinks", testDir).Stdout.Trimmed() + + // Get and verify symlink was dereferenced to regular file + outDir, err := os.MkdirTemp(node.Dir, "symlink-get-out") + require.NoError(t, err) + node.IPFS("get", "-o", outDir, dirCID) + + linkPath := filepath.Join(outDir, "link.txt") + fi, err := os.Lstat(linkPath) + require.NoError(t, err) + + // Should be a regular file, not a symlink + require.False(t, fi.Mode()&os.ModeSymlink != 0, + "link.txt should be dereferenced to regular file, not preserved as symlink") + + // Content should match the target file + content, err := os.ReadFile(linkPath) + require.NoError(t, err) + require.Equal(t, "target content", string(content)) + }) + + t.Run("--dereference-args is deprecated", func(t *testing.T) { + t.Parallel() + node := harness.NewT(t).NewNode().Init().StartDaemon() + defer node.StopDaemon() + + testDir := setupTestDir(t, node) + + res := node.RunIPFS("add", "-Q", "--dereference-args", filepath.Join(testDir, "link.txt")) + require.Error(t, res.Err) + require.Contains(t, res.Stderr.String(), "--dereference-args is deprecated") + }) + }) } func TestAddFastProvide(t *testing.T) { diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 234086a25..7c9419af4 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -141,7 +141,7 @@ require ( github.com/ipfs/go-cid v0.6.0 // indirect github.com/ipfs/go-datastore v0.9.0 // indirect github.com/ipfs/go-dsqueue v0.1.1 // indirect - github.com/ipfs/go-ipfs-cmds v0.15.0 // indirect + github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.2 // indirect github.com/ipfs/go-ipld-cbor v0.2.1 // indirect github.com/ipfs/go-ipld-format v0.6.3 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index c99ca0aa4..ebf3395c1 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -312,8 +312,8 @@ github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-dsqueue v0.1.1 h1:6PQlHDyf9PSTN69NmwUir5+0is3tU0vRJj8zLlgK8Mc= github.com/ipfs/go-dsqueue v0.1.1/go.mod h1:Xxg353WSwwzYn3FGSzZ+taSQII3pIZ+EJC8/oWRDM10= -github.com/ipfs/go-ipfs-cmds v0.15.0 h1:nQDgKadrzyiFyYoZMARMIoVoSwe3gGTAfGvrWLeAQbQ= -github.com/ipfs/go-ipfs-cmds v0.15.0/go.mod h1:VABf/mv/wqvYX6hLG6Z+40eNAEw3FQO0bSm370Or3Wk= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294 h1:pQ6LhtU+nEBajAgFz3uU7ta6JN4KY0W5T7JxuaRQJVE= +github.com/ipfs/go-ipfs-cmds v0.15.1-0.20260117043932-17687e216294/go.mod h1:WG//DD2nimQcQ/+MTqB8mSeZQZBZC8KLZ+OeVGk9We0= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE=