mirror of
https://github.com/ipfs/kubo.git
synced 2026-03-03 07:18:12 +08:00
feat: dag import --stats (#8237)
* feat: report block count on `dag import`
* fix: clean-up dag import message format
* Only print stats when --stats flag is passed
This applies to both text and json output encoding.
- Stats data is now contained within a Stats datastructure
- Stats are printed after root so that first line of output is the same as previously, even when stats are output using --stats
* fix sharness test
* Add PayloadBytesCount to stats
* Attempt to stabilize flaky tests
* Rename PayloadBytesCount to BlockBytesCount
* Correctly calculate size or imported dag
* Use RawSize of original block for import bytes calc
* test: dag import without --stats
basic regression tests for the default output (text and json)
Co-authored-by: gammazero <gammazero@users.noreply.github.com>
Co-authored-by: Marcin Rataj <lidel@lidel.org>
(cherry picked from commit 0057199af4)
This commit is contained in:
Rod Vagg
Adin Schmahmann
parent
f7fd3e57a1
commit
b3c3b2b92d
@ -16,9 +16,10 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
pinRootsOptionName = "pin-roots"
|
||||
progressOptionName = "progress"
|
||||
silentOptionName = "silent"
|
||||
pinRootsOptionName = "pin-roots"
|
||||
statsOptionName = "stats"
|
||||
)
|
||||
|
||||
// DagCmd provides a subset of commands for interacting with ipld dag objects
|
||||
@ -53,9 +54,15 @@ type ResolveOutput struct {
|
||||
RemPath string
|
||||
}
|
||||
|
||||
type CarImportStats struct {
|
||||
BlockCount uint64
|
||||
BlockBytesCount uint64
|
||||
}
|
||||
|
||||
// CarImportOutput is the output type of the 'dag import' commands
|
||||
type CarImportOutput struct {
|
||||
Root RootMeta
|
||||
Root *RootMeta `json:",omitempty"`
|
||||
Stats *CarImportStats `json:",omitempty"`
|
||||
}
|
||||
|
||||
// RootMeta is the metadata for a root pinning response
|
||||
@ -160,8 +167,10 @@ var DagResolveCmd = &cmds.Command{
|
||||
}
|
||||
|
||||
type importResult struct {
|
||||
roots map[cid.Cid]struct{}
|
||||
err error
|
||||
blockCount uint64
|
||||
blockBytesCount uint64
|
||||
roots map[cid.Cid]struct{}
|
||||
err error
|
||||
}
|
||||
|
||||
// DagImportCmd is a command for importing a car to ipfs
|
||||
@ -193,8 +202,9 @@ Maximum supported CAR version: 1
|
||||
cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(),
|
||||
},
|
||||
Options: []cmds.Option{
|
||||
cmds.BoolOption(silentOptionName, "No output."),
|
||||
cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
|
||||
cmds.BoolOption(silentOptionName, "No output."),
|
||||
cmds.BoolOption(statsOptionName, "Output stats."),
|
||||
},
|
||||
Type: CarImportOutput{},
|
||||
Run: dagImport,
|
||||
@ -206,6 +216,22 @@ Maximum supported CAR version: 1
|
||||
return nil
|
||||
}
|
||||
|
||||
// event should have only one of `Root` or `Stats` set, not both
|
||||
if event.Root == nil {
|
||||
if event.Stats == nil {
|
||||
return fmt.Errorf("Unexpected message from DAG import")
|
||||
}
|
||||
stats, _ := req.Options[statsOptionName].(bool)
|
||||
if stats {
|
||||
fmt.Fprintf(w, "Imported %d blocks (%d bytes)\n", event.Stats.BlockCount, event.Stats.BlockBytesCount)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if event.Stats != nil {
|
||||
return fmt.Errorf("Unexpected message from DAG import")
|
||||
}
|
||||
|
||||
enc, err := cmdenv.GetLowLevelCidEncoder(req)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@ -101,7 +101,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
|
||||
failedPins++
|
||||
}
|
||||
|
||||
if err := res.Emit(&CarImportOutput{Root: ret}); err != nil {
|
||||
if err := res.Emit(&CarImportOutput{Root: &ret}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@ -115,6 +115,19 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
|
||||
}
|
||||
}
|
||||
|
||||
stats, _ := req.Options[statsOptionName].(bool)
|
||||
if stats {
|
||||
err = res.Emit(&CarImportOutput{
|
||||
Stats: &CarImportStats{
|
||||
BlockCount: done.blockCount,
|
||||
BlockBytesCount: done.blockBytesCount,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -126,6 +139,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
|
||||
batch := ipld.NewBatch(req.Context, api.Dag())
|
||||
|
||||
roots := make(map[cid.Cid]struct{})
|
||||
var blockCount, blockBytesCount uint64
|
||||
|
||||
it := req.Files.Entries()
|
||||
for it.Next() {
|
||||
@ -176,6 +190,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
|
||||
if err := batch.Add(req.Context, nd); err != nil {
|
||||
return err
|
||||
}
|
||||
blockCount++
|
||||
blockBytesCount += uint64(len(block.RawData()))
|
||||
}
|
||||
|
||||
return nil
|
||||
@ -197,5 +213,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
|
||||
return
|
||||
}
|
||||
|
||||
ret <- importResult{roots: roots}
|
||||
ret <- importResult{
|
||||
blockCount: blockCount,
|
||||
blockBytesCount: blockBytesCount,
|
||||
roots: roots}
|
||||
}
|
||||
|
||||
@ -269,7 +269,7 @@ test_launch_ipfs_daemon() {
|
||||
|
||||
# wait for api file to show up
|
||||
test_expect_success "api file shows up" '
|
||||
test_wait_for_file 50 100ms "$IPFS_PATH/api"
|
||||
test_wait_for_file 50 200ms "$IPFS_PATH/api"
|
||||
'
|
||||
|
||||
test_set_address_vars actual_daemon
|
||||
|
||||
@ -43,7 +43,7 @@ test_expect_success "test ping 0" '
|
||||
'
|
||||
|
||||
test_expect_success "test ping offline" '
|
||||
iptb stop 1 &&
|
||||
iptb stop 1 && sleep 2 &&
|
||||
! ipfsi 0 ping -n2 -- "$PEERID_1"
|
||||
'
|
||||
|
||||
|
||||
@ -55,18 +55,25 @@ run_online_imp_exp_tests() {
|
||||
reset_blockstore 0
|
||||
reset_blockstore 1
|
||||
|
||||
cat > basic_import_expected <<EOE
|
||||
cat > basic_import_stats_expected <<EOE
|
||||
Imported 1198 blocks (468513 bytes)
|
||||
Pinned root${tab}bafkqaaa${tab}success
|
||||
Pinned root${tab}bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u${tab}success
|
||||
Pinned root${tab}bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy${tab}success
|
||||
EOE
|
||||
# output without the --stats line at the top
|
||||
tail -n +2 basic_import_stats_expected > basic_import_expected
|
||||
|
||||
# Explainer:
|
||||
# naked_root_import_json_expected output is produced by dag import of combined_naked_roots_genesis_and_128.car
|
||||
# executed when roots are already present in the repo - thus the BlockCount=0
|
||||
# (if blocks were not present in the repo, blockstore: block not found would be returned)
|
||||
cat >naked_root_import_json_expected <<EOE
|
||||
{"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}}
|
||||
{"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}}
|
||||
{"Stats":{"BlockCount":0,"BlockBytesCount":0}}
|
||||
EOE
|
||||
|
||||
|
||||
test_expect_success "basic import" '
|
||||
do_import 0 \
|
||||
../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
|
||||
@ -79,6 +86,18 @@ EOE
|
||||
test_cmp_sorted basic_import_expected basic_import_actual
|
||||
'
|
||||
|
||||
test_expect_success "basic import with --stats" '
|
||||
do_import 0 --stats \
|
||||
../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
|
||||
../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car \
|
||||
../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car \
|
||||
> basic_import_actual
|
||||
'
|
||||
|
||||
test_expect_success "basic import output with --stats as expected" '
|
||||
test_cmp_sorted basic_import_stats_expected basic_import_actual
|
||||
'
|
||||
|
||||
test_expect_success "basic fetch+export 1" '
|
||||
ipfsi 1 dag export bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy > reexported_testnet_128.car
|
||||
'
|
||||
@ -98,7 +117,7 @@ EOE
|
||||
'
|
||||
|
||||
test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" '
|
||||
ipfsi 1 dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
|
||||
ipfsi 1 dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
|
||||
> naked_import_result_json_actual
|
||||
'
|
||||
|
||||
@ -117,7 +136,7 @@ EOE
|
||||
cat ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car > pipe_testnet &
|
||||
cat ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car > pipe_devnet &
|
||||
|
||||
do_import 0 \
|
||||
do_import 0 --stats \
|
||||
pipe_testnet \
|
||||
pipe_devnet \
|
||||
../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
|
||||
@ -134,7 +153,7 @@ EOE
|
||||
'
|
||||
|
||||
test_expect_success "fifo-import output as expected" '
|
||||
test_cmp_sorted basic_import_expected basic_fifo_import_actual
|
||||
test_cmp_sorted basic_import_stats_expected basic_fifo_import_actual
|
||||
'
|
||||
}
|
||||
|
||||
@ -168,33 +187,46 @@ test_expect_success "correct error" '
|
||||
test_cmp_sorted offline_fetch_error_expected offline_fetch_error_actual
|
||||
'
|
||||
|
||||
|
||||
cat >multiroot_import_json_expected <<EOE
|
||||
cat >multiroot_import_json_stats_expected <<EOE
|
||||
{"Root":{"Cid":{"/":"bafy2bzaceb55n7uxyfaelplulk3ev2xz7gnq6crncf3ahnvu46hqqmpucizcw"},"PinErrorMsg":""}}
|
||||
{"Root":{"Cid":{"/":"bafy2bzacebedrc4n2ac6cqdkhs7lmj5e4xiif3gu7nmoborihajxn3fav3vdq"},"PinErrorMsg":""}}
|
||||
{"Root":{"Cid":{"/":"bafy2bzacede2hsme6hparlbr4g2x6pylj43olp4uihwjq3plqdjyrdhrv7cp4"},"PinErrorMsg":""}}
|
||||
{"Stats":{"BlockCount":2825,"BlockBytesCount":1339709}}
|
||||
EOE
|
||||
test_expect_success "multiroot import works" '
|
||||
# output without --stats line
|
||||
head -3 multiroot_import_json_stats_expected > multiroot_import_json_expected
|
||||
|
||||
test_expect_success "multiroot import works (--enc=json)" '
|
||||
ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
|
||||
'
|
||||
test_expect_success "multiroot import expected output" '
|
||||
test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual
|
||||
'
|
||||
|
||||
test_expect_success "multiroot import works with --stats" '
|
||||
ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
|
||||
'
|
||||
test_expect_success "multiroot import expected output" '
|
||||
test_cmp_sorted multiroot_import_json_stats_expected multiroot_import_json_actual
|
||||
'
|
||||
|
||||
|
||||
cat >pin_import_expected << EOE
|
||||
{"Stats":{"BlockCount":1198,"BlockBytesCount":468513}}
|
||||
EOE
|
||||
test_expect_success "pin-less import works" '
|
||||
ipfs dag import --enc=json --pin-roots=false \
|
||||
ipfs dag import --stats --enc=json --pin-roots=false \
|
||||
../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \
|
||||
../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \
|
||||
> no-pin_import_actual
|
||||
'
|
||||
test_expect_success "expected silence on --pin-roots=false" '
|
||||
test_cmp /dev/null no-pin_import_actual
|
||||
test_expect_success "expected no pins on --pin-roots=false" '
|
||||
test_cmp pin_import_expected no-pin_import_actual
|
||||
'
|
||||
|
||||
|
||||
test_expect_success "naked root import works" '
|
||||
ipfs dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
|
||||
ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
|
||||
> naked_root_import_json_actual
|
||||
'
|
||||
test_expect_success "naked root import expected output" '
|
||||
|
||||
Loading…
Reference in New Issue
Block a user