coosld/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2026-02-21 18:37:45 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #5834 from ipfs/feat/urlstore-pin

Browse Source 
add pinning support to the urlstore
This commit is contained in:
Steven Allen 2018-12-12 10:39:17 -08:00 committed by GitHub
commit cb57105dfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 36 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
core/commands/urlstore.go
View File

@ -7,6 +7,7 @@ import (
cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
filestore "github.com/ipfs/go-ipfs/filestore"
pin "github.com/ipfs/go-ipfs/pin"
chunk "gx/ipfs/QmR4QQVkBZsZENRjYFVi8dEtPL3daZRNKk24m4r6WKJHNm/go-ipfs-chunker"
cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
@ -36,9 +37,6 @@ control.
The file is added using raw-leaves but otherwise using the default
settings for 'ipfs add'.
The file is not pinned, so this command should be followed by an 'ipfs
pin add'.
This command is considered temporary until a better solution can be
found. It may disappear or the semantics can change at any
time.
@ -46,6 +44,7 @@ time.
},
Options: []cmdkit.Option{
cmdkit.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
cmdkit.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true),
},
Arguments: []cmdkit.Argument{
cmdkit.StringArg("url", true, false, "URL to add to IPFS"),
@ -73,6 +72,7 @@ time.
}
useTrickledag, _ := req.Options[trickleOptionName].(bool)
dopin, _ := req.Options[pinOptionName].(bool)
hreq, err := http.NewRequest("GET", url, nil)
if err != nil {
@ -87,6 +87,11 @@ time.
return fmt.Errorf("expected code 200, got: %d", hres.StatusCode)
}
if dopin {
// Take the pinlock
defer n.Blockstore.PinLock().Unlock()
}
chk := chunk.NewSizeSplitter(hres.Body, chunk.DefaultBlockSize)
prefix := cid.NewPrefixV1(cid.DagProtobuf, mh.SHA2_256)
dbp := &ihelper.DagBuilderParams{
@ -102,13 +107,22 @@ time.
if useTrickledag {
layout = trickle.Layout
}
root, err := layout(dbp.New(chk))
if err != nil {
return err
}
c := root.Cid()
if dopin {
n.Pinning.PinWithMode(c, pin.Recursive)
if err := n.Pinning.Flush(); err != nil {
return err
}
}
return cmds.EmitOnce(res, &BlockStat{
Key: root.Cid().String(),
Key: c.String(),
Size: int(hres.ContentLength),
})
},

23
test/sharness/t0272-urlstore.sh
View File

@ -46,9 +46,8 @@ test_expect_success "enable urlstore" '
test_launch_ipfs_daemon --offline
test_expect_success "add files using gateway address via url store" '
HASH1=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a) &&
HASH2=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a) &&
ipfs pin add $HASH1 $HASH2
HASH1=$(ipfs urlstore add --pin=false http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a) &&
HASH2=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a)
'
test_expect_success "make sure hashes are different" '
@ -86,6 +85,21 @@ test_expect_success "ipfs filestore verify works with urls" '
test_cmp verify_expect verify_actual
'
test_expect_success "garbage collect file1 from the urlstore" '
ipfs repo gc > /dev/null
'
test_expect_success "can no longer retrieve file1 from urlstore" '
rm -f file1.actual &&
test_must_fail ipfs get $HASH1 -o file1.actual
'
test_expect_success "can still retrieve file2 from urlstore" '
rm -f file2.actual &&
ipfs get $HASH2 -o file2.actual &&
test_cmp file2 file2.actual
'
test_expect_success "remove original hashes from local gateway" '
ipfs pin rm $HASH1a $HASH2a &&
ipfs repo gc > /dev/null
@ -99,7 +113,6 @@ test_expect_success "gatway no longer has files" '
cat <<EOF | sort > verify_expect_2
error zb2rhX1q5oFFzEkPNsTe1Y8osUdFqSQGjUWRZsqC9fbY6WVSk 262144 http://127.0.0.1:$GWAY_PORT/ipfs/QmUow2T4P69nEsqTQDZCt8yg9CPS8GFmpuDAr5YtsPhTdM 0
error zb2rhYbKFn1UWGHXaAitcdVTkDGTykX8RFpGWzRFuLpoe9VE4 237856 http://127.0.0.1:$GWAY_PORT/ipfs/QmUow2T4P69nEsqTQDZCt8yg9CPS8GFmpuDAr5YtsPhTdM 262144
error zb2rhjddJ5DNzBrFu8G6CP1ApY25BukwCeskXHzN1H18CiVVZ 2222 http://127.0.0.1:$GWAY_PORT/ipfs/QmcHm3BL2cXuQ6rJdKQgPrmT9suqGkfy2KzH3MkXPEBXU6 0
EOF
test_expect_success "ipfs filestore verify is correct" '
@ -113,7 +126,7 @@ test_expect_success "files can not be retrieved via the urlstore" '
'
test_expect_success "remove broken files" '
ipfs pin rm $HASH1 $HASH2 &&
ipfs pin rm $HASH2 &&
ipfs repo gc > /dev/null
'