mirror of
https://github.com/ipfs/kubo.git
synced 2026-02-27 21:37:57 +08:00
include pebble as built-in plugin
Pebble provides a high-performance alternative to leveldb as the datastore, and will serve as a replacement for badger1. There are a number of tuning parameters available for tuning pebble's performance to your specific needs. Default values are used for any that are not configured or are set to the parameter's zero-value. Requires https://github.com/ipfs/go-ds-pebble/pull/39 Closes #10347
This commit is contained in:
gammazero
parent
a17830754c
commit
584c70f50b
@ -138,6 +138,28 @@ func DefaultDatastoreConfig() Datastore {
|
||||
}
|
||||
}
|
||||
|
||||
func pebbleSpec() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": "measure",
|
||||
"prefix": "pebble.datastore",
|
||||
"child": map[string]interface{}{
|
||||
"type": "pebbleds",
|
||||
"path": "pebbleds",
|
||||
"bytesPerSync": 0,
|
||||
"bisableWAL": false,
|
||||
"cacheSize": 0,
|
||||
"l0CompactionThreshold": 0,
|
||||
"l0StopWritesThreshold": 0,
|
||||
"lBaseMaxBytes": 0,
|
||||
"maxConcurrentCompactions": 0,
|
||||
"memTableSize": 0,
|
||||
"memTableStopWritesThreshold": 0,
|
||||
"walBytesPerSync": 0,
|
||||
"walMinSyncSeconds": 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func badgerSpec() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": "measure",
|
||||
|
||||
@ -144,6 +144,27 @@ This profile may only be applied when first initializing the node.
|
||||
return nil
|
||||
},
|
||||
},
|
||||
"pebbleds": {
|
||||
Description: `Configures the node to use the pebble high-performance datastore.
|
||||
|
||||
Pebble is a LevelDB/RocksDB inspired key-value store focused on performance
|
||||
and internal usage by CockroachDB.
|
||||
You should use this datastore if:
|
||||
|
||||
- You need a datastore that is focused on performance.
|
||||
- You need reliability by default, but may choose to disable WAL for maximum performance when reliability is not critical.
|
||||
- This datastore is good for multi-terrabyte data sets.
|
||||
- May benefit from tuning depeending on read/write patterns and throughput.
|
||||
- Performance is helped significnatly by running on a system with plenty of memory.
|
||||
|
||||
This profile may only be applied when first initializing the node.`,
|
||||
|
||||
InitOnly: true,
|
||||
Transform: func(c *Config) error {
|
||||
c.Datastore.Spec = pebbleSpec()
|
||||
return nil
|
||||
},
|
||||
},
|
||||
"badgerds": {
|
||||
Description: `Configures the node to use the legacy badgerv1 datastore.
|
||||
|
||||
|
||||
@ -181,6 +181,7 @@ config file at runtime.
|
||||
- [`local-discovery` profile](#local-discovery-profile)
|
||||
- [`default-networking` profile](#default-networking-profile)
|
||||
- [`flatfs` profile](#flatfs-profile)
|
||||
- [`pebbleds` profile](#pebbleds-profile)
|
||||
- [`badgerds` profile](#badgerds-profile)
|
||||
- [`lowpower` profile](#lowpower-profile)
|
||||
- [`legacy-cid-v0` profile](#legacy-cid-v0-profile)
|
||||
@ -2416,6 +2417,21 @@ You should use this datastore if:
|
||||
|
||||
This profile may only be applied when first initializing the node.
|
||||
|
||||
### `pebbleds` profile
|
||||
|
||||
Configures the node to use the pebble high-performance datastore.
|
||||
|
||||
Pebble is a LevelDB/RocksDB inspired key-value store focused on performance and internal usage by CockroachDB.
|
||||
You should use this datastore if:
|
||||
|
||||
- You need a datastore that is focused on performance.
|
||||
- You need reliability by default, but may choose to disable WAL for maximum performance when reliability is not critical.
|
||||
- This datastore is good for multi-terrabyte data sets.
|
||||
- May benefit from tuning depeending on read/write patterns and throughput.
|
||||
- Performance is helped significnatly by running on a system with plenty of memory.
|
||||
|
||||
This profile may only be applied when first initializing the node.
|
||||
|
||||
### `badgerds` profile
|
||||
|
||||
Configures the node to use the legacy badgerv1 datastore.
|
||||
|
||||
@ -35,6 +35,32 @@ Uses a leveldb database to store key value pairs.
|
||||
}
|
||||
```
|
||||
|
||||
## pebbleds
|
||||
|
||||
Uses [pebble](https://github.com/cockroachdb/pebble) as a key value store.
|
||||
|
||||
#### The following options are availble for tuning pebble. If they are not configured (or assigned their zero-valued), then default values are used. Start using pebble with only default values and configure tuning items are needed for your needs. For a more complete description of these values, see: https://pkg.go.dev/github.com/cockroachdb/pebble@v1.1.2#Options
|
||||
|
||||
* `bytesPerSync`: int, Sync sstables periodically in order to smooth out writes to disk. (default: 512KB)
|
||||
* `bisableWAL`: true|false, Disable the write-ahead log (WAL) at expense of prohibiting crash recovery. (default: false)
|
||||
* `cacheSize`: Size of pebble's shared block cache. (default: 8MB)
|
||||
* `l0CompactionThreshold`: int, Count of L0 files necessary to trigger an L0 compaction.
|
||||
* `l0StopWritesThreshold`: int, Limit on L0 read-amplification, computed as the number of L0 sublevels.
|
||||
* `lBaseMaxBytes`: int, Maximum number of bytes for LBase. The base level is the level which L0 is compacted into.
|
||||
* `maxConcurrentCompactions`: int, Maximum number of concurrent compactions. (default: 1)
|
||||
* `memTableSize`: int, Size of a MemTable in steady state. The actual MemTable size starts at min(256KB, MemTableSize) and doubles for each subsequent MemTable up to MemTableSize (default: 4MB)
|
||||
* `memTableStopWritesThreshold`: int, Limit on the number of queued of MemTables. (default: 2)
|
||||
* `walBytesPerSync`: int: Sets the number of bytes to write to a WAL before calling Sync on it in the background. (default: 0, no background syncing)
|
||||
* `walMinSyncSeconds`: int: Sets the minimum duration between syncs of the WAL. (default: 0)
|
||||
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "pebbleds",
|
||||
"path": "<location of pebble inside repo>",
|
||||
}
|
||||
```
|
||||
|
||||
## badgerds
|
||||
|
||||
Uses [badger](https://github.com/dgraph-io/badger) as a key value store.
|
||||
|
||||
14
go.mod
14
go.mod
@ -8,6 +8,7 @@ require (
|
||||
github.com/cenkalti/backoff/v4 v4.3.0
|
||||
github.com/ceramicnetwork/go-dag-jose v0.1.0
|
||||
github.com/cheggaaa/pb v1.0.29
|
||||
github.com/cockroachdb/pebble v1.1.2
|
||||
github.com/coreos/go-systemd/v22 v22.5.0
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302
|
||||
@ -28,6 +29,7 @@ require (
|
||||
github.com/ipfs/go-ds-flatfs v0.5.1
|
||||
github.com/ipfs/go-ds-leveldb v0.5.0
|
||||
github.com/ipfs/go-ds-measure v0.2.0
|
||||
github.com/ipfs/go-ds-pebble v0.3.2-0.20241002075519-c174835dc84a
|
||||
github.com/ipfs/go-fs-lock v0.0.7
|
||||
github.com/ipfs/go-ipfs-cmds v0.13.0
|
||||
github.com/ipfs/go-ipld-cbor v0.1.0
|
||||
@ -93,12 +95,18 @@ require (
|
||||
|
||||
require (
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
|
||||
github.com/DataDog/zstd v1.4.5 // indirect
|
||||
github.com/Jorropo/jsync v1.0.1 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect
|
||||
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash v1.1.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cockroachdb/errors v1.11.3 // indirect
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
|
||||
github.com/cockroachdb/redact v1.1.5 // indirect
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
|
||||
github.com/containerd/cgroups v1.1.0 // indirect
|
||||
github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
|
||||
github.com/cskr/pubsub v1.0.2 // indirect
|
||||
@ -113,6 +121,7 @@ require (
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/francoispqt/gojay v1.2.13 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.4 // indirect
|
||||
github.com/getsentry/sentry-go v0.27.0 // indirect
|
||||
github.com/go-kit/log v0.2.1 // indirect
|
||||
github.com/go-logfmt/logfmt v0.5.1 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
@ -149,6 +158,8 @@ require (
|
||||
github.com/klauspost/compress v1.17.9 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/koron/go-ssdp v0.0.4 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
|
||||
github.com/libp2p/go-cidranger v1.1.0 // indirect
|
||||
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
|
||||
@ -162,7 +173,7 @@ require (
|
||||
github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
|
||||
github.com/libp2p/zeroconf/v2 v2.2.0 // indirect
|
||||
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
|
||||
github.com/mattn/go-colorable v0.1.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.4 // indirect
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
|
||||
@ -208,6 +219,7 @@ require (
|
||||
github.com/quic-go/quic-go v0.45.2 // indirect
|
||||
github.com/quic-go/webtransport-go v0.8.0 // indirect
|
||||
github.com/raulk/go-watchdog v1.3.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.12.0 // indirect
|
||||
github.com/rs/cors v1.10.1 // indirect
|
||||
github.com/samber/lo v1.46.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
|
||||
33
go.sum
33
go.sum
@ -47,6 +47,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
|
||||
github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
|
||||
github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU=
|
||||
github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
@ -104,6 +106,20 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
|
||||
github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
|
||||
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
|
||||
github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
|
||||
github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA=
|
||||
github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
|
||||
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
|
||||
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
|
||||
github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
|
||||
github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
|
||||
github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
|
||||
@ -177,10 +193,14 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
|
||||
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
||||
github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I=
|
||||
github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s=
|
||||
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
|
||||
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
@ -365,6 +385,8 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN
|
||||
github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q=
|
||||
github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ=
|
||||
github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE=
|
||||
github.com/ipfs/go-ds-pebble v0.3.2-0.20241002075519-c174835dc84a h1:OouXE/TSvgFP3Scv/3WA1brxKC+FM40EeusWnLGilgY=
|
||||
github.com/ipfs/go-ds-pebble v0.3.2-0.20241002075519-c174835dc84a/go.mod h1:ZyYU+weIni+4NG/Yjva+cPkU3ghlsU1HA2R/VLHJ9sM=
|
||||
github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U=
|
||||
github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc=
|
||||
github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ=
|
||||
@ -558,15 +580,15 @@ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
|
||||
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
|
||||
@ -683,6 +705,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
|
||||
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo=
|
||||
github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI=
|
||||
github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
|
||||
@ -725,6 +749,7 @@ github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
|
||||
github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
|
||||
github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I=
|
||||
github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
@ -783,6 +808,7 @@ github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtB
|
||||
github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||
github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo=
|
||||
@ -1198,6 +1224,7 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
pluginipldgit "github.com/ipfs/kubo/plugin/plugins/git"
|
||||
pluginlevelds "github.com/ipfs/kubo/plugin/plugins/levelds"
|
||||
pluginnopfs "github.com/ipfs/kubo/plugin/plugins/nopfs"
|
||||
pluginpebbleds "github.com/ipfs/kubo/plugin/plugins/pebbleds"
|
||||
pluginpeerlog "github.com/ipfs/kubo/plugin/plugins/peerlog"
|
||||
)
|
||||
|
||||
@ -21,6 +22,7 @@ func init() {
|
||||
Preload(pluginbadgerds.Plugins...)
|
||||
Preload(pluginflatfs.Plugins...)
|
||||
Preload(pluginlevelds.Plugins...)
|
||||
Preload(pluginpebbleds.Plugins...)
|
||||
Preload(pluginpeerlog.Plugins...)
|
||||
Preload(pluginfxtest.Plugins...)
|
||||
Preload(pluginnopfs.Plugins...)
|
||||
|
||||
@ -9,6 +9,7 @@ iplddagjose github.com/ipfs/kubo/plugin/plugins/dagjose *
|
||||
badgerds github.com/ipfs/kubo/plugin/plugins/badgerds *
|
||||
flatfs github.com/ipfs/kubo/plugin/plugins/flatfs *
|
||||
levelds github.com/ipfs/kubo/plugin/plugins/levelds *
|
||||
pebbleds github.com/ipfs/kubo/plugin/plugins/pebbleds *
|
||||
peerlog github.com/ipfs/kubo/plugin/plugins/peerlog *
|
||||
fxtest github.com/ipfs/kubo/plugin/plugins/fxtest *
|
||||
nopfs github.com/ipfs/kubo/plugin/plugins/nopfs *
|
||||
nopfs github.com/ipfs/kubo/plugin/plugins/nopfs *
|
||||
|
||||
@ -42,7 +42,7 @@ type datastoreConfig struct {
|
||||
syncField bool
|
||||
}
|
||||
|
||||
// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
|
||||
// BadgerdsDatastoreConfig returns a configuration stub for a flatfs datastore
|
||||
// from the given parameters.
|
||||
func (*flatfsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
|
||||
return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
|
||||
|
||||
@ -42,7 +42,7 @@ type datastoreConfig struct {
|
||||
compression ldbopts.Compression
|
||||
}
|
||||
|
||||
// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
|
||||
// DatastoreConfigParser returns a configuration stub for a badger datastore
|
||||
// from the given parameters.
|
||||
func (*leveldsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
|
||||
return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
|
||||
|
||||
258
plugin/plugins/pebbleds/pebbleds.go
Normal file
258
plugin/plugins/pebbleds/pebbleds.go
Normal file
@ -0,0 +1,258 @@
|
||||
package pebbleds
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/cockroachdb/pebble"
|
||||
pebbleds "github.com/ipfs/go-ds-pebble"
|
||||
"github.com/ipfs/kubo/plugin"
|
||||
"github.com/ipfs/kubo/repo"
|
||||
"github.com/ipfs/kubo/repo/fsrepo"
|
||||
)
|
||||
|
||||
// Plugins is exported list of plugins that will be loaded.
|
||||
var Plugins = []plugin.Plugin{
|
||||
&pebbledsPlugin{},
|
||||
}
|
||||
|
||||
type pebbledsPlugin struct{}
|
||||
|
||||
var _ plugin.PluginDatastore = (*pebbledsPlugin)(nil)
|
||||
|
||||
func (*pebbledsPlugin) Name() string {
|
||||
return "ds-pebble"
|
||||
}
|
||||
|
||||
func (*pebbledsPlugin) Version() string {
|
||||
return "0.1.0"
|
||||
}
|
||||
|
||||
func (*pebbledsPlugin) Init(_ *plugin.Environment) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (*pebbledsPlugin) DatastoreTypeName() string {
|
||||
return "pebbleds"
|
||||
}
|
||||
|
||||
type datastoreConfig struct {
|
||||
path string
|
||||
cacheSize int64
|
||||
|
||||
// Documentation of these values: https://pkg.go.dev/github.com/cockroachdb/pebble@v1.1.2#Options
|
||||
pebbleOpts *pebble.Options
|
||||
}
|
||||
|
||||
// PebbleDatastoreConfig returns a configuration stub for a pebble datastore
|
||||
// from the given parameters.
|
||||
func (*pebbledsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
|
||||
return func(params map[string]any) (fsrepo.DatastoreConfig, error) {
|
||||
var c datastoreConfig
|
||||
var ok bool
|
||||
|
||||
c.path, ok = params["path"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("'path' field is missing or not string")
|
||||
}
|
||||
|
||||
cacheSize, err := getConfigInt("cacheSize", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.cacheSize = int64(cacheSize)
|
||||
|
||||
bytesPerSync, err := getConfigInt("bytesPerSync", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
disableWAL, err := getConfigBool("disableWAL", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l0CompactionThreshold, err := getConfigInt("l0CompactionThreshold", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l0StopWritesThreshold, err := getConfigInt("l0StopWritesThreshold", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lBaseMaxBytes, err := getConfigInt("lBaseMaxBytes", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
maxConcurrentCompactions, err := getConfigInt("maxConcurrentCompactions", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
memTableSize, err := getConfigInt("memTableSize", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
memTableStopWritesThreshold, err := getConfigInt("memTableStopWritesThreshold", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
walBytesPerSync, err := getConfigInt("walBytesPerSync", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
walMinSyncSec, err := getConfigInt("walMinSyncIntervalSeconds", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if bytesPerSync != 0 || disableWAL || l0CompactionThreshold != 0 || l0StopWritesThreshold != 0 || lBaseMaxBytes != 0 || maxConcurrentCompactions != 0 || memTableSize != 0 || memTableStopWritesThreshold != 0 || walBytesPerSync != 0 || walMinSyncSec != 0 {
|
||||
c.pebbleOpts = &pebble.Options{
|
||||
//BytesPerSync: bytesPerSyn,
|
||||
DisableWAL: disableWAL,
|
||||
L0CompactionThreshold: l0CompactionThreshold,
|
||||
L0StopWritesThreshold: l0StopWritesThreshold,
|
||||
LBaseMaxBytes: int64(lBaseMaxBytes),
|
||||
MemTableSize: uint64(memTableSize),
|
||||
MemTableStopWritesThreshold: memTableStopWritesThreshold,
|
||||
WALBytesPerSync: walBytesPerSync,
|
||||
}
|
||||
if maxConcurrentCompactions != 0 {
|
||||
c.pebbleOpts.MaxConcurrentCompactions = func() int { return maxConcurrentCompactions }
|
||||
}
|
||||
if walMinSyncSec != 0 {
|
||||
c.pebbleOpts.WALMinSyncInterval = func() time.Duration { return time.Duration(walMinSyncSec) * time.Second }
|
||||
}
|
||||
}
|
||||
|
||||
return &c, nil
|
||||
}
|
||||
}
|
||||
|
||||
func getConfigBool(name string, params map[string]any) (bool, error) {
|
||||
val, ok := params[name]
|
||||
if ok {
|
||||
bval, ok := val.(bool)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("%q field was not a bool", name)
|
||||
}
|
||||
return bval, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func getConfigString(name string, params map[string]any) (string, error) {
|
||||
val, ok := params[name]
|
||||
if ok {
|
||||
sval, ok := val.(string)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%q field was not a string", name)
|
||||
}
|
||||
return sval, nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func getConfigInt(name string, params map[string]any) (int, error) {
|
||||
val, ok := params[name]
|
||||
if ok {
|
||||
// TODO: see why val may be an int or a float64.
|
||||
ival, ok := val.(int)
|
||||
if !ok {
|
||||
fval, ok := val.(float64)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("%q field was not an integer or a float64", name)
|
||||
}
|
||||
return int(fval), nil
|
||||
}
|
||||
return ival, nil
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec {
|
||||
return map[string]interface{}{
|
||||
"type": "pebbleds",
|
||||
"path": c.path,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *datastoreConfig) Create(path string) (repo.Datastore, error) {
|
||||
p := c.path
|
||||
if !filepath.IsAbs(p) {
|
||||
p = filepath.Join(path, p)
|
||||
}
|
||||
|
||||
if err := dirWritable(p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return pebbleds.NewDatastore(p, pebbleds.WithCacheSize(c.cacheSize), pebbleds.WithPebbleOpts(c.pebbleOpts))
|
||||
}
|
||||
|
||||
// dirWritable checks if a directory is writable. If the directory does
|
||||
// not exist it is created with writable permission.
|
||||
func dirWritable(dir string) error {
|
||||
if dir == "" {
|
||||
return errors.New("directory not specified")
|
||||
}
|
||||
var err error
|
||||
dir, err = expandHome(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fi, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// Directory does not exist, so create it.
|
||||
err = os.Mkdir(dir, 0775)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if errors.Is(err, fs.ErrPermission) {
|
||||
err = fs.ErrPermission
|
||||
}
|
||||
return fmt.Errorf("directory not writable: %s: %w", dir, err)
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return fmt.Errorf("not a directory: %s", dir)
|
||||
}
|
||||
|
||||
// Directory exists, check that a file can be written.
|
||||
file, err := os.CreateTemp(dir, "writetest")
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrPermission) {
|
||||
err = fs.ErrPermission
|
||||
}
|
||||
return fmt.Errorf("directory not writable: %s: %w", dir, err)
|
||||
}
|
||||
file.Close()
|
||||
return os.Remove(file.Name())
|
||||
}
|
||||
|
||||
// expandHome expands the path to include the home directory if the path is
|
||||
// prefixed with `~`. If it isn't prefixed with `~`, the path is returned
|
||||
// as-is.
|
||||
func expandHome(path string) (string, error) {
|
||||
if path == "" {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
if path[0] != '~' {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
if len(path) > 1 && path[1] != '/' && path[1] != '\\' {
|
||||
return "", errors.New("cannot expand user-specific home dir")
|
||||
}
|
||||
|
||||
dir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return filepath.Join(dir, path[1:]), nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user