From 8a88718ea4dc47bbb995345b9c666ff3d4d57ded Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 13 Dec 2021 16:52:07 -0500 Subject: [PATCH 1/7] Release v0.12.0-rc1 --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index e6aa56b97..533657815 100644 --- a/version.go +++ b/version.go @@ -4,7 +4,7 @@ package ipfs var CurrentCommit string // CurrentVersionNumber is the current application's version literal -const CurrentVersionNumber = "0.12.0-dev" +const CurrentVersionNumber = "0.12.0-rc1" const ApiVersion = "/go-ipfs/" + CurrentVersionNumber + "/" From db85387c6736a243b3921e81b2320c8f82d4dbe7 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 12 Jan 2022 19:40:21 +0100 Subject: [PATCH 2/7] test: restore ipfs-interop (#8613) * chore: ipfs@0.61.0 https://github.com/ipfs/js-ipfs/blob/master/packages/ipfs/CHANGELOG.md#0610-2021-12-15 * chore: ipfs-interop@8.0.0 (cherry picked from commit 14046d000276c5f1c8b1aed50634f18cd2ef23d7) --- .circleci/main.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.circleci/main.yml b/.circleci/main.yml index 7e197701a..19e6f0573 100644 --- a/.circleci/main.yml +++ b/.circleci/main.yml @@ -227,22 +227,13 @@ jobs: name: Installing dependencies command: | npm init -y - npm install ipfs@^0.59.1 - npm install ipfs/interop#master + npm install ipfs@^0.61.0 + npm install ipfs-interop@^8.0.0 npm install mocha-circleci-reporter@0.0.3 working_directory: ~/ipfs/go-ipfs/interop - run: name: Running tests command: | - WORKDIR=$(pwd) - cd /tmp - git clone https://github.com/ipfs/js-ipfs.git - cd js-ipfs - git checkout 1dcac76f56972fc3519526e93567e39d685033dd - npm install - npm run build - npm run link - cd $WORKDIR mkdir -p /tmp/test-results/interop/ export MOCHA_FILE="$(mktemp /tmp/test-results/interop/unit.XXXXXX.xml)" npx ipfs-interop -- -t node -f $(sed -n -e "s|^require('\(.*\)')$|test/\1|p" node_modules/ipfs-interop/test/node.js | circleci tests split) -- --reporter mocha-circleci-reporter @@ -251,9 +242,6 @@ jobs: LIBP2P_TCP_REUSEPORT: false LIBP2P_ALLOW_WEAK_RSA_KEYS: 1 IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs - IPFS_JS_EXEC: /tmp/js-ipfs/packages/ipfs/src/cli.js - IPFS_JS_MODULE: /tmp/js-ipfs/packages/ipfs/dist/cjs/src/index.js - IPFS_JS_HTTP_MODULE: /tmp/js-ipfs/packages/ipfs-http-client/dist/cjs/src/index.js - store_test_results: path: /tmp/test-results go-ipfs-api: From 418bc6e709b2fc86bb6b2dee264aea013d5bdce7 Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Thu, 10 Feb 2022 17:48:38 -0500 Subject: [PATCH 3/7] feat: log multifetcher errors This is to make it easier to understand why the multifetcher is falling back to a different fetcher. (cherry picked from commit 4f3eb4cdd4bc8d7dcec54a37e7f888cce7c01cc9) --- repo/fsrepo/migrations/fetcher.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/repo/fsrepo/migrations/fetcher.go b/repo/fsrepo/migrations/fetcher.go index 26a9275de..06794236e 100644 --- a/repo/fsrepo/migrations/fetcher.go +++ b/repo/fsrepo/migrations/fetcher.go @@ -2,6 +2,7 @@ package migrations import ( "context" + "fmt" "io" "os" @@ -39,7 +40,8 @@ type limitReadCloser struct { // NewMultiFetcher creates a MultiFetcher with the given Fetchers. The // Fetchers are tried in order ther passed to this function. -func NewMultiFetcher(f ...Fetcher) Fetcher { +func NewMultiFetcher(f ...Fetcher) *MultiFetcher { + mf := &MultiFetcher{ fetchers: make([]Fetcher, len(f)), } @@ -56,6 +58,7 @@ func (f *MultiFetcher) Fetch(ctx context.Context, ipfsPath string) (io.ReadClose if err == nil { return rc, nil } + fmt.Printf("Error fetching: %s\n", err.Error()) errs = multierror.Append(errs, err) } return nil, errs From bf76ebe6671a04fc1399346695a33fee31f759d3 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 11 Feb 2022 14:57:51 -0500 Subject: [PATCH 4/7] feat: refactor Fetcher interface used for downloading migrations (#8728) * feat: refactor Fetcher interface used for downloading migrations * feat: add RetryFetcher for migration downloads * feat: 3 retries for each HTTP migration download (cherry picked from commit b1ffc870d5f2fe167458918311b06ecf795d3422) --- repo/fsrepo/migrations/fetch.go | 7 ++-- repo/fsrepo/migrations/fetch_test.go | 14 +++----- repo/fsrepo/migrations/fetcher.go | 10 +++--- repo/fsrepo/migrations/httpfetcher.go | 14 +++++--- .../migrations/ipfsfetcher/ipfsfetcher.go | 16 ++++++--- .../ipfsfetcher/ipfsfetcher_test.go | 9 +++-- repo/fsrepo/migrations/migrations.go | 5 +-- repo/fsrepo/migrations/migrations_test.go | 13 +++++--- repo/fsrepo/migrations/retryfetcher.go | 33 +++++++++++++++++++ repo/fsrepo/migrations/versions.go | 6 ++-- 10 files changed, 83 insertions(+), 44 deletions(-) create mode 100644 repo/fsrepo/migrations/retryfetcher.go diff --git a/repo/fsrepo/migrations/fetch.go b/repo/fsrepo/migrations/fetch.go index a3493e750..ff202088a 100644 --- a/repo/fsrepo/migrations/fetch.go +++ b/repo/fsrepo/migrations/fetch.go @@ -111,15 +111,14 @@ func FetchBinary(ctx context.Context, fetcher Fetcher, dist, ver, binName, out s } defer arcFile.Close() - // Open connection to download archive from ipfs path - rc, err := fetcher.Fetch(ctx, arcDistPath) + // Open connection to download archive from ipfs path and write to file + arcBytes, err := fetcher.Fetch(ctx, arcDistPath) if err != nil { return "", err } - defer rc.Close() // Write download data - _, err = io.Copy(arcFile, rc) + _, err = io.Copy(arcFile, bytes.NewReader(arcBytes)) if err != nil { return "", err } diff --git a/repo/fsrepo/migrations/fetch_test.go b/repo/fsrepo/migrations/fetch_test.go index ec7c6d5e7..2273cb5e9 100644 --- a/repo/fsrepo/migrations/fetch_test.go +++ b/repo/fsrepo/migrations/fetch_test.go @@ -2,10 +2,10 @@ package migrations import ( "bufio" + "bytes" "context" "fmt" "io" - "io/ioutil" "net/http" "net/http/httptest" "os" @@ -96,14 +96,13 @@ func TestHttpFetch(t *testing.T) { fetcher := NewHttpFetcher("", ts.URL, "", 0) - rc, err := fetcher.Fetch(ctx, "/versions") + out, err := fetcher.Fetch(ctx, "/versions") if err != nil { t.Fatal(err) } - defer rc.Close() var lines []string - scan := bufio.NewScanner(rc) + scan := bufio.NewScanner(bytes.NewReader(out)) for scan.Scan() { lines = append(lines, scan.Text()) } @@ -232,16 +231,11 @@ func TestMultiFetcher(t *testing.T) { mf := NewMultiFetcher(badFetcher, fetcher) - rc, err := mf.Fetch(ctx, "/versions") + vers, err := mf.Fetch(ctx, "/versions") if err != nil { t.Fatal(err) } - defer rc.Close() - vers, err := ioutil.ReadAll(rc) - if err != nil { - t.Fatal("could not read versions:", err) - } if len(vers) < 45 { fmt.Println("unexpected more data") } diff --git a/repo/fsrepo/migrations/fetcher.go b/repo/fsrepo/migrations/fetcher.go index 06794236e..a63ba7276 100644 --- a/repo/fsrepo/migrations/fetcher.go +++ b/repo/fsrepo/migrations/fetcher.go @@ -21,8 +21,7 @@ const ( type Fetcher interface { // Fetch attempts to fetch the file at the given ipfs path. - // Returns io.ReadCloser on success, which caller must close. - Fetch(ctx context.Context, filePath string) (io.ReadCloser, error) + Fetch(ctx context.Context, filePath string) ([]byte, error) // Close performs any cleanup after the fetcher is not longer needed. Close() error } @@ -50,13 +49,12 @@ func NewMultiFetcher(f ...Fetcher) *MultiFetcher { } // Fetch attempts to fetch the file at each of its fetchers until one succeeds. -// Returns io.ReadCloser on success, which caller must close. -func (f *MultiFetcher) Fetch(ctx context.Context, ipfsPath string) (io.ReadCloser, error) { +func (f *MultiFetcher) Fetch(ctx context.Context, ipfsPath string) ([]byte, error) { var errs error for _, fetcher := range f.fetchers { - rc, err := fetcher.Fetch(ctx, ipfsPath) + out, err := fetcher.Fetch(ctx, ipfsPath) if err == nil { - return rc, nil + return out, nil } fmt.Printf("Error fetching: %s\n", err.Error()) errs = multierror.Append(errs, err) diff --git a/repo/fsrepo/migrations/httpfetcher.go b/repo/fsrepo/migrations/httpfetcher.go index 6fb20bb45..3f74a084e 100644 --- a/repo/fsrepo/migrations/httpfetcher.go +++ b/repo/fsrepo/migrations/httpfetcher.go @@ -60,9 +60,8 @@ func NewHttpFetcher(distPath, gateway, userAgent string, fetchLimit int64) *Http } // Fetch attempts to fetch the file at the given path, from the distribution -// site configured for this HttpFetcher. Returns io.ReadCloser on success, -// which caller must close. -func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) (io.ReadCloser, error) { +// site configured for this HttpFetcher. +func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error) { gwURL := f.gateway + path.Join(f.distPath, filePath) fmt.Printf("Fetching with HTTP: %q\n", gwURL) @@ -89,10 +88,15 @@ func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) (io.ReadCloser return nil, fmt.Errorf("GET %s error: %s: %s", gwURL, resp.Status, string(mes)) } + var rc io.ReadCloser if f.limit != 0 { - return NewLimitReadCloser(resp.Body, f.limit), nil + rc = NewLimitReadCloser(resp.Body, f.limit) + } else { + rc = resp.Body } - return resp.Body, nil + defer rc.Close() + + return ioutil.ReadAll(rc) } func (f *HttpFetcher) Close() error { diff --git a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go index 88f07b502..11203ed5a 100644 --- a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go +++ b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go @@ -52,6 +52,8 @@ type IpfsFetcher struct { addrInfo peer.AddrInfo } +var _ migrations.Fetcher = (*IpfsFetcher)(nil) + // NewIpfsFetcher creates a new IpfsFetcher // // Specifying "" for distPath sets the default IPNS path. @@ -85,9 +87,8 @@ func NewIpfsFetcher(distPath string, fetchLimit int64, repoRoot *string) *IpfsFe } // Fetch attempts to fetch the file at the given path, from the distribution -// site configured for this HttpFetcher. Returns io.ReadCloser on success, -// which caller must close. -func (f *IpfsFetcher) Fetch(ctx context.Context, filePath string) (io.ReadCloser, error) { +// site configured for this HttpFetcher. +func (f *IpfsFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error) { // Initialize and start IPFS node on first call to Fetch, since the fetcher // may be created by not used. f.openOnce.Do(func() { @@ -123,10 +124,15 @@ func (f *IpfsFetcher) Fetch(ctx context.Context, filePath string) (io.ReadCloser return nil, fmt.Errorf("%q is not a file", filePath) } + var rc io.ReadCloser if f.limit != 0 { - return migrations.NewLimitReadCloser(fileNode, f.limit), nil + rc = migrations.NewLimitReadCloser(fileNode, f.limit) + } else { + rc = fileNode } - return fileNode, nil + defer rc.Close() + + return ioutil.ReadAll(rc) } func (f *IpfsFetcher) Close() error { diff --git a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher_test.go b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher_test.go index 877de5e78..e300371a6 100644 --- a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher_test.go +++ b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher_test.go @@ -2,6 +2,7 @@ package ipfsfetcher import ( "bufio" + "bytes" "context" "fmt" "os" @@ -28,14 +29,13 @@ func TestIpfsFetcher(t *testing.T) { fetcher := NewIpfsFetcher("", 0, nil) defer fetcher.Close() - rc, err := fetcher.Fetch(ctx, "go-ipfs/versions") + out, err := fetcher.Fetch(ctx, "go-ipfs/versions") if err != nil { t.Fatal(err) } - defer rc.Close() var lines []string - scan := bufio.NewScanner(rc) + scan := bufio.NewScanner(bytes.NewReader(out)) for scan.Scan() { lines = append(lines, scan.Text()) } @@ -52,8 +52,7 @@ func TestIpfsFetcher(t *testing.T) { } // Check not found - _, err = fetcher.Fetch(ctx, "/no_such_file") - if err == nil { + if _, err = fetcher.Fetch(ctx, "/no_such_file"); err == nil { t.Fatal("expected error 404") } diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index 861dc7b7c..5eac91b29 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -155,13 +155,14 @@ func ReadMigrationConfig(repoRoot string) (*config.Migration, error) { // downloadSources, func GetMigrationFetcher(downloadSources []string, distPath string, newIpfsFetcher func(string) Fetcher) (Fetcher, error) { const httpUserAgent = "go-ipfs" + const numTriesPerHTTP = 3 var fetchers []Fetcher for _, src := range downloadSources { src := strings.TrimSpace(src) switch src { case "HTTPS", "https", "HTTP", "http": - fetchers = append(fetchers, NewHttpFetcher(distPath, "", httpUserAgent, 0)) + fetchers = append(fetchers, &RetryFetcher{NewHttpFetcher(distPath, "", httpUserAgent, 0), numTriesPerHTTP}) case "IPFS", "ipfs": if newIpfsFetcher != nil { fetchers = append(fetchers, newIpfsFetcher(distPath)) @@ -178,7 +179,7 @@ func GetMigrationFetcher(downloadSources []string, distPath string, newIpfsFetch default: return nil, errors.New("bad gateway address: url scheme must be http or https") } - fetchers = append(fetchers, NewHttpFetcher(distPath, u.String(), httpUserAgent, 0)) + fetchers = append(fetchers, &RetryFetcher{NewHttpFetcher(distPath, u.String(), httpUserAgent, 0), numTriesPerHTTP}) case "": // Ignore empty string } diff --git a/repo/fsrepo/migrations/migrations_test.go b/repo/fsrepo/migrations/migrations_test.go index b09174a49..0e52b3a65 100644 --- a/repo/fsrepo/migrations/migrations_test.go +++ b/repo/fsrepo/migrations/migrations_test.go @@ -3,7 +3,6 @@ package migrations import ( "context" "fmt" - "io" "log" "os" "path/filepath" @@ -290,7 +289,9 @@ func TestReadMigrationConfig(t *testing.T) { type mockIpfsFetcher struct{} -func (m *mockIpfsFetcher) Fetch(ctx context.Context, filePath string) (io.ReadCloser, error) { +var _ Fetcher = (*mockIpfsFetcher)(nil) + +func (m *mockIpfsFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error) { return nil, nil } @@ -323,7 +324,9 @@ func TestGetMigrationFetcher(t *testing.T) { if err != nil { t.Fatal(err) } - if _, ok := f.(*HttpFetcher); !ok { + if rf, ok := f.(*RetryFetcher); !ok { + t.Fatal("expected RetryFetcher") + } else if _, ok := rf.Fetcher.(*HttpFetcher); !ok { t.Fatal("expected HttpFetcher") } @@ -341,7 +344,9 @@ func TestGetMigrationFetcher(t *testing.T) { if err != nil { t.Fatal(err) } - if _, ok := f.(*HttpFetcher); !ok { + if rf, ok := f.(*RetryFetcher); !ok { + t.Fatal("expected RetryFetcher") + } else if _, ok := rf.Fetcher.(*HttpFetcher); !ok { t.Fatal("expected HttpFetcher") } diff --git a/repo/fsrepo/migrations/retryfetcher.go b/repo/fsrepo/migrations/retryfetcher.go new file mode 100644 index 000000000..81415bb67 --- /dev/null +++ b/repo/fsrepo/migrations/retryfetcher.go @@ -0,0 +1,33 @@ +package migrations + +import ( + "context" + "fmt" +) + +type RetryFetcher struct { + Fetcher + MaxTries int +} + +var _ Fetcher = (*RetryFetcher)(nil) + +func (r *RetryFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error) { + var lastErr error + for i := 0; i < r.MaxTries; i++ { + out, err := r.Fetcher.Fetch(ctx, filePath) + if err == nil { + return out, nil + } + + if ctx.Err() != nil { + return nil, ctx.Err() + } + lastErr = err + } + return nil, fmt.Errorf("exceeded number of retries. last error was %w", lastErr) +} + +func (r *RetryFetcher) Close() error { + return r.Fetcher.Close() +} diff --git a/repo/fsrepo/migrations/versions.go b/repo/fsrepo/migrations/versions.go index 69b2e290b..af5bbbbd9 100644 --- a/repo/fsrepo/migrations/versions.go +++ b/repo/fsrepo/migrations/versions.go @@ -2,6 +2,7 @@ package migrations import ( "bufio" + "bytes" "context" "errors" "fmt" @@ -39,16 +40,15 @@ func LatestDistVersion(ctx context.Context, fetcher Fetcher, dist string, stable // available on the distriburion site. List is in ascending order, unless // sortDesc is true. func DistVersions(ctx context.Context, fetcher Fetcher, dist string, sortDesc bool) ([]string, error) { - rc, err := fetcher.Fetch(ctx, path.Join(dist, distVersions)) + versionBytes, err := fetcher.Fetch(ctx, path.Join(dist, distVersions)) if err != nil { return nil, err } - defer rc.Close() prefix := "v" var vers []semver.Version - scan := bufio.NewScanner(rc) + scan := bufio.NewScanner(bytes.NewReader(versionBytes)) for scan.Scan() { ver, err := semver.Make(strings.TrimLeft(scan.Text(), prefix)) if err != nil { From 815e64f4a68acda116e4ef3cf36713fd71b455c9 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 17 Feb 2022 20:54:54 -0500 Subject: [PATCH 5/7] chore: bump migrations dist.ipfs.io CID to contain fs-repo-11-to-12 v1.0.2 --- repo/fsrepo/migrations/fetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/fsrepo/migrations/fetcher.go b/repo/fsrepo/migrations/fetcher.go index a63ba7276..ef605153a 100644 --- a/repo/fsrepo/migrations/fetcher.go +++ b/repo/fsrepo/migrations/fetcher.go @@ -11,7 +11,7 @@ import ( const ( // Current distribution to fetch migrations from - CurrentIpfsDist = "/ipfs/QmPweMoUxWFt1MSpYwLWsHB1GBcyYYDKPnANdERMY4U6hK" // fs-repo-11-to-12 v1.0.1 + CurrentIpfsDist = "/ipfs/QmdaCHYBDHEhXCMoynH5UcohEay6m1XayZCcxWZzKAHNVN" // fs-repo-11-to-12 v1.0.2 // Latest distribution path. Default for fetchers. LatestIpfsDist = "/ipns/dist.ipfs.io" From 51773ac3db05f63008fe356289dbf65f7d08af46 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 17 Feb 2022 12:00:03 -0500 Subject: [PATCH 6/7] docs: v0.12.0 release notes --- CHANGELOG.md | 153 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 121198c81..3d4cbefe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,158 @@ # go-ipfs changelog +## v0.12.0 2022-02-17 + +We're happy to announce go-ipfs 0.12.0. This release switches the storage of IPLD blocks to be keyed by multihash instead of CID. + +As usual, this release includes important fixes, some of which may be critical for security. Unless the fix addresses a bug being exploited in the wild, the fix will _not_ be called out in the release notes. Please make sure to update ASAP. See our [release process](https://github.com/ipfs/go-ipfs/tree/master/docs/releases.md#security-fix-policy) for details. + +### 🛠 BREAKING CHANGES + +- `ipfs refs local` will now list all blocks as if they were [raw]() CIDv1 instead of with whatever CID version and IPLD codecs they were stored with. All other functionality should remain the same. + +Keep reading to learn more details. + +#### 🔦 Highlights + +There is only one change since 0.11: + +##### Blockstore migration from full CID to Multihash keys + +We are switching the default low level [datastore](https://docs.ipfs.io/concepts/glossary/#datastore) to be keyed only by the [Multihash](https://docs.ipfs.io/concepts/glossary/#multihash) part of the [CID](https://docs.ipfs.io/concepts/glossary/#cid), and deduplicate some [blocks](https://docs.ipfs.io/concepts/glossary/#block) in the process. The blockstore will become [codec](https://docs.ipfs.io/concepts/glossary/#codec)-agnostic. + +###### Rationale + +The blockstore/datastore layers are not concerned with data interpretation, only with storage of binary blocks and verification that the Multihash they are addressed with (which comes from the CID), matches the block. In fact, different CIDs, with different codecs prefixes, may be carrying the same multihash, and referencing the same block. Carrying the CID abstraction so low on the stack means potentially fetching and storing the same blocks multiple times just because they are referenced by different CIDs. Prior to this change, a CIDv1 with a `dag-cbor` codec and a CIDv1 with a `raw` codec, both containing the same multihash, would result in two identical blocks stored. A CIDv0 and CIDv1 both being the same `dag-pb` block would also result in two copies. + +###### How migration works + +In order to perform the switch, and start referencing all blocks by their multihash, a migration will occur on update. This migration will take the repository version from 11 (current) to 12. + +One thing to note is that any content addressed CIDv0 (all the hashes that start with `Qm...`, the current default in go-ipfs), does not need any migration, as CIDv0 are raw multihashes already. This means the migration will be very lightweight for the majority of users. + +The migration process will take care of re-keying any CIDv1 block so that it is only addressed by its multihash. Large nodes with lots of CIDv1-addressed content will need to go through a heavier process as the migration happens. This is how the migration works: + +1. Phase 1: The migration script will perform a pass for every block in the datastore and will add all CIDv1s found to a file named `11-to-12-cids.txt`, in the go-ipfs configuration folder. Nothing is written in this first phase and it only serves to identify keys that will be migrated in phase 2. +2. Phase 2: The migration script will perform a second pass where every CIDv1 block will be read and re-written with its raw-multihash as key. There is 1 worker performing this task, although more can be configured. Every 100MiB-worth of blocks (this is configurable), each worker will trigger a datastore "sync" (to ensure all written data is flushed to disk) and delete the CIDv1-addressed blocks that were just renamed. This provides a good compromise between speed and resources needed to run the migration. + +At every sync, the migration emits a log message showing how many blocks need to be rewritten and how far the process is. + +####### FlatFS specific migration + +For those using a single FlatFS datastore as their backing blockstore (i.e. the default behavior), the migration (but not reversion) will take advantage of the ability to easily move/rename the blocks to improve migration performance. + +Unfortunately, other common datastores do not support renames which is what makes this FlatFS specific. If you are running a large custom datastore that supports renames you may want to consider running a fork of [fs-repo-11-to-12](https://github.com/ipfs/fs-repo-migrations/tree/master/fs-repo-11-to-12) specific to your datastore. + +If you want to disable this behavior, set the environment variable `IPFS_FS_MIGRATION_11_TO_12_ENABLE_FLATFS_FASTPATH` to `false`. + +####### Migration configuration + +For those who want to tune the migration more precisely for their setups, there are two environment variables to configure: + +- `IPFS_FS_MIGRATION_11_TO_12_NWORKERS` : an integer describing the number of migration workers - defaults to 1 +- `IPFS_FS_MIGRATION_11_TO_12_SYNC_SIZE_BYTES` : an integer describing the number of bytes after which migration workers will sync - defaults to 104857600 (i.e. 100MiB) + +###### Migration caveats + +Large repositories with very large numbers of CIDv1s should be mindful of the migration process: + +* We recommend ensuring that IPFS runs with an appropriate (high) file-descriptor limit, particularly when Badger is use as datastore backend. Badger is known to open many tables when experiencing a high number of writes, which may trigger "too many files open" type of errors during the migrations. If this happens, the migration can be retried with a higher FD limit (see below). +* Migrations using the Badger datastore may not immediately reclaim the space freed by the deletion of migrated blocks, thus space requirements may grow considerably. A periodic Badger-GC is run every 2 minutes, which will reclaim space used by deleted and de-duplicated blocks. The last portion of the space will only be reclaimed after go-ipfs starts (the Badger-GC cycle will trigger after 15 minutes). +* While there is a revert process detailed below, we recommend keeping a backup of the repository, particularly for very large ones, in case an issue happens, so that the revert can happen immediately and cases of repository corruption due to crashes or unexpected circumstances are not catastrophic. + +###### Migration interruptions and retries + +If a problem occurs during the migration, it is be possible to simply re-start and retry it: + +1. Phase 1 will never overwrite the `11-to-12-cids.txt` file, but only append to it (so that a list of things we were supposed to have migrated during our first attempt is not lost - this is important for reverts, see below). +2. Phase 2 will proceed to continue re-keying blocks that were not re-keyed during previous attempts. + +###### Migration reverts + +It is also possible to revert the migration after it has succeeded, for example to go to a previous go-ipfs version (<=[0.11](https://github.com/ipfs/go-ipfs/releases/tag/v0.11.0)), even after starting and using go-ipfs in the new version (>=0.12). The revert process works as follows: + +1. The `11-to-12-cids.txt` file is read, which has the list of all the CIDv1s that had to be rewritten for the migration. +2. A CIDv1-addressed block is written for every item on the list. This work is performed by 1 worker (configurable), syncing every 100MiB (configurable). +3. It is ensured that every CIDv1 pin, and every CIDv1 reference in MFS, are also written as CIDV1-addressed blocks, regardless of whether they were part of the original migration or were added later. + +The revert process does not delete any blocks--it only makes sure that blocks that were accessible with CIDv1s before the migration are again keyed with CIDv1s. This may result in a datastore becoming twice as large (i.e. if all the blocks were CIDv1-addressed before the migration). This is however done this way to cover corner cases: user can add CIDv1s after migration, which may reference blocks that existed as CIDv0 before migration. The revert aims to ensure that no data becomes unavailable on downgrade. + +While go-ipfs will auto-run the migration for you, it will not run the reversion. To do so you can download the [latest migration binary](https://dist.ipfs.io/fs-repo-11-to-12) or use [ipfs-update](https://dist.ipfs.io/#ipfs-update). + +###### Custom datastores + +As with previous migrations if you work with custom datastores and want to leverage the migration you can run a fork of [fs-repo-11-to-12](https://github.com/ipfs/fs-repo-migrations/tree/master/fs-repo-11-to-12) specific to your datastore. The repo includes instructions on building for different datastores. + +For this migration, if your datastore has fast renames you may want to consider writing some code to leverage the particular efficiencies of your datastore similar to what was done for FlatFS. + +### Changelog + +- github.com/ipfs/go-ipfs: + - Release v0.12.0 + - docs: v0.12.0 release notes + - chore: bump migrations dist.ipfs.io CID to contain fs-repo-11-to-12 v1.0.2 + - feat: refactor Fetcher interface used for downloading migrations (#8728) ([ipfs/go-ipfs#8728](https://github.com/ipfs/go-ipfs/pull/8728)) + - feat: log multifetcher errors + - Release v0.12.0-rc1 + - chore: bump Go version to 1.16.12 + - feat: switch to raw multihashes for blocks ([ipfs/go-ipfs#6816](https://github.com/ipfs/go-ipfs/pull/6816)) + - chore: add release template snippet for fetching artifact tarball + - chore: bump Go version to 1.16.11 + - chore: add release steps for upgrading Go + - Merge branch 'release' + - fix(corehttp): adjust peer counting metrics (#8577) ([ipfs/go-ipfs#8577](https://github.com/ipfs/go-ipfs/pull/8577)) + - chore: update version to v0.12.0-dev +- github.com/ipfs/go-filestore (v0.1.0 -> v1.1.0): + - Version 1.1.0 + - feat: plumb through context changes + - sync: update CI config files (#54) ([ipfs/go-filestore#54](https://github.com/ipfs/go-filestore/pull/54)) + - fix staticcheck ([ipfs/go-filestore#48](https://github.com/ipfs/go-filestore/pull/48)) + - Work with Multihashes directly (#21) ([ipfs/go-filestore#21](https://github.com/ipfs/go-filestore/pull/21)) +- github.com/ipfs/go-ipfs-blockstore (v0.2.1 -> v1.1.2): + - Release v1.1.2 + - feat: per-cid locking + - Version 1.1.1 + - fix: remove context from HashOnRead + - Version 1.1.0 (#91) ([ipfs/go-ipfs-blockstore#91](https://github.com/ipfs/go-ipfs-blockstore/pull/91)) + - feat: add context to interfaces (#90) ([ipfs/go-ipfs-blockstore#90](https://github.com/ipfs/go-ipfs-blockstore/pull/90)) + - sync: update CI config files (#88) ([ipfs/go-ipfs-blockstore#88](https://github.com/ipfs/go-ipfs-blockstore/pull/88)) + - add constructor that doesnt mess with datastore keys ([ipfs/go-ipfs-blockstore#83](https://github.com/ipfs/go-ipfs-blockstore/pull/83)) + - Use bloom filter in GetSize + - fix staticcheck ([ipfs/go-ipfs-blockstore#73](https://github.com/ipfs/go-ipfs-blockstore/pull/73)) + - add BenchmarkARCCacheConcurrentOps ([ipfs/go-ipfs-blockstore#70](https://github.com/ipfs/go-ipfs-blockstore/pull/70)) + - fix(arc): striped locking on last byte of CID ([ipfs/go-ipfs-blockstore#67](https://github.com/ipfs/go-ipfs-blockstore/pull/67)) + - make idstore implement io.Closer. (#60) ([ipfs/go-ipfs-blockstore#60](https://github.com/ipfs/go-ipfs-blockstore/pull/60)) + - add View() to all the various blockstores. (#59) ([ipfs/go-ipfs-blockstore#59](https://github.com/ipfs/go-ipfs-blockstore/pull/59)) + - Optimize id store ([ipfs/go-ipfs-blockstore#56](https://github.com/ipfs/go-ipfs-blockstore/pull/56)) + - add race fix for HashOnRead ([ipfs/go-ipfs-blockstore#50](https://github.com/ipfs/go-ipfs-blockstore/pull/50)) + - Add test to maintain Put contract of calling Has first ([ipfs/go-ipfs-blockstore#47](https://github.com/ipfs/go-ipfs-blockstore/pull/47)) + - Update readme and license ([ipfs/go-ipfs-blockstore#44](https://github.com/ipfs/go-ipfs-blockstore/pull/44)) + - feat: switch to raw multihashes for blocks ([ipfs/go-ipfs-blockstore#38](https://github.com/ipfs/go-ipfs-blockstore/pull/38)) +- github.com/ipfs/go-ipfs-ds-help (v0.1.1 -> v1.1.0): + - Update version.json (#38) ([ipfs/go-ipfs-ds-help#38](https://github.com/ipfs/go-ipfs-ds-help/pull/38)) + - sync: update CI config files (#37) ([ipfs/go-ipfs-ds-help#37](https://github.com/ipfs/go-ipfs-ds-help/pull/37)) + - feat: switch to raw multihashes for blocks ([ipfs/go-ipfs-ds-help#18](https://github.com/ipfs/go-ipfs-ds-help/pull/18)) + +### ❤️ Contributors + +| Contributor | Commits | Lines ± | Files Changed | +|-------------|---------|---------|---------------| +| Gus Eggert | 10 | +333/-321 | 24 | +| Steven Allen | 7 | +289/-190 | 13 | +| Hector Sanjuan | 9 | +134/-109 | 18 | +| Adin Schmahmann | 11 | +179/-55 | 21 | +| Raúl Kripalani | 2 | +152/-42 | 5 | +| Daniel Martí | 1 | +120/-1 | 1 | +| frrist | 1 | +95/-13 | 2 | +| Alex Trottier | 2 | +22/-11 | 4 | +| Andrey Petrov | 1 | +32/-0 | 1 | +| Lucas Molas | 1 | +18/-7 | 2 | +| Marten Seemann | 2 | +11/-7 | 3 | +| whyrusleeping | 1 | +10/-0 | 1 | +| web3-bot | 3 | +9/-0 | 3 | +| postables | 1 | +5/-3 | 1 | +| Dr Ian Preston | 1 | +4/-0 | 1 | + ## v0.11.0 2021-12-08 We're happy to announce go-ipfs 0.11.0. This release comes with improvements to the UnixFS Sharding and PubSub experiments as well as support for Circuit-Relay v2 which sets the network up for decentralized hole punching support. From dd7737b931d8bc5cf62c85e77d4428890b813eb6 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 17 Feb 2022 21:01:55 -0500 Subject: [PATCH 7/7] Release v0.12.0 --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index 533657815..07019c1aa 100644 --- a/version.go +++ b/version.go @@ -4,7 +4,7 @@ package ipfs var CurrentCommit string // CurrentVersionNumber is the current application's version literal -const CurrentVersionNumber = "0.12.0-rc1" +const CurrentVersionNumber = "0.12.0" const ApiVersion = "/go-ipfs/" + CurrentVersionNumber + "/"