From f9e244a7dbb139e939d8baee21a0ecd55317dbb1 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 1 Dec 2021 10:07:08 -0500 Subject: [PATCH 1/6] chore: update version to v0.12.0-dev --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index 6199a3294..e6aa56b97 100644 --- a/version.go +++ b/version.go @@ -4,7 +4,7 @@ package ipfs var CurrentCommit string // CurrentVersionNumber is the current application's version literal -const CurrentVersionNumber = "0.11.0-dev" +const CurrentVersionNumber = "0.12.0-dev" const ApiVersion = "/go-ipfs/" + CurrentVersionNumber + "/" From 9d197ca732b5b1f80a907d35f36d6d0c0b6828cf Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Thu, 2 Dec 2021 22:03:00 -0300 Subject: [PATCH 2/6] fix(corehttp): adjust peer counting metrics (#8577) --- core/corehttp/metrics.go | 13 +++++++++++-- core/corehttp/metrics_test.go | 12 +++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/core/corehttp/metrics.go b/core/corehttp/metrics.go index e7f36113b..c3dbde0ac 100644 --- a/core/corehttp/metrics.go +++ b/core/corehttp/metrics.go @@ -160,9 +160,18 @@ func (c IpfsNodeCollector) PeersTotalValues() map[string]float64 { if c.Node.PeerHost == nil { return vals } - for _, conn := range c.Node.PeerHost.Network().Conns() { + for _, peerID := range c.Node.PeerHost.Network().Peers() { + // Each peer may have more than one connection (see for an explanation + // https://github.com/libp2p/go-libp2p-swarm/commit/0538806), so we grab + // only one, the first (an arbitrary and non-deterministic choice), which + // according to ConnsToPeer is the oldest connection in the list + // (https://github.com/libp2p/go-libp2p-swarm/blob/v0.2.6/swarm.go#L362-L364). + conns := c.Node.PeerHost.Network().ConnsToPeer(peerID) + if len(conns) == 0 { + continue + } tr := "" - for _, proto := range conn.RemoteMultiaddr().Protocols() { + for _, proto := range conns[0].RemoteMultiaddr().Protocols() { tr = tr + "/" + proto.Name } vals[tr] = vals[tr] + 1 diff --git a/core/corehttp/metrics_test.go b/core/corehttp/metrics_test.go index dc3b68bc9..76f79d2ee 100644 --- a/core/corehttp/metrics_test.go +++ b/core/corehttp/metrics_test.go @@ -44,11 +44,13 @@ func TestPeersTotal(t *testing.T) { node := &core.IpfsNode{PeerHost: hosts[0]} collector := IpfsNodeCollector{Node: node} - actual := collector.PeersTotalValues() - if len(actual) != 1 { - t.Fatalf("expected 1 peers transport, got %d, transport map %v", len(actual), actual) + peersTransport := collector.PeersTotalValues() + if len(peersTransport) > 2 { + t.Fatalf("expected at most 2 peers transport (tcp and upd/quic), got %d, transport map %v", + len(peersTransport), peersTransport) } - if actual["/ip4/tcp"] != float64(3) { - t.Fatalf("expected 3 peers, got %f", actual["/ip4/tcp"]) + totalPeers := peersTransport["/ip4/tcp"] + peersTransport["/ip4/udp/quic"] + if totalPeers != 3 { + t.Fatalf("expected 3 peers in either tcp or upd/quic transport, got %f", totalPeers) } } From 024e84adec9c51e4076124a3b2eb9ad0c1bcf0bd Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Fri, 3 Dec 2021 13:35:35 -0500 Subject: [PATCH 3/6] fix: update go-ds-flatfs to v0.5.1 Fixes #8579 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9480df873..4a737f326 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/ipfs/go-datastore v0.5.1 github.com/ipfs/go-detect-race v0.0.1 github.com/ipfs/go-ds-badger v0.3.0 - github.com/ipfs/go-ds-flatfs v0.5.0 + github.com/ipfs/go-ds-flatfs v0.5.1 github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-ds-measure v0.2.0 github.com/ipfs/go-fetcher v1.6.1 diff --git a/go.sum b/go.sum index 02764d239..ea3efa172 100644 --- a/go.sum +++ b/go.sum @@ -441,8 +441,8 @@ github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBR github.com/ipfs/go-ds-badger v0.2.7/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA= github.com/ipfs/go-ds-badger v0.3.0 h1:xREL3V0EH9S219kFFueOYJJTcjgNSZ2HY1iSvN7U1Ro= github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= -github.com/ipfs/go-ds-flatfs v0.5.0 h1:ntq2P05cC51VOSZ0VO/HRaCoLefVXNKGRu9gYr9Ajko= -github.com/ipfs/go-ds-flatfs v0.5.0/go.mod h1:RWTV7oZD/yZYBKdbVIFXTX2fdY2Tbvl94NsWqmoyAX4= +github.com/ipfs/go-ds-flatfs v0.5.1 h1:ZCIO/kQOS/PSh3vcF1H6a8fkRGS7pOfwfPdx4n/KJH4= +github.com/ipfs/go-ds-flatfs v0.5.1/go.mod h1:RWTV7oZD/yZYBKdbVIFXTX2fdY2Tbvl94NsWqmoyAX4= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= From 7026952c71eca57d30709f3446d1c7456627602f Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 6 Dec 2021 20:24:26 +0100 Subject: [PATCH 4/6] fix(ci): dist.ipfs.io sync preserving binaries tar.gz got mangled because of reasons known to github, this does the correct upload every time by forcing github to interpret payload as an opaque stream of bytes without anything fancy --- .github/workflows/sync-release-assets.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-release-assets.yml b/.github/workflows/sync-release-assets.yml index 40351bb39..4cdeb4fc5 100644 --- a/.github/workflows/sync-release-assets.yml +++ b/.github/workflows/sync-release-assets.yml @@ -107,14 +107,17 @@ jobs: for (const file of missing_files) { console.log("fetching", file, "from dist.ipfs.io") await exec.exec('ipfs', ['get', p + '/' + file]) - const data = await fs.readFile(file, "binary") console.log("uploading", file, "to github release", release.tag_name) resp = await github.repos.uploadReleaseAsset({ owner: context.repo.owner, repo: context.repo.repo, release_id: release.id, + headers: { + "content-type": "application/octet-stream", + "content-length": `${(await fs.stat(file)).size}` + }, name: file, - data: data, + data: await fs.readFile(file), }) } // summary of assets on both sides From 8cfc88961d4081b2749fba135b63ffcca5a62456 Mon Sep 17 00:00:00 2001 From: Petar Maymounkov Date: Tue, 7 Dec 2021 16:45:42 -0800 Subject: [PATCH 5/6] chore: verify checksums during dist.ipfs.io sync (#8587) * only copy files from dist.ipfs.io which have associated checksum files (also copy the checksum files) * verify checksums before copying * also, ignore path from sha512sum output, which sometimes has absolute path on dist.ipfs.io website Co-authored-by: Marcin Rataj --- .github/workflows/sync-release-assets.yml | 59 ++++++++++++++++++----- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/.github/workflows/sync-release-assets.yml b/.github/workflows/sync-release-assets.yml index 4cdeb4fc5..d1e0826ac 100644 --- a/.github/workflows/sync-release-assets.yml +++ b/.github/workflows/sync-release-assets.yml @@ -72,11 +72,9 @@ jobs: } num_synced += 1 - const github_assets = []; - github_map = {}; + const github_assets = new Set() for (const asset of release.assets) { - github_assets.push(asset.name); - github_map[asset.name] = true; + github_assets.add(asset.name) } // fetch asset info from dist.ipfs.io @@ -90,14 +88,14 @@ jobs: } await exec.exec('ipfs', ['ls', p], options) - const dist_assets = [] - missing_files = [] + const dist_assets = new Set() + const missing_files = [] for (const raw_line of stdout.split("\n")) { line = raw_line.trim(); if (line.length != 0) { file = line.split(/(\s+)/).filter( function(e) { return e.trim().length > 0; } )[2] - dist_assets.push(file) - if (!github_map[file]) { + dist_assets.add(file) + if (!github_assets.has(file)) { missing_files.push(file) } } @@ -105,23 +103,60 @@ jobs: // if dist.ipfs.io has files not found in github, copy them over for (const file of missing_files) { + file_sha = file + ".sha512" + file_cid = file + ".cid" + + // skip files that don't have .cid and .sha512 checksum files + if (!dist_assets.has(file_sha) || !dist_assets.has(file_cid)) { + if (!file.endsWith('.cid') && !file.endsWith('.sha512')) { // silent skip of .sha512.sha512 :) + console.log(`skipping "${file}" as dist.ipfs.io does not provide .cid and .sha512 checksum files for it`) + } + continue + } + console.log("fetching", file, "from dist.ipfs.io") await exec.exec('ipfs', ['get', p + '/' + file]) + await exec.exec('ipfs', ['get', p + '/' + file_sha]) + await exec.exec('ipfs', ['get', p + '/' + file_cid]) + console.log("verifying contents of", file) + + // compute sha512 output for file + let sha_stdout = '' + const sha_options = {} + sha_options.listeners = { + stdout: (data) => { + sha_stdout += data.toString(); + } + } + await exec.exec('sha512sum', [file], sha_options) + // read expected sha512 output + const sha_data = await fs.readFile(file_sha, "utf8") + const digest = (s) => s.split(' ').shift() + if (digest(sha_data) != digest(sha_stdout)) { + console.log(`${file}.sha512: ${sha_data}`) + console.log(`sha512sum ${file}: ${sha_stdout}`) + throw "checksum verification failed for " + file + } + console.log("uploading", file, "to github release", release.tag_name) - resp = await github.repos.uploadReleaseAsset({ + const uploadReleaseAsset = async (file) => github.repos.uploadReleaseAsset({ owner: context.repo.owner, repo: context.repo.repo, release_id: release.id, headers: { "content-type": "application/octet-stream", "content-length": `${(await fs.stat(file)).size}` - }, + }, name: file, - data: await fs.readFile(file), + data: await fs.readFile(file) }) + await uploadReleaseAsset(file) + await uploadReleaseAsset(file_sha) + await uploadReleaseAsset(file_cid) + } // summary of assets on both sides - release_assets.push({ tag: release.tag_name, github_assets: github_assets, dist_assets: dist_assets }) + release_assets.push({ tag: release.tag_name, github_assets, dist_assets }) } console.log(release_assets) return release_assets From 615fb80d37b97ba966d166c43c82db1142723dfa Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 7 Dec 2021 14:26:44 -0500 Subject: [PATCH 6/6] chore: update go-ipld-prime --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 4a737f326..c3c87b567 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,7 @@ require ( github.com/ipfs/tar-utils v0.0.2 github.com/ipld/go-car v0.3.2 github.com/ipld/go-codec-dagpb v1.3.0 - github.com/ipld/go-ipld-prime v0.14.1 + github.com/ipld/go-ipld-prime v0.14.2 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 github.com/jbenet/goprocess v0.1.4 diff --git a/go.sum b/go.sum index ea3efa172..68cac021e 100644 --- a/go.sum +++ b/go.sum @@ -586,8 +586,9 @@ github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/j github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.12.3/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8= -github.com/ipld/go-ipld-prime v0.14.1 h1:n9obcUnuqPK34HlfbiB+o9GhXE/x59uue4z9YTsaoj4= github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= +github.com/ipld/go-ipld-prime v0.14.2 h1:P5fO2usnisXwrN/1sR5exCgEvINg/w/27EuYPKB/zx8= +github.com/ipld/go-ipld-prime v0.14.2/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=