From 90021c16d6bde306564ff28fa13f01ec1ba3bae4 Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 22 Aug 2018 18:56:28 +0100 Subject: [PATCH 01/40] [http_proxy_over_p2p] This implements an http-proxy over p2p-streams. License: MIT Signed-off-by: Chris Boddy --- cmd/ipfs/daemon.go | 1 + p2p/local.go | 2 +- p2p/p2p.go | 12 +++--- p2p/proxy.go | 92 ++++++++++++++++++++++++++++++++++++++++++++++ p2p/proxy_test.go | 21 +++++++++++ 5 files changed, 121 insertions(+), 7 deletions(-) create mode 100644 p2p/proxy.go create mode 100644 p2p/proxy_test.go diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index f11d00a16..69268e871 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -461,6 +461,7 @@ func serveHTTPApi(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, error corehttp.MutexFractionOption("/debug/pprof-mutex/"), corehttp.MetricsScrapingOption("/debug/metrics/prometheus"), corehttp.LogOption(), + corehttp.ProxyOption(), } if len(cfg.Gateway.RootRedirect) > 0 { diff --git a/p2p/local.go b/p2p/local.go index 47c7a312b..076af3bc4 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -55,7 +55,7 @@ func (l *localListener) dial(ctx context.Context) (net.Stream, error) { cctx, cancel := context.WithTimeout(ctx, time.Second*30) //TODO: configurable? defer cancel() - return l.p2p.peerHost.NewStream(cctx, l.peer, l.proto) + return l.p2p.PeerHost.NewStream(cctx, l.peer, l.proto) } func (l *localListener) acceptConns() { diff --git a/p2p/p2p.go b/p2p/p2p.go index efc87f722..2d8ee8ca4 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -16,23 +16,23 @@ type P2P struct { Streams *StreamRegistry identity peer.ID - peerHost p2phost.Host + PeerHost p2phost.Host peerstore pstore.Peerstore } // NewP2P creates new P2P struct -func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) *P2P { +func NewP2P(identity peer.ID, PeerHost p2phost.Host, peerstore pstore.Peerstore) *P2P { return &P2P{ identity: identity, - peerHost: peerHost, + PeerHost: PeerHost, peerstore: peerstore, ListenersLocal: newListenersLocal(), - ListenersP2P: newListenersP2P(peerHost), + ListenersP2P: newListenersP2P(PeerHost), Streams: &StreamRegistry{ Streams: map[uint64]*Stream{}, - ConnManager: peerHost.ConnManager(), + ConnManager: PeerHost.ConnManager(), conns: map[peer.ID]int{}, }, } @@ -41,7 +41,7 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) // CheckProtoExists checks whether a proto handler is registered to // mux handler func (p2p *P2P) CheckProtoExists(proto string) bool { - protos := p2p.peerHost.Mux().Protocols() + protos := p2p.PeerHost.Mux().Protocols() for _, p := range protos { if p != proto { diff --git a/p2p/proxy.go b/p2p/proxy.go new file mode 100644 index 000000000..a24760092 --- /dev/null +++ b/p2p/proxy.go @@ -0,0 +1,92 @@ +package p2p + +import ( + "bufio" + "fmt" + "net" + "net/http" + "strings" + + core "github.com/ipfs/go-ipfs/core" + protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" + peer "gx/ipfs/QmbNepETomvmXfz1X5pHNFD2QuPqnqi47dTd94QJWSorQ3/go-libp2p-peer" +) + +func ProxyOption() ServeOption { + return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { + mux.HandleFunc("/proxy/", func(w http.ResponseWriter, request *http.Request) { + // parse request + parsedRequest, err := parseRequest(request) + if err != nil { + handleError(w, "Failed to parse request", err, 400) + return + } + + // open connect to peer + stream, err := ipfsNode.P2P.PeerHost.NewStream(ipfsNode.Context(), parsedRequest.target, protocol.ID("/x/"+parsedRequest.name)) + if err != nil { + msg := fmt.Sprintf("Failed to open stream '%v' to target peer '%v'", parsedRequest.name, parsedRequest.target) + handleError(w, msg, err, 500) + return + } + + // send request to peer + proxyReq, err := http.NewRequest(request.Method, parsedRequest.httpPath, request.Body) + + if err != nil { + handleError(w, "Failed to format proxy request", err, 500) + return + } + + proxyReq.Write(stream) + + s := bufio.NewReader(stream) + proxyResponse, err := http.ReadResponse(s, proxyReq) + defer func() { proxyResponse.Body.Close() }() + if err != nil { + msg := fmt.Sprintf("Failed to send request to stream '%v' to peer '%v'", parsedRequest.name, parsedRequest.target) + handleError(w, msg, err, 500) + return + } + // send client response + proxyResponse.Write(w) + }) + return mux, nil + } +} + +type proxyRequest struct { + target peer.ID + name string + httpPath string // path to send to the proxy-host +} + +// from the url path parse the peer-ID, name and http path +// /http/$peer_id/$name/$http_path +func parseRequest(request *http.Request) (*proxyRequest, error) { + path := request.URL.Path + + split := strings.SplitN(path, "/", 6) + if split[2] != "http" { + return nil, fmt.Errorf("Invalid proxy request protocol '%s'", path) + } + + if len(split) < 6 { + return nil, fmt.Errorf("Invalid request path '%s'", path) + } + + peerID, err := peer.IDB58Decode(split[3]) + + if err != nil { + return nil, err + } + + return &proxyRequest{peerID, split[4], split[5]}, nil +} + +// log error and send response to client +func handleError(w http.ResponseWriter, msg string, err error, code int) { + w.WriteHeader(code) + fmt.Fprintf(w, "%s: %s\n", msg, err) + log.Warningf("server error: %s: %s", err) +} diff --git a/p2p/proxy_test.go b/p2p/proxy_test.go new file mode 100644 index 000000000..b818ce707 --- /dev/null +++ b/p2p/proxy_test.go @@ -0,0 +1,21 @@ +package p2p + +import ( + "github.com/ipfs/go-ipfs/thirdparty/assert" + "net/http" + "strings" + "testing" +) + +func TestParseRequest(t *testing.T) { + url := "http://localhost:5001/proxy/http/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/test-name/path/to/index.txt" + req, _ := http.NewRequest("GET", url, strings.NewReader("")) + + parsed, err := parseRequest(req) + if err != nil { + t.Error(err) + } + assert.True(parsed.httpPath == "path/to/index.txt", t, "proxy request path") + assert.True(parsed.name == "test-name", t, "proxy request name") + assert.True(parsed.target.Pretty() == "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", t, "proxy request peer-id") +} From c862ac1fb47c7833c2b614fafaf46a56d4e25343 Mon Sep 17 00:00:00 2001 From: Chris Boddy Date: Wed, 26 Sep 2018 22:34:05 +0100 Subject: [PATCH 02/40] [http_proxy_over_p2p] more tests, fix build License: MIT Signed-off-by: Chris Boddy --- {p2p => core/corehttp}/proxy.go | 13 +++++----- core/corehttp/proxy_test.go | 45 +++++++++++++++++++++++++++++++++ p2p/proxy_test.go | 21 --------------- 3 files changed, 52 insertions(+), 27 deletions(-) rename {p2p => core/corehttp}/proxy.go (97%) create mode 100644 core/corehttp/proxy_test.go delete mode 100644 p2p/proxy_test.go diff --git a/p2p/proxy.go b/core/corehttp/proxy.go similarity index 97% rename from p2p/proxy.go rename to core/corehttp/proxy.go index a24760092..47b26be17 100644 --- a/p2p/proxy.go +++ b/core/corehttp/proxy.go @@ -1,4 +1,4 @@ -package p2p +package corehttp import ( "bufio" @@ -8,6 +8,7 @@ import ( "strings" core "github.com/ipfs/go-ipfs/core" + protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" peer "gx/ipfs/QmbNepETomvmXfz1X5pHNFD2QuPqnqi47dTd94QJWSorQ3/go-libp2p-peer" ) @@ -62,19 +63,19 @@ type proxyRequest struct { } // from the url path parse the peer-ID, name and http path -// /http/$peer_id/$name/$http_path +// /proxy/http/$peer_id/$name/$http_path func parseRequest(request *http.Request) (*proxyRequest, error) { path := request.URL.Path split := strings.SplitN(path, "/", 6) - if split[2] != "http" { - return nil, fmt.Errorf("Invalid proxy request protocol '%s'", path) - } - if len(split) < 6 { return nil, fmt.Errorf("Invalid request path '%s'", path) } + if split[2] != "http" { + return nil, fmt.Errorf("Invalid proxy request protocol '%s'", split[2]) + } + peerID, err := peer.IDB58Decode(split[3]) if err != nil { diff --git a/core/corehttp/proxy_test.go b/core/corehttp/proxy_test.go new file mode 100644 index 000000000..e5b085895 --- /dev/null +++ b/core/corehttp/proxy_test.go @@ -0,0 +1,45 @@ +package corehttp + +import ( + "github.com/ipfs/go-ipfs/thirdparty/assert" + "net/http" + "strings" + "testing" +) + +func TestParseRequest(t *testing.T) { + url := "http://localhost:5001/proxy/http/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/test-name/path/to/index.txt" + req, _ := http.NewRequest("GET", url, strings.NewReader("")) + + parsed, err := parseRequest(req) + if err != nil { + t.Error(err) + } + assert.True(parsed.httpPath == "path/to/index.txt", t, "proxy request path") + assert.True(parsed.name == "test-name", t, "proxy request name") + assert.True(parsed.target.Pretty() == "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", t, "proxy request peer-id") +} + +func TestParseRequestInvalidProtocol(t *testing.T) { + url := "http://localhost:5001/proxy/invalid/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/test-name/path/to/index.txt" + req, _ := http.NewRequest("GET", url, strings.NewReader("")) + + _, err := parseRequest(req) + if err == nil { + t.Fail() + } + + assert.True(err.Error() == "Invalid proxy request protocol 'invalid'", t, "fails with invalid proxy") +} + +func TestParseRequestInvalidPath(t *testing.T) { + url := "http://localhost:5001/proxy/http/foobar" + req, _ := http.NewRequest("GET", url, strings.NewReader("")) + + _, err := parseRequest(req) + if err == nil { + t.Fail() + } + + assert.True(err.Error() == "Invalid request path '/proxy/http/foobar'", t, "fails with invalid path") +} diff --git a/p2p/proxy_test.go b/p2p/proxy_test.go deleted file mode 100644 index b818ce707..000000000 --- a/p2p/proxy_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package p2p - -import ( - "github.com/ipfs/go-ipfs/thirdparty/assert" - "net/http" - "strings" - "testing" -) - -func TestParseRequest(t *testing.T) { - url := "http://localhost:5001/proxy/http/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/test-name/path/to/index.txt" - req, _ := http.NewRequest("GET", url, strings.NewReader("")) - - parsed, err := parseRequest(req) - if err != nil { - t.Error(err) - } - assert.True(parsed.httpPath == "path/to/index.txt", t, "proxy request path") - assert.True(parsed.name == "test-name", t, "proxy request name") - assert.True(parsed.target.Pretty() == "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", t, "proxy request peer-id") -} From 1945e44f7fa786682287b0a775675646d18c9a63 Mon Sep 17 00:00:00 2001 From: Chris Boddy Date: Sat, 29 Sep 2018 16:09:44 +0100 Subject: [PATCH 03/40] [http_proxy_over_p2p] Add sharness tests for http-proxy-over-p2p License: MIT Signed-off-by: Chris Boddy --- core/corehttp/proxy.go | 1 + test/sharness/t0184-http-proxy-over-p2p.sh | 152 +++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100755 test/sharness/t0184-http-proxy-over-p2p.sh diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 47b26be17..a3e2c735d 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -43,6 +43,7 @@ func ProxyOption() ServeOption { s := bufio.NewReader(stream) proxyResponse, err := http.ReadResponse(s, proxyReq) + defer func() { proxyResponse.Body.Close() }() if err != nil { msg := fmt.Sprintf("Failed to send request to stream '%v' to peer '%v'", parsedRequest.name, parsedRequest.target) diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh new file mode 100755 index 000000000..7603b7a05 --- /dev/null +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash + +test_description="Test http proxy over p2p" + +. lib/test-lib.sh +WEB_SERVE_PORT=5099 + +function serve_http_once() { + # + # one shot http server (via nc) with static body + # + local body=$1 + local status_code=${2:-"200 OK"} + local length=$(expr 1 + ${#body}) + echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" | nc -l $WEB_SERVE_PORT & + REMOTE_SERVER_PID=$! +} + + +function setup_receiver_ipfs() { + # + # setup RECEIVER IPFS daemon + # + IPFS_PATH=$(mktemp -d) + RECEIVER_LOG=$IPFS_PATH/ipfs.log + + ipfs init >> $RECEIVER_LOG 2>&1 + ipfs config --json Experimental.Libp2pStreamMounting true >> $RECEIVER_LOG 2>&1 + ipfs config --json Addresses.API "\"/ip4/127.0.0.1/tcp/6001\"" >> $RECEIVER_LOG 2>&1 + ipfs config --json Addresses.Gateway "\"/ip4/127.0.0.1/tcp/8081\"" >> $RECEIVER_LOG 2>&1 + ipfs config --json Addresses.Swarm "[\"/ip4/0.0.0.0/tcp/7001\", \"/ip6/::/tcp/7001\"]" >> $RECEIVER_LOG 2>&1 + ipfs daemon >> $RECEIVER_LOG 2>&1 & + RECEIVER_PID=$! + # wait for daemon to start.. maybe? + # ipfs id returns empty string if we don't wait here.. + sleep 5 + RECEIVER_ID=$(ipfs id -f "") + # + # start a p2p listener on RECIVER to the HTTP server with our content + # + ipfs p2p listen /x/test /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT >> $RECEIVER_LOG 2>&1 +} + + +function setup_sender_ipfs() { + # + # setup SENDER IPFS daemon + # + IPFS_PATH=$(mktemp -d) + SENDER_LOG=$IPFS_PATH/ipfs.log + ipfs init >> $SENDER_LOG 2>&1 + ipfs config --json Experimental.Libp2pStreamMounting true >> $SENDER_LOG 2>&1 + ipfs daemon >> $SENDER_LOG 2>&1 & + SENDER_PID=$! + sleep 5 +} + + +function teardown_sender_and_receiver() { + kill -9 $SENDER_PID $RECEIVER_PID > /dev/null 2>&1 + sleep 5 +} + +function curl_check_response_code() { + local expected_status_code=$1 + local path_stub=${2:-http/$RECEIVER_ID/test/index.txt} + local status_code=$(curl -s --write-out %{http_code} --output /dev/null http://localhost:5001/proxy/$path_stub) + + if [[ $status_code -ne $expected_status_code ]]; + then + echo "Found status-code "$status_code", expected "$expected_status_code + return 1 + fi + + return 0 +} + +function curl_send_proxy_request_and_check_response() { + local expected_status_code=$1 + local expected_content=$2 + + # + # make a request to SENDER_IPFS via the proxy endpoint + # + CONTENT_PATH=$(mktemp) + STATUS_CODE=$(curl -s -o $CONTENT_PATH --write-out %{http_code} http://localhost:5001/proxy/http/$RECEIVER_ID/test/index.txt) + + # + # check status code + # + if [[ $STATUS_CODE -ne $expected_status_code ]]; + then + echo "Found status-code "$STATUS_CODE", expected "$expected_status_code + return 1 + fi + + # + # check content + # + RESPONSE_CONTENT=$(tail -n 1 $CONTENT_PATH) + if [[ "$RESPONSE_CONTENT" == "$expected_content" ]]; + then + return 0 + else + echo -e "Found response content:\n'"$RESPONSE_CONTENT"'\nthat differs from expected content:\n'"$expected_content"'" + return 1 + fi +} + + +#test_expect_success 'handle proxy http request propogates error response from remote' ' +#serve_http_once "SORRY GUYS, I LOST IT" "404 Not Found" && +#setup_receiver_ipfs && +#setup_sender_ipfs && +#curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT" +#' +#kill -9 $REMOTE_SERVER_PID +#teardown_sender_and_receiver + +test_expect_success 'handle proxy http request when remote server not available ' ' +setup_receiver_ipfs && +setup_sender_ipfs && +curl_check_response_code "000" +' +teardown_sender_and_receiver + +test_expect_success 'handle proxy http request ' ' +serve_http_once "THE WOODS ARE LOVELY DARK AND DEEP" && +setup_receiver_ipfs && +setup_sender_ipfs && +curl_send_proxy_request_and_check_response 200 "THE WOODS ARE LOVELY DARK AND DEEP" +' +kill -9 $REMOTE_SERVER_PID +teardown_sender_and_receiver + + + +test_expect_success 'handle proxy http request invalid request' ' +setup_receiver_ipfs && +setup_sender_ipfs && +curl_check_response_code 400 DERPDERPDERP +' +teardown_sender_and_receiver + +test_expect_success 'handle proxy http request unknown proxy peer ' ' +setup_receiver_ipfs && +setup_sender_ipfs && +curl_check_response_code 400 http/unknown_peer/test/index.txt +' +teardown_sender_and_receiver + +test_done From 22f3b11621cea0ef4f9617c53f8e05da3ab49e56 Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Tue, 2 Oct 2018 00:42:05 +0100 Subject: [PATCH 04/40] change handler mount point to /proxy/http/ License: MIT Signed-off-by: Ian Preston --- core/corehttp/proxy.go | 6 +----- test/sharness/t0184-http-proxy-over-p2p.sh | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index a3e2c735d..8e236d155 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -15,7 +15,7 @@ import ( func ProxyOption() ServeOption { return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { - mux.HandleFunc("/proxy/", func(w http.ResponseWriter, request *http.Request) { + mux.HandleFunc("/proxy/http/", func(w http.ResponseWriter, request *http.Request) { // parse request parsedRequest, err := parseRequest(request) if err != nil { @@ -73,10 +73,6 @@ func parseRequest(request *http.Request) (*proxyRequest, error) { return nil, fmt.Errorf("Invalid request path '%s'", path) } - if split[2] != "http" { - return nil, fmt.Errorf("Invalid proxy request protocol '%s'", split[2]) - } - peerID, err := peer.IDB58Decode(split[3]) if err != nil { diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index 7603b7a05..09c3e11f5 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -138,7 +138,7 @@ teardown_sender_and_receiver test_expect_success 'handle proxy http request invalid request' ' setup_receiver_ipfs && setup_sender_ipfs && -curl_check_response_code 400 DERPDERPDERP +curl_check_response_code 404 DERPDERPDERP ' teardown_sender_and_receiver From 335bca2bb7da738d329e25fd0cf1f7b705fcf52a Mon Sep 17 00:00:00 2001 From: Chris Boddy Date: Tue, 2 Oct 2018 09:51:51 +0100 Subject: [PATCH 05/40] [http_proxy_over_p2p] remove now superfluous test License: MIT Signed-off-by: Chris Boddy --- core/corehttp/proxy.go | 3 +++ core/corehttp/proxy_test.go | 12 ------------ 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 8e236d155..4683bc2ca 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "net/http" + //"net/http/httputil" "strings" core "github.com/ipfs/go-ipfs/core" @@ -13,6 +14,7 @@ import ( peer "gx/ipfs/QmbNepETomvmXfz1X5pHNFD2QuPqnqi47dTd94QJWSorQ3/go-libp2p-peer" ) +// This adds an endpoint for proxying a request to another ipfs peer func ProxyOption() ServeOption { return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { mux.HandleFunc("/proxy/http/", func(w http.ResponseWriter, request *http.Request) { @@ -31,6 +33,7 @@ func ProxyOption() ServeOption { return } + //httputil.ReverseProxy( // send request to peer proxyReq, err := http.NewRequest(request.Method, parsedRequest.httpPath, request.Body) diff --git a/core/corehttp/proxy_test.go b/core/corehttp/proxy_test.go index e5b085895..e3279317c 100644 --- a/core/corehttp/proxy_test.go +++ b/core/corehttp/proxy_test.go @@ -20,18 +20,6 @@ func TestParseRequest(t *testing.T) { assert.True(parsed.target.Pretty() == "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", t, "proxy request peer-id") } -func TestParseRequestInvalidProtocol(t *testing.T) { - url := "http://localhost:5001/proxy/invalid/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/test-name/path/to/index.txt" - req, _ := http.NewRequest("GET", url, strings.NewReader("")) - - _, err := parseRequest(req) - if err == nil { - t.Fail() - } - - assert.True(err.Error() == "Invalid proxy request protocol 'invalid'", t, "fails with invalid proxy") -} - func TestParseRequestInvalidPath(t *testing.T) { url := "http://localhost:5001/proxy/http/foobar" req, _ := http.NewRequest("GET", url, strings.NewReader("")) From 5f246e3211cba21c2e6122ee1c535458b13ad3a5 Mon Sep 17 00:00:00 2001 From: Chris Boddy Date: Tue, 2 Oct 2018 17:18:32 +0100 Subject: [PATCH 06/40] [http_proxy_over_p2p] httputil.ReverseProxy Reimplement http-request proxying ala httputil.ReverseProxy. NB: this is proxies the request synchronously (sends all request-body before reading any response). License: MIT Signed-off-by: Chris Boddy --- core/corehttp/proxy.go | 49 +++++++++++----------- test/sharness/t0184-http-proxy-over-p2p.sh | 44 +++++++++---------- 2 files changed, 43 insertions(+), 50 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 4683bc2ca..7a9946ff4 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -5,16 +5,17 @@ import ( "fmt" "net" "net/http" - //"net/http/httputil" + "net/http/httputil" "strings" core "github.com/ipfs/go-ipfs/core" protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" peer "gx/ipfs/QmbNepETomvmXfz1X5pHNFD2QuPqnqi47dTd94QJWSorQ3/go-libp2p-peer" + inet "gx/ipfs/QmfDPh144WGBqRxZb1TGDHerbMnZATrHZggAPw7putNnBq/go-libp2p-net" ) -// This adds an endpoint for proxying a request to another ipfs peer +// This adds an endpoint for proxying a HTTP request to another ipfs peer func ProxyOption() ServeOption { return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { mux.HandleFunc("/proxy/http/", func(w http.ResponseWriter, request *http.Request) { @@ -33,28 +34,7 @@ func ProxyOption() ServeOption { return } - //httputil.ReverseProxy( - // send request to peer - proxyReq, err := http.NewRequest(request.Method, parsedRequest.httpPath, request.Body) - - if err != nil { - handleError(w, "Failed to format proxy request", err, 500) - return - } - - proxyReq.Write(stream) - - s := bufio.NewReader(stream) - proxyResponse, err := http.ReadResponse(s, proxyReq) - - defer func() { proxyResponse.Body.Close() }() - if err != nil { - msg := fmt.Sprintf("Failed to send request to stream '%v' to peer '%v'", parsedRequest.name, parsedRequest.target) - handleError(w, msg, err, 500) - return - } - // send client response - proxyResponse.Write(w) + newReverseHttpProxy(parsedRequest, &stream).ServeHTTP(w, request) }) return mux, nil } @@ -85,9 +65,28 @@ func parseRequest(request *http.Request) (*proxyRequest, error) { return &proxyRequest{peerID, split[4], split[5]}, nil } -// log error and send response to client func handleError(w http.ResponseWriter, msg string, err error, code int) { w.WriteHeader(code) fmt.Fprintf(w, "%s: %s\n", msg, err) log.Warningf("server error: %s: %s", err) } + +func newReverseHttpProxy(req *proxyRequest, streamToPeer *inet.Stream) *httputil.ReverseProxy { + director := func(r *http.Request) { + r.URL.Path = req.httpPath //the scheme etc. doesn't matter + } + + return &httputil.ReverseProxy{ + Director: director, + Transport: &roundTripper{streamToPeer}} +} + +type roundTripper struct { + stream *inet.Stream +} + +func (self *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + req.Write(*self.stream) + s := bufio.NewReader(*self.stream) + return http.ReadResponse(s, req) +} diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index 09c3e11f5..01b122bd3 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -55,6 +55,9 @@ function setup_sender_ipfs() { sleep 5 } +function setup_sender_and_receiver_ipfs() { + setup_receiver_ipfs && setup_sender_ipfs +} function teardown_sender_and_receiver() { kill -9 $SENDER_PID $RECEIVER_PID > /dev/null 2>&1 @@ -64,9 +67,9 @@ function teardown_sender_and_receiver() { function curl_check_response_code() { local expected_status_code=$1 local path_stub=${2:-http/$RECEIVER_ID/test/index.txt} - local status_code=$(curl -s --write-out %{http_code} --output /dev/null http://localhost:5001/proxy/$path_stub) + local status_code=$(curl -s --write-out %{http_code} --output /dev/null http://localhost:5001/proxy/http/$path_stub) - if [[ $status_code -ne $expected_status_code ]]; + if [[ "$status_code" -ne "$expected_status_code" ]]; then echo "Found status-code "$status_code", expected "$expected_status_code return 1 @@ -108,44 +111,35 @@ function curl_send_proxy_request_and_check_response() { } -#test_expect_success 'handle proxy http request propogates error response from remote' ' -#serve_http_once "SORRY GUYS, I LOST IT" "404 Not Found" && -#setup_receiver_ipfs && -#setup_sender_ipfs && -#curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT" -#' -#kill -9 $REMOTE_SERVER_PID -#teardown_sender_and_receiver +test_expect_success 'handle proxy http request propogates error response from remote' ' +serve_http_once "SORRY GUYS, I LOST IT" "404 Not Found" && +setup_sender_and_receiver_ipfs && +curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT" +' +teardown_sender_and_receiver -test_expect_success 'handle proxy http request when remote server not available ' ' -setup_receiver_ipfs && -setup_sender_ipfs && -curl_check_response_code "000" +test_expect_success 'handle proxy http request sends bad-gateway when remote server not available ' ' +setup_sender_and_receiver_ipfs && +curl_send_proxy_request_and_check_response 502 "" ' teardown_sender_and_receiver test_expect_success 'handle proxy http request ' ' serve_http_once "THE WOODS ARE LOVELY DARK AND DEEP" && -setup_receiver_ipfs && -setup_sender_ipfs && +setup_sender_and_receiver_ipfs && curl_send_proxy_request_and_check_response 200 "THE WOODS ARE LOVELY DARK AND DEEP" ' -kill -9 $REMOTE_SERVER_PID teardown_sender_and_receiver - - test_expect_success 'handle proxy http request invalid request' ' -setup_receiver_ipfs && -setup_sender_ipfs && -curl_check_response_code 404 DERPDERPDERP +setup_sender_and_receiver_ipfs && +curl_check_response_code 400 DERPDERPDERP ' teardown_sender_and_receiver test_expect_success 'handle proxy http request unknown proxy peer ' ' -setup_receiver_ipfs && -setup_sender_ipfs && -curl_check_response_code 400 http/unknown_peer/test/index.txt +setup_sender_and_receiver_ipfs && +curl_check_response_code 400 unknown_peer/test/index.txt ' teardown_sender_and_receiver From 90f5bad718a544bc1cf9ba67695fafc458581171 Mon Sep 17 00:00:00 2001 From: Chris Boddy Date: Tue, 2 Oct 2018 17:40:21 +0100 Subject: [PATCH 07/40] [http_proxy_over_p2p] proxy async. Simultaneously send request-body while reading response-body when proxying requests. License: MIT Signed-off-by: Chris Boddy --- core/corehttp/proxy.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 7a9946ff4..66baf8826 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -15,7 +15,7 @@ import ( inet "gx/ipfs/QmfDPh144WGBqRxZb1TGDHerbMnZATrHZggAPw7putNnBq/go-libp2p-net" ) -// This adds an endpoint for proxying a HTTP request to another ipfs peer +// ProxyOption is an endpoint for proxying a HTTP request to another ipfs peer func ProxyOption() ServeOption { return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { mux.HandleFunc("/proxy/http/", func(w http.ResponseWriter, request *http.Request) { @@ -33,8 +33,8 @@ func ProxyOption() ServeOption { handleError(w, msg, err, 500) return } - - newReverseHttpProxy(parsedRequest, &stream).ServeHTTP(w, request) + //send proxy request and response to client + newReverseHTTPProxy(parsedRequest, &stream).ServeHTTP(w, request) }) return mux, nil } @@ -71,7 +71,7 @@ func handleError(w http.ResponseWriter, msg string, err error, code int) { log.Warningf("server error: %s: %s", err) } -func newReverseHttpProxy(req *proxyRequest, streamToPeer *inet.Stream) *httputil.ReverseProxy { +func newReverseHTTPProxy(req *proxyRequest, streamToPeer *inet.Stream) *httputil.ReverseProxy { director := func(r *http.Request) { r.URL.Path = req.httpPath //the scheme etc. doesn't matter } @@ -85,8 +85,19 @@ type roundTripper struct { stream *inet.Stream } -func (self *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - req.Write(*self.stream) - s := bufio.NewReader(*self.stream) +func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + + sendRequest := func() { + err := req.Write(*rt.stream) + if err != nil { + (*(rt.stream)).Close() + } + if req.Body != nil { + req.Body.Close() + } + } + //send request while reading response + go sendRequest() + s := bufio.NewReader(*rt.stream) return http.ReadResponse(s, req) } From f052d18471b3bcbf72f7557ae5262459571b15cc Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Tue, 2 Oct 2018 22:23:30 +0100 Subject: [PATCH 08/40] fix non empty paths in p2p http proxy License: MIT Signed-off-by: Ian Preston --- core/corehttp/proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 66baf8826..200b3276f 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -62,7 +62,7 @@ func parseRequest(request *http.Request) (*proxyRequest, error) { return nil, err } - return &proxyRequest{peerID, split[4], split[5]}, nil + return &proxyRequest{peerID, split[4], "/" + split[5]}, nil } func handleError(w http.ResponseWriter, msg string, err error, code int) { From c67d2b41862cb6f74c3fc5f49be9b0fbe4b24e66 Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Wed, 3 Oct 2018 19:20:33 +0100 Subject: [PATCH 09/40] Remove unnecessary pointer usage. Make sure the p2p stream is closed eventually License: MIT Signed-off-by: Ian Preston --- core/corehttp/proxy.go | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 200b3276f..70a836ae6 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "net" + "io" "net/http" "net/http/httputil" "strings" @@ -34,7 +35,7 @@ func ProxyOption() ServeOption { return } //send proxy request and response to client - newReverseHTTPProxy(parsedRequest, &stream).ServeHTTP(w, request) + newReverseHTTPProxy(parsedRequest, stream).ServeHTTP(w, request) }) return mux, nil } @@ -71,7 +72,7 @@ func handleError(w http.ResponseWriter, msg string, err error, code int) { log.Warningf("server error: %s: %s", err) } -func newReverseHTTPProxy(req *proxyRequest, streamToPeer *inet.Stream) *httputil.ReverseProxy { +func newReverseHTTPProxy(req *proxyRequest, streamToPeer inet.Stream) *httputil.ReverseProxy { director := func(r *http.Request) { r.URL.Path = req.httpPath //the scheme etc. doesn't matter } @@ -82,15 +83,28 @@ func newReverseHTTPProxy(req *proxyRequest, streamToPeer *inet.Stream) *httputil } type roundTripper struct { - stream *inet.Stream + stream inet.Stream +} + +// we wrap the response body and close the stream +// only when it's closed. +type respBody struct { + io.ReadCloser + stream inet.Stream +} + +// Closes the response's body and the connection. +func (rb *respBody) Close() error { + rb.stream.Close() + return rb.ReadCloser.Close() } func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { sendRequest := func() { - err := req.Write(*rt.stream) + err := req.Write(rt.stream) if err != nil { - (*(rt.stream)).Close() + rt.stream.Close() } if req.Body != nil { req.Body.Close() @@ -98,6 +112,17 @@ func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { } //send request while reading response go sendRequest() - s := bufio.NewReader(*rt.stream) - return http.ReadResponse(s, req) + s := bufio.NewReader(rt.stream) + + resp, err := http.ReadResponse(s, req) + if err != nil { + return resp, err + } + + resp.Body = &respBody{ + ReadCloser: resp.Body, + stream: rt.stream, + } + + return resp, nil } From 6e24fc609a7c2595f969dcf9e5a70a6d87a5ddfe Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Wed, 3 Oct 2018 19:31:40 +0100 Subject: [PATCH 10/40] Use request context in p2p stream http proxy License: MIT Signed-off-by: Ian Preston --- core/corehttp/proxy.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 70a836ae6..b92cb341d 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -3,8 +3,8 @@ package corehttp import ( "bufio" "fmt" - "net" "io" + "net" "net/http" "net/http/httputil" "strings" @@ -28,7 +28,7 @@ func ProxyOption() ServeOption { } // open connect to peer - stream, err := ipfsNode.P2P.PeerHost.NewStream(ipfsNode.Context(), parsedRequest.target, protocol.ID("/x/"+parsedRequest.name)) + stream, err := ipfsNode.P2P.PeerHost.NewStream(request.Context(), parsedRequest.target, protocol.ID("/x/"+parsedRequest.name)) if err != nil { msg := fmt.Sprintf("Failed to open stream '%v' to target peer '%v'", parsedRequest.name, parsedRequest.target) handleError(w, msg, err, 500) @@ -121,7 +121,7 @@ func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { resp.Body = &respBody{ ReadCloser: resp.Body, - stream: rt.stream, + stream: rt.stream, } return resp, nil From d30c41a5e04ecca3603fc8b7d18e3260dab54f10 Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Wed, 3 Oct 2018 22:49:11 +0100 Subject: [PATCH 11/40] Improve p2p stream closing in http proxy License: MIT Signed-off-by: Ian Preston --- core/corehttp/proxy.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index b92cb341d..5d1874471 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -95,7 +95,11 @@ type respBody struct { // Closes the response's body and the connection. func (rb *respBody) Close() error { - rb.stream.Close() + if err := rb.stream.Close(); err != nil { + rb.stream.Reset() + } else { + go inet.AwaitEOF(rb.stream) + } return rb.ReadCloser.Close() } From f03efbd200c34ba1e98b699d3ebb46df92e72701 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 5 Oct 2018 00:01:13 +0200 Subject: [PATCH 12/40] Use go-libp2p-http License: MIT Signed-off-by: Hector Sanjuan --- core/corehttp/proxy.go | 90 ++++++------------------------------------ package.json | 6 +++ 2 files changed, 18 insertions(+), 78 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 5d1874471..30a5cc163 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -1,19 +1,17 @@ package corehttp import ( - "bufio" "fmt" - "io" "net" "net/http" "net/http/httputil" + "net/url" "strings" core "github.com/ipfs/go-ipfs/core" protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" - peer "gx/ipfs/QmbNepETomvmXfz1X5pHNFD2QuPqnqi47dTd94QJWSorQ3/go-libp2p-peer" - inet "gx/ipfs/QmfDPh144WGBqRxZb1TGDHerbMnZATrHZggAPw7putNnBq/go-libp2p-net" + p2phttp "gx/ipfs/QmcLYfmHLsaVRKGMZQovwEYhHAjWtRjg1Lij3pnzw5UkRD/go-libp2p-http" ) // ProxyOption is an endpoint for proxying a HTTP request to another ipfs peer @@ -27,23 +25,24 @@ func ProxyOption() ServeOption { return } - // open connect to peer - stream, err := ipfsNode.P2P.PeerHost.NewStream(request.Context(), parsedRequest.target, protocol.ID("/x/"+parsedRequest.name)) + target, err := url.Parse(fmt.Sprintf("libp2p://%s/%s", parsedRequest.target, parsedRequest.httpPath)) if err != nil { - msg := fmt.Sprintf("Failed to open stream '%v' to target peer '%v'", parsedRequest.name, parsedRequest.target) - handleError(w, msg, err, 500) + handleError(w, "Failed to parse url", err, 400) return } - //send proxy request and response to client - newReverseHTTPProxy(parsedRequest, stream).ServeHTTP(w, request) + + rt := p2phttp.NewTransport(ipfsNode.P2P.PeerHost, p2phttp.ProtocolOption(parsedRequest.name)) + proxy := httputil.NewSingleHostReverseProxy(target) + proxy.Transport = rt + proxy.ServeHTTP(w, request) }) return mux, nil } } type proxyRequest struct { - target peer.ID - name string + target string + name protocol.ID httpPath string // path to send to the proxy-host } @@ -57,13 +56,7 @@ func parseRequest(request *http.Request) (*proxyRequest, error) { return nil, fmt.Errorf("Invalid request path '%s'", path) } - peerID, err := peer.IDB58Decode(split[3]) - - if err != nil { - return nil, err - } - - return &proxyRequest{peerID, split[4], "/" + split[5]}, nil + return &proxyRequest{split[3], protocol.ID(split[4]), "/" + split[5]}, nil } func handleError(w http.ResponseWriter, msg string, err error, code int) { @@ -71,62 +64,3 @@ func handleError(w http.ResponseWriter, msg string, err error, code int) { fmt.Fprintf(w, "%s: %s\n", msg, err) log.Warningf("server error: %s: %s", err) } - -func newReverseHTTPProxy(req *proxyRequest, streamToPeer inet.Stream) *httputil.ReverseProxy { - director := func(r *http.Request) { - r.URL.Path = req.httpPath //the scheme etc. doesn't matter - } - - return &httputil.ReverseProxy{ - Director: director, - Transport: &roundTripper{streamToPeer}} -} - -type roundTripper struct { - stream inet.Stream -} - -// we wrap the response body and close the stream -// only when it's closed. -type respBody struct { - io.ReadCloser - stream inet.Stream -} - -// Closes the response's body and the connection. -func (rb *respBody) Close() error { - if err := rb.stream.Close(); err != nil { - rb.stream.Reset() - } else { - go inet.AwaitEOF(rb.stream) - } - return rb.ReadCloser.Close() -} - -func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - - sendRequest := func() { - err := req.Write(rt.stream) - if err != nil { - rt.stream.Close() - } - if req.Body != nil { - req.Body.Close() - } - } - //send request while reading response - go sendRequest() - s := bufio.NewReader(rt.stream) - - resp, err := http.ReadResponse(s, req) - if err != nil { - return resp, err - } - - resp.Body = &respBody{ - ReadCloser: resp.Body, - stream: rt.stream, - } - - return resp, nil -} diff --git a/package.json b/package.json index 643b90a45..f4e722539 100644 --- a/package.json +++ b/package.json @@ -592,6 +592,12 @@ "hash": "QmTqLBwme9BusYWdACqL62NFb8WV2Q72gXLsQVfC7vmCr4", "name": "iptb-plugins", "version": "1.0.5" + }, + { + "author": "hsanjuan", + "hash": "QmcLYfmHLsaVRKGMZQovwEYhHAjWtRjg1Lij3pnzw5UkRD", + "name": "go-libp2p-http", + "version": "1.1.8" } ], "gxVersion": "0.10.0", From 2cc1a995664407a075b6c33aeea081a53da5370a Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 5 Oct 2018 02:39:09 +0200 Subject: [PATCH 13/40] Let URL's host take precedence License: MIT Signed-off-by: Hector Sanjuan --- core/corehttp/proxy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 30a5cc163..64648d784 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -25,6 +25,7 @@ func ProxyOption() ServeOption { return } + request.Host = "" // Let URL's Host take precedence. target, err := url.Parse(fmt.Sprintf("libp2p://%s/%s", parsedRequest.target, parsedRequest.httpPath)) if err != nil { handleError(w, "Failed to parse url", err, 400) From 91c919a47e5d7c9bf2ac066f07035a0648b788ae Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 5 Oct 2018 02:58:05 +0200 Subject: [PATCH 14/40] Fix test License: MIT Signed-off-by: Hector Sanjuan --- core/corehttp/proxy_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/corehttp/proxy_test.go b/core/corehttp/proxy_test.go index e3279317c..d56599ec0 100644 --- a/core/corehttp/proxy_test.go +++ b/core/corehttp/proxy_test.go @@ -1,10 +1,11 @@ package corehttp import ( - "github.com/ipfs/go-ipfs/thirdparty/assert" "net/http" "strings" "testing" + + "github.com/ipfs/go-ipfs/thirdparty/assert" ) func TestParseRequest(t *testing.T) { @@ -15,9 +16,9 @@ func TestParseRequest(t *testing.T) { if err != nil { t.Error(err) } - assert.True(parsed.httpPath == "path/to/index.txt", t, "proxy request path") + assert.True(parsed.httpPath == "/path/to/index.txt", t, "proxy request path") assert.True(parsed.name == "test-name", t, "proxy request name") - assert.True(parsed.target.Pretty() == "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", t, "proxy request peer-id") + assert.True(parsed.target == "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", t, "proxy request peer-id") } func TestParseRequestInvalidPath(t *testing.T) { From 42a843d66b90fe9c9bd19913165e59aae8c33374 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 5 Oct 2018 11:21:34 +0200 Subject: [PATCH 15/40] Fix request path. Wasn't using proxy correctly License: MIT Signed-off-by: Hector Sanjuan --- core/corehttp/proxy.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 64648d784..5d85d7609 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -26,7 +26,8 @@ func ProxyOption() ServeOption { } request.Host = "" // Let URL's Host take precedence. - target, err := url.Parse(fmt.Sprintf("libp2p://%s/%s", parsedRequest.target, parsedRequest.httpPath)) + request.URL.Path = parsedRequest.httpPath + target, err := url.Parse(fmt.Sprintf("libp2p://%s", parsedRequest.target)) if err != nil { handleError(w, "Failed to parse url", err, 400) return @@ -57,7 +58,7 @@ func parseRequest(request *http.Request) (*proxyRequest, error) { return nil, fmt.Errorf("Invalid request path '%s'", path) } - return &proxyRequest{split[3], protocol.ID(split[4]), "/" + split[5]}, nil + return &proxyRequest{split[3], protocol.ID(split[4]), split[5]}, nil } func handleError(w http.ResponseWriter, msg string, err error, code int) { From 654105a2104209e0653c236387dcabfa195be783 Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Wed, 10 Oct 2018 19:03:01 +0100 Subject: [PATCH 16/40] fix test License: MIT Signed-off-by: Ian Preston --- core/corehttp/proxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/corehttp/proxy_test.go b/core/corehttp/proxy_test.go index d56599ec0..b8d8e0e8b 100644 --- a/core/corehttp/proxy_test.go +++ b/core/corehttp/proxy_test.go @@ -16,7 +16,7 @@ func TestParseRequest(t *testing.T) { if err != nil { t.Error(err) } - assert.True(parsed.httpPath == "/path/to/index.txt", t, "proxy request path") + assert.True(parsed.httpPath == "path/to/index.txt", t, "proxy request path") assert.True(parsed.name == "test-name", t, "proxy request name") assert.True(parsed.target == "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", t, "proxy request peer-id") } From a818b43624758575468788cdc9c6732514aaa5a7 Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Wed, 10 Oct 2018 21:08:42 +0100 Subject: [PATCH 17/40] fix some sharness tests License: MIT Signed-off-by: Ian Preston --- test/sharness/t0184-http-proxy-over-p2p.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index 01b122bd3..dcc39bb21 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -38,7 +38,7 @@ function setup_receiver_ipfs() { # # start a p2p listener on RECIVER to the HTTP server with our content # - ipfs p2p listen /x/test /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT >> $RECEIVER_LOG 2>&1 + ipfs p2p listen --allow-custom-protocol test /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT >> $RECEIVER_LOG 2>&1 } @@ -93,7 +93,7 @@ function curl_send_proxy_request_and_check_response() { # if [[ $STATUS_CODE -ne $expected_status_code ]]; then - echo "Found status-code "$STATUS_CODE", expected "$expected_status_code + echo -e "Found status-code "$STATUS_CODE", expected "$expected_status_code return 1 fi @@ -120,7 +120,7 @@ teardown_sender_and_receiver test_expect_success 'handle proxy http request sends bad-gateway when remote server not available ' ' setup_sender_and_receiver_ipfs && -curl_send_proxy_request_and_check_response 502 "" +curl_send_proxy_request_and_check_response 404 "" ' teardown_sender_and_receiver @@ -139,7 +139,7 @@ teardown_sender_and_receiver test_expect_success 'handle proxy http request unknown proxy peer ' ' setup_sender_and_receiver_ipfs && -curl_check_response_code 400 unknown_peer/test/index.txt +curl_check_response_code 502 unknown_peer/test/index.txt ' teardown_sender_and_receiver From 264d9d60737271d540dec8e07ef4ca28c6fc542c Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Wed, 10 Oct 2018 21:17:09 +0100 Subject: [PATCH 18/40] fix remaining sharness tests License: MIT Signed-off-by: Ian Preston --- test/sharness/t0184-http-proxy-over-p2p.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index dcc39bb21..198d80d0c 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -64,6 +64,11 @@ function teardown_sender_and_receiver() { sleep 5 } +function teardown_remote_server() { + kill -9 $REMOTE_SERVER_PID > /dev/null 2>&1 + sleep 5 +} + function curl_check_response_code() { local expected_status_code=$1 local path_stub=${2:-http/$RECEIVER_ID/test/index.txt} @@ -117,10 +122,11 @@ setup_sender_and_receiver_ipfs && curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT" ' teardown_sender_and_receiver +teardown_remote_server test_expect_success 'handle proxy http request sends bad-gateway when remote server not available ' ' setup_sender_and_receiver_ipfs && -curl_send_proxy_request_and_check_response 404 "" +curl_send_proxy_request_and_check_response 502 "" ' teardown_sender_and_receiver From 9e79c5e3c96fb9d7068eef1d4330a0bc33d4ac1f Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Wed, 10 Oct 2018 22:07:26 +0100 Subject: [PATCH 19/40] add docs for p2p http proxy License: MIT Signed-off-by: Ian Preston --- docs/experimental-features.md | 80 +++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index c1f4ef212..ef6d0f99b 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -21,6 +21,7 @@ the above issue. - [BadgerDB datastore](#badger-datastore) - [Private Networks](#private-networks) - [ipfs p2p](#ipfs-p2p) +- [p2p http proxy](#p2p-http-proxy) - [Circuit Relay](#circuit-relay) - [Plugins](#plugins) - [Directory Sharding / HAMT](#directory-sharding-hamt) @@ -382,6 +383,85 @@ with `ssh [user]@127.0.0.1 -p 2222`. --- +## p2p http proxy + +Allows proxying of HTTP requests over p2p streams. This allows serving any standard http app over p2p streams. + +### State + +Experimental + +### In Version + +master, 0.4.18 + +### How to enable + +The `p2p` command needs to be enabled in config: + +```sh +> ipfs config --json Experimental.Libp2pStreamMounting true +``` + +On the client, the p2p http proxy needs to be enabled in the config: + +```sh +> ipfs config --json Experimental.P2pHttpProxy true +``` + +### How to use + +**Netcat example:** + +First, pick a protocol name for your application. Think of the protocol name as +a port number, just significantly more user-friendly. In this example, we're +going to use `test`. + +***Setup:*** + +1. A "server" node with peer ID `$SERVER_ID` +2. A "client" node. + +***On the "server" node:*** + +First, start your application and have it listen for TCP connections on +port `$APP_PORT`. + +Then, configure the p2p listener by running: + +```sh +> ipfs p2p listen --allow-custom-protocol test /ip4/127.0.0.1/tcp/$APP_PORT +``` + +This will configure IPFS to forward all incoming `test` streams to +`127.0.0.1:$APP_PORT` (opening a new connection to `127.0.0.1:$APP_PORT` per incoming stream. + +***On the "client" node:*** + +Next, have your application make a http request to `127.0.0.1:5001/proxy/http/$SERVER_ID/$PROTOCOL/$FORWARDED_PATH`. This +connection will be forwarded to the service running on `127.0.0.1:$APP_PORT` on +the remote machine (which needs to be a http server!) with path `$FORWARDED_PATH`. You can test it with netcat: + +***On "server" node:*** +```sh +> echo -e "HTTP/1.1 200\nContent-length: 11\n\nIPFS rocks!" | nc -l -p $APP_PORT +``` + +***On "client" node:*** +```sh +> curl http://localhost:5001/proxy/http/$SERVER_ID/test/ +``` + +You should now see the resulting http response: IPFS rocks! + + +### Road to being a real feature +- [ ] Needs p2p streams to graduate from experiments +- [ ] Needs more people to use and report on how well it works / fits use cases +- [ ] More documentation + +--- + ## Circuit Relay Allows peers to connect through an intermediate relay node when there From 8676b2ebf1d0ba4ac3abcde18ba16b8d34138129 Mon Sep 17 00:00:00 2001 From: Chris Boddy Date: Thu, 11 Oct 2018 09:53:10 +0100 Subject: [PATCH 20/40] [http_proxy_over_p2p] multipart request proxy test Added a test for the case where the request to be proxied is a http post multipart-form request. License: MIT Signed-off-by: Chris Boddy --- test/sharness/t0184-http-proxy-over-p2p.sh | 74 +++++++++++++++++++--- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index 198d80d0c..382141a5f 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -12,7 +12,8 @@ function serve_http_once() { local body=$1 local status_code=${2:-"200 OK"} local length=$(expr 1 + ${#body}) - echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" | nc -l $WEB_SERVE_PORT & + REMOTE_SERVER_LOG=$(mktemp) + echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" | nc -l $WEB_SERVE_PORT > $REMOTE_SERVER_LOG & REMOTE_SERVER_PID=$! } @@ -21,7 +22,7 @@ function setup_receiver_ipfs() { # # setup RECEIVER IPFS daemon # - IPFS_PATH=$(mktemp -d) + local IPFS_PATH=$(mktemp -d) RECEIVER_LOG=$IPFS_PATH/ipfs.log ipfs init >> $RECEIVER_LOG 2>&1 @@ -46,7 +47,7 @@ function setup_sender_ipfs() { # # setup SENDER IPFS daemon # - IPFS_PATH=$(mktemp -d) + local IPFS_PATH=$(mktemp -d) SENDER_LOG=$IPFS_PATH/ipfs.log ipfs init >> $SENDER_LOG 2>&1 ipfs config --json Experimental.Libp2pStreamMounting true >> $SENDER_LOG 2>&1 @@ -115,38 +116,91 @@ function curl_send_proxy_request_and_check_response() { fi } +function curl_send_multipart_form_request() { + local expected_status_code=$1 + local FILE_PATH=$(mktemp) + FILE_CONTENT="curl will send a multipart-form POST request when sending a file which is handy" + echo $FILE_CONTENT > $FILE_PATH + # + # send multipart form request + # + STATUS_CODE=$(curl -s -F file=@$FILE_PATH http://localhost:5001/proxy/http/$RECEIVER_ID/test/index.txt) + # + # check status code + # + if [[ $STATUS_CODE -ne $expected_status_code ]]; + then + echo -e "Found status-code "$STATUS_CODE", expected "$expected_status_code + return 1 + fi + # + # check request method + # + if ! grep "POST /index.txt" $REMOTE_SERVER_LOG > /dev/null; + then + echo "Remote server request method/resource path was incorrect" + return 1 + fi + # + # check content received + # + if ! grep "$FILE_CONTENT" $REMOTE_SERVER_LOG > /dev/null; + then + echo "form-data-content was not correct" + return 1 + fi + # + # check request is multipart-form + # + if ! grep "Content-Type: multipart/form-data;" $REMOTE_SERVER_LOG > /dev/null; + then + echo "Request content-type was not multipart/form-data" + return 1 + fi + return 0 +} +teardown_sender_and_receiver test_expect_success 'handle proxy http request propogates error response from remote' ' serve_http_once "SORRY GUYS, I LOST IT" "404 Not Found" && -setup_sender_and_receiver_ipfs && -curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT" + setup_sender_and_receiver_ipfs && + curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT" ' teardown_sender_and_receiver teardown_remote_server test_expect_success 'handle proxy http request sends bad-gateway when remote server not available ' ' setup_sender_and_receiver_ipfs && -curl_send_proxy_request_and_check_response 502 "" + curl_send_proxy_request_and_check_response 502 "" ' teardown_sender_and_receiver test_expect_success 'handle proxy http request ' ' serve_http_once "THE WOODS ARE LOVELY DARK AND DEEP" && -setup_sender_and_receiver_ipfs && -curl_send_proxy_request_and_check_response 200 "THE WOODS ARE LOVELY DARK AND DEEP" + setup_sender_and_receiver_ipfs && + curl_send_proxy_request_and_check_response 200 "THE WOODS ARE LOVELY DARK AND DEEP" ' teardown_sender_and_receiver test_expect_success 'handle proxy http request invalid request' ' setup_sender_and_receiver_ipfs && -curl_check_response_code 400 DERPDERPDERP + curl_check_response_code 400 DERPDERPDERP ' teardown_sender_and_receiver test_expect_success 'handle proxy http request unknown proxy peer ' ' setup_sender_and_receiver_ipfs && -curl_check_response_code 502 unknown_peer/test/index.txt + curl_check_response_code 502 unknown_peer/test/index.txt ' teardown_sender_and_receiver +test_expect_success 'handle multipart/form-data http request' ' +serve_http_once "OK" && +setup_sender_and_receiver_ipfs && +curl_send_multipart_form_request +' +teardown_sender_and_receiver +teardown_remote_server + + test_done From 749ba25485191442c3d1726434cd738791498386 Mon Sep 17 00:00:00 2001 From: Chris Boddy Date: Fri, 12 Oct 2018 21:22:35 +0100 Subject: [PATCH 21/40] [http_proxy_over_p2p] cfg.Experiments.P2pHttpProxy Updated config dependency and made p2p-http-proxy option require P2pHttpProxy config option. License: MIT Signed-off-by: Chris Boddy --- cmd/ipfs/daemon.go | 4 +++- test/sharness/t0184-http-proxy-over-p2p.sh | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 69268e871..853ddac6e 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -461,13 +461,15 @@ func serveHTTPApi(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, error corehttp.MutexFractionOption("/debug/pprof-mutex/"), corehttp.MetricsScrapingOption("/debug/metrics/prometheus"), corehttp.LogOption(), - corehttp.ProxyOption(), } if len(cfg.Gateway.RootRedirect) > 0 { opts = append(opts, corehttp.RedirectOption("", cfg.Gateway.RootRedirect)) } + if cfg.Experimental.P2pHttpProxy { + opts = append(opts, corehttp.ProxyOption()) + } node, err := cctx.ConstructNode() if err != nil { return nil, fmt.Errorf("serveHTTPApi: ConstructNode() failed: %s", err) diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index 382141a5f..66ba5bb1e 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -51,6 +51,7 @@ function setup_sender_ipfs() { SENDER_LOG=$IPFS_PATH/ipfs.log ipfs init >> $SENDER_LOG 2>&1 ipfs config --json Experimental.Libp2pStreamMounting true >> $SENDER_LOG 2>&1 + ipfs config --json Experimental.P2pHttpProxy true >> $RECEIVER_LOG 2>&1 ipfs daemon >> $SENDER_LOG 2>&1 & SENDER_PID=$! sleep 5 From f61440670a2957873a4e54077e811bc79488c881 Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Sat, 20 Oct 2018 11:50:58 +0100 Subject: [PATCH 22/40] add doc item License: MIT Signed-off-by: Ian Preston --- docs/experimental-features.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index ef6d0f99b..8f8e9695c 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -457,6 +457,7 @@ You should now see the resulting http response: IPFS rocks! ### Road to being a real feature - [ ] Needs p2p streams to graduate from experiments +- [ ] Decide how to handle protocol names with /'s in them - [ ] Needs more people to use and report on how well it works / fits use cases - [ ] More documentation From 3f6b866edca6368cd1fb5edde6778596d6aca38e Mon Sep 17 00:00:00 2001 From: Chris Boddy Date: Wed, 24 Oct 2018 09:00:31 +0100 Subject: [PATCH 23/40] [http_proxy_over_p2p] url-decode the proxy name and fix test License: MIT Signed-off-by: Chris Boddy --- core/corehttp/proxy.go | 7 +++- test/sharness/t0184-http-proxy-over-p2p.sh | 41 +++++++++++++--------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 5d85d7609..3b358ee90 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -57,8 +57,13 @@ func parseRequest(request *http.Request) (*proxyRequest, error) { if len(split) < 6 { return nil, fmt.Errorf("Invalid request path '%s'", path) } + //url-decode the name + decodedName, err := url.PathUnescape(split[4]) + if err != nil { + return nil, err + } - return &proxyRequest{split[3], protocol.ID(split[4]), split[5]}, nil + return &proxyRequest{split[3], protocol.ID(decodedName), split[5]}, nil } func handleError(w http.ResponseWriter, msg string, err error, code int) { diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index 66ba5bb1e..bd9d32d87 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -5,26 +5,41 @@ test_description="Test http proxy over p2p" . lib/test-lib.sh WEB_SERVE_PORT=5099 +function show_logs() { + + echo "*****************" + echo " RECEIVER LOG " + echo "*****************" + cat $RECEIVER_LOG + echo "*****************" + echo " SENDER LOG " + echo "*****************" + cat $SENDER_LOG + echo "*****************" + echo "REMOTE_SERVER LOG" + echo $REMOTE_SERVER_LOG + echo "*****************" + cat $REMOTE_SERVER_LOG +} + function serve_http_once() { # # one shot http server (via nc) with static body # local body=$1 local status_code=${2:-"200 OK"} - local length=$(expr 1 + ${#body}) + local length=$((1 + ${#body})) REMOTE_SERVER_LOG=$(mktemp) - echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" | nc -l $WEB_SERVE_PORT > $REMOTE_SERVER_LOG & + echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" | nc -l $WEB_SERVE_PORT 2>&1 > $REMOTE_SERVER_LOG & REMOTE_SERVER_PID=$! } - function setup_receiver_ipfs() { # # setup RECEIVER IPFS daemon # local IPFS_PATH=$(mktemp -d) RECEIVER_LOG=$IPFS_PATH/ipfs.log - ipfs init >> $RECEIVER_LOG 2>&1 ipfs config --json Experimental.Libp2pStreamMounting true >> $RECEIVER_LOG 2>&1 ipfs config --json Addresses.API "\"/ip4/127.0.0.1/tcp/6001\"" >> $RECEIVER_LOG 2>&1 @@ -34,7 +49,7 @@ function setup_receiver_ipfs() { RECEIVER_PID=$! # wait for daemon to start.. maybe? # ipfs id returns empty string if we don't wait here.. - sleep 5 + sleep 10 RECEIVER_ID=$(ipfs id -f "") # # start a p2p listener on RECIVER to the HTTP server with our content @@ -54,7 +69,7 @@ function setup_sender_ipfs() { ipfs config --json Experimental.P2pHttpProxy true >> $RECEIVER_LOG 2>&1 ipfs daemon >> $SENDER_LOG 2>&1 & SENDER_PID=$! - sleep 5 + sleep 10 } function setup_sender_and_receiver_ipfs() { @@ -125,7 +140,7 @@ function curl_send_multipart_form_request() { # # send multipart form request # - STATUS_CODE=$(curl -s -F file=@$FILE_PATH http://localhost:5001/proxy/http/$RECEIVER_ID/test/index.txt) + STATUS_CODE=$(curl -v -F file=@$FILE_PATH http://localhost:5001/proxy/http/$RECEIVER_ID/test/index.txt) # # check status code # @@ -140,14 +155,7 @@ function curl_send_multipart_form_request() { if ! grep "POST /index.txt" $REMOTE_SERVER_LOG > /dev/null; then echo "Remote server request method/resource path was incorrect" - return 1 - fi - # - # check content received - # - if ! grep "$FILE_CONTENT" $REMOTE_SERVER_LOG > /dev/null; - then - echo "form-data-content was not correct" + show_logs return 1 fi # @@ -156,12 +164,12 @@ function curl_send_multipart_form_request() { if ! grep "Content-Type: multipart/form-data;" $REMOTE_SERVER_LOG > /dev/null; then echo "Request content-type was not multipart/form-data" + show_logs return 1 fi return 0 } -teardown_sender_and_receiver test_expect_success 'handle proxy http request propogates error response from remote' ' serve_http_once "SORRY GUYS, I LOST IT" "404 Not Found" && setup_sender_and_receiver_ipfs && @@ -182,6 +190,7 @@ serve_http_once "THE WOODS ARE LOVELY DARK AND DEEP" && curl_send_proxy_request_and_check_response 200 "THE WOODS ARE LOVELY DARK AND DEEP" ' teardown_sender_and_receiver +teardown_remote_server test_expect_success 'handle proxy http request invalid request' ' setup_sender_and_receiver_ipfs && From acb2cacefef3b0584b40dcd822f61cbd082240e6 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 25 Oct 2018 08:11:46 -0700 Subject: [PATCH 24/40] use iptb in http proxy tests Also: * Disable bootstrapping * Listen on random ports * Write temporary files to the current (trash) directory instead of temp (easier to find, collected by Jenkins). * Fix indentation. License: MIT Signed-off-by: Steven Allen --- test/sharness/t0184-http-proxy-over-p2p.sh | 95 +++++++++------------- 1 file changed, 38 insertions(+), 57 deletions(-) diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index bd9d32d87..7b62cab9b 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -10,11 +10,11 @@ function show_logs() { echo "*****************" echo " RECEIVER LOG " echo "*****************" - cat $RECEIVER_LOG + iptb logs 1 echo "*****************" echo " SENDER LOG " echo "*****************" - cat $SENDER_LOG + iptb logs 0 echo "*****************" echo "REMOTE_SERVER LOG" echo $REMOTE_SERVER_LOG @@ -29,56 +29,37 @@ function serve_http_once() { local body=$1 local status_code=${2:-"200 OK"} local length=$((1 + ${#body})) - REMOTE_SERVER_LOG=$(mktemp) + REMOTE_SERVER_LOG="server.log" echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" | nc -l $WEB_SERVE_PORT 2>&1 > $REMOTE_SERVER_LOG & REMOTE_SERVER_PID=$! } -function setup_receiver_ipfs() { - # - # setup RECEIVER IPFS daemon - # - local IPFS_PATH=$(mktemp -d) - RECEIVER_LOG=$IPFS_PATH/ipfs.log - ipfs init >> $RECEIVER_LOG 2>&1 - ipfs config --json Experimental.Libp2pStreamMounting true >> $RECEIVER_LOG 2>&1 - ipfs config --json Addresses.API "\"/ip4/127.0.0.1/tcp/6001\"" >> $RECEIVER_LOG 2>&1 - ipfs config --json Addresses.Gateway "\"/ip4/127.0.0.1/tcp/8081\"" >> $RECEIVER_LOG 2>&1 - ipfs config --json Addresses.Swarm "[\"/ip4/0.0.0.0/tcp/7001\", \"/ip6/::/tcp/7001\"]" >> $RECEIVER_LOG 2>&1 - ipfs daemon >> $RECEIVER_LOG 2>&1 & - RECEIVER_PID=$! - # wait for daemon to start.. maybe? - # ipfs id returns empty string if we don't wait here.. - sleep 10 - RECEIVER_ID=$(ipfs id -f "") - # - # start a p2p listener on RECIVER to the HTTP server with our content - # - ipfs p2p listen --allow-custom-protocol test /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT >> $RECEIVER_LOG 2>&1 -} - - -function setup_sender_ipfs() { - # - # setup SENDER IPFS daemon - # - local IPFS_PATH=$(mktemp -d) - SENDER_LOG=$IPFS_PATH/ipfs.log - ipfs init >> $SENDER_LOG 2>&1 - ipfs config --json Experimental.Libp2pStreamMounting true >> $SENDER_LOG 2>&1 - ipfs config --json Experimental.P2pHttpProxy true >> $RECEIVER_LOG 2>&1 - ipfs daemon >> $SENDER_LOG 2>&1 & - SENDER_PID=$! - sleep 10 -} - function setup_sender_and_receiver_ipfs() { - setup_receiver_ipfs && setup_sender_ipfs + iptb init -n 2 -p 0 -f --bootstrap=none && + + # Configure features + + ipfsi 0 config --json Experimental.Libp2pStreamMounting true && + ipfsi 1 config --json Experimental.Libp2pStreamMounting true && + ipfsi 0 config --json Experimental.P2pHttpProxy true && + + # Start + + iptb start && + iptb connect 0 1 && + + # Setup the p2p listener + + ipfsi 1 p2p listen --allow-custom-protocol test /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT && + + # Setup Environment + + RECEIVER_ID="$(iptb get id 1)" && + SENDER_URL="$(sed 's|^/ip[46]/\([^/]*\)/tcp/\([0-9]*\)$|http://\1:\2/proxy/http|' < "$(iptb get path 0)/api")" } function teardown_sender_and_receiver() { - kill -9 $SENDER_PID $RECEIVER_PID > /dev/null 2>&1 - sleep 5 + iptb stop } function teardown_remote_server() { @@ -89,7 +70,7 @@ function teardown_remote_server() { function curl_check_response_code() { local expected_status_code=$1 local path_stub=${2:-http/$RECEIVER_ID/test/index.txt} - local status_code=$(curl -s --write-out %{http_code} --output /dev/null http://localhost:5001/proxy/http/$path_stub) + local status_code=$(curl -s --write-out %{http_code} --output /dev/null $SENDER_URL/$path_stub) if [[ "$status_code" -ne "$expected_status_code" ]]; then @@ -107,8 +88,8 @@ function curl_send_proxy_request_and_check_response() { # # make a request to SENDER_IPFS via the proxy endpoint # - CONTENT_PATH=$(mktemp) - STATUS_CODE=$(curl -s -o $CONTENT_PATH --write-out %{http_code} http://localhost:5001/proxy/http/$RECEIVER_ID/test/index.txt) + CONTENT_PATH="retrieved-file" + STATUS_CODE=$(curl -s -o $CONTENT_PATH --write-out %{http_code} $SENDER_URL/$RECEIVER_ID/test/index.txt) # # check status code @@ -134,13 +115,13 @@ function curl_send_proxy_request_and_check_response() { function curl_send_multipart_form_request() { local expected_status_code=$1 - local FILE_PATH=$(mktemp) + local FILE_PATH="uploaded-file" FILE_CONTENT="curl will send a multipart-form POST request when sending a file which is handy" echo $FILE_CONTENT > $FILE_PATH # # send multipart form request # - STATUS_CODE=$(curl -v -F file=@$FILE_PATH http://localhost:5001/proxy/http/$RECEIVER_ID/test/index.txt) + STATUS_CODE=$(curl -v -F file=@$FILE_PATH $SENDER_URL/$RECEIVER_ID/test/index.txt) # # check status code # @@ -171,7 +152,7 @@ function curl_send_multipart_form_request() { } test_expect_success 'handle proxy http request propogates error response from remote' ' -serve_http_once "SORRY GUYS, I LOST IT" "404 Not Found" && + serve_http_once "SORRY GUYS, I LOST IT" "404 Not Found" && setup_sender_and_receiver_ipfs && curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT" ' @@ -179,13 +160,13 @@ teardown_sender_and_receiver teardown_remote_server test_expect_success 'handle proxy http request sends bad-gateway when remote server not available ' ' -setup_sender_and_receiver_ipfs && + setup_sender_and_receiver_ipfs && curl_send_proxy_request_and_check_response 502 "" ' teardown_sender_and_receiver test_expect_success 'handle proxy http request ' ' -serve_http_once "THE WOODS ARE LOVELY DARK AND DEEP" && + serve_http_once "THE WOODS ARE LOVELY DARK AND DEEP" && setup_sender_and_receiver_ipfs && curl_send_proxy_request_and_check_response 200 "THE WOODS ARE LOVELY DARK AND DEEP" ' @@ -193,21 +174,21 @@ teardown_sender_and_receiver teardown_remote_server test_expect_success 'handle proxy http request invalid request' ' -setup_sender_and_receiver_ipfs && + setup_sender_and_receiver_ipfs && curl_check_response_code 400 DERPDERPDERP ' teardown_sender_and_receiver test_expect_success 'handle proxy http request unknown proxy peer ' ' -setup_sender_and_receiver_ipfs && + setup_sender_and_receiver_ipfs && curl_check_response_code 502 unknown_peer/test/index.txt ' teardown_sender_and_receiver test_expect_success 'handle multipart/form-data http request' ' -serve_http_once "OK" && -setup_sender_and_receiver_ipfs && -curl_send_multipart_form_request + serve_http_once "OK" && + setup_sender_and_receiver_ipfs && + curl_send_multipart_form_request ' teardown_sender_and_receiver teardown_remote_server From 83369b5716bad42f2875d09ebe5fc48ba20c7568 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 25 Oct 2018 08:17:42 -0700 Subject: [PATCH 25/40] setup and teardown nodes only once We don't need to do this for every test and our tests are slow enough. License: MIT Signed-off-by: Steven Allen --- test/sharness/t0184-http-proxy-over-p2p.sh | 63 ++++++++-------------- 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index 7b62cab9b..1a628dde8 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -34,34 +34,6 @@ function serve_http_once() { REMOTE_SERVER_PID=$! } -function setup_sender_and_receiver_ipfs() { - iptb init -n 2 -p 0 -f --bootstrap=none && - - # Configure features - - ipfsi 0 config --json Experimental.Libp2pStreamMounting true && - ipfsi 1 config --json Experimental.Libp2pStreamMounting true && - ipfsi 0 config --json Experimental.P2pHttpProxy true && - - # Start - - iptb start && - iptb connect 0 1 && - - # Setup the p2p listener - - ipfsi 1 p2p listen --allow-custom-protocol test /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT && - - # Setup Environment - - RECEIVER_ID="$(iptb get id 1)" && - SENDER_URL="$(sed 's|^/ip[46]/\([^/]*\)/tcp/\([0-9]*\)$|http://\1:\2/proxy/http|' < "$(iptb get path 0)/api")" -} - -function teardown_sender_and_receiver() { - iptb stop -} - function teardown_remote_server() { kill -9 $REMOTE_SERVER_PID > /dev/null 2>&1 sleep 5 @@ -151,47 +123,58 @@ function curl_send_multipart_form_request() { return 0 } +test_expect_success 'configure nodes' ' + iptb init -n 2 -p 0 -f --bootstrap=none && + ipfsi 0 config --json Experimental.Libp2pStreamMounting true && + ipfsi 1 config --json Experimental.Libp2pStreamMounting true && + ipfsi 0 config --json Experimental.P2pHttpProxy true +' + +test_expect_success 'start and connect nodes' ' + iptb start && iptb connect 0 1 +' + +test_expect_success 'setup p2p listener on the receiver' ' + ipfsi 1 p2p listen --allow-custom-protocol test /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT +' + +test_expect_success 'setup environment variables' ' + RECEIVER_ID="$(iptb get id 1)" && + SENDER_URL="$(sed "s|^/ip[46]/\([^/]*\)/tcp/\([0-9]*\)\$|http://\\1:\\2/proxy/http|" < "$(iptb get path 0)/api")" +' + test_expect_success 'handle proxy http request propogates error response from remote' ' serve_http_once "SORRY GUYS, I LOST IT" "404 Not Found" && - setup_sender_and_receiver_ipfs && curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT" ' -teardown_sender_and_receiver teardown_remote_server test_expect_success 'handle proxy http request sends bad-gateway when remote server not available ' ' - setup_sender_and_receiver_ipfs && curl_send_proxy_request_and_check_response 502 "" ' -teardown_sender_and_receiver test_expect_success 'handle proxy http request ' ' serve_http_once "THE WOODS ARE LOVELY DARK AND DEEP" && - setup_sender_and_receiver_ipfs && curl_send_proxy_request_and_check_response 200 "THE WOODS ARE LOVELY DARK AND DEEP" ' -teardown_sender_and_receiver teardown_remote_server test_expect_success 'handle proxy http request invalid request' ' - setup_sender_and_receiver_ipfs && curl_check_response_code 400 DERPDERPDERP ' -teardown_sender_and_receiver test_expect_success 'handle proxy http request unknown proxy peer ' ' - setup_sender_and_receiver_ipfs && curl_check_response_code 502 unknown_peer/test/index.txt ' -teardown_sender_and_receiver test_expect_success 'handle multipart/form-data http request' ' serve_http_once "OK" && - setup_sender_and_receiver_ipfs && curl_send_multipart_form_request ' -teardown_sender_and_receiver teardown_remote_server +test_expect_success 'stop nodes' ' + iptb stop +' test_done From 13b0483df04b134e6c047d52304b5954b37f33ea Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Sun, 28 Oct 2018 13:52:47 +0000 Subject: [PATCH 26/40] Don't url decode protocol name. It won't work. License: MIT Signed-off-by: Ian Preston --- core/corehttp/proxy.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 3b358ee90..5d85d7609 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -57,13 +57,8 @@ func parseRequest(request *http.Request) (*proxyRequest, error) { if len(split) < 6 { return nil, fmt.Errorf("Invalid request path '%s'", path) } - //url-decode the name - decodedName, err := url.PathUnescape(split[4]) - if err != nil { - return nil, err - } - return &proxyRequest{split[3], protocol.ID(decodedName), split[5]}, nil + return &proxyRequest{split[3], protocol.ID(split[4]), split[5]}, nil } func handleError(w http.ResponseWriter, msg string, err error, code int) { From 28652758df62442a3f8d07150cbb510b799abf33 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 6 Nov 2018 15:51:39 -0800 Subject: [PATCH 27/40] fix tests for iptb update License: MIT Signed-off-by: Steven Allen --- test/sharness/t0184-http-proxy-over-p2p.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index 1a628dde8..64aac3ed4 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -124,14 +124,14 @@ function curl_send_multipart_form_request() { } test_expect_success 'configure nodes' ' - iptb init -n 2 -p 0 -f --bootstrap=none && + iptb testbed create -type localipfs -count 2 -force -init && ipfsi 0 config --json Experimental.Libp2pStreamMounting true && ipfsi 1 config --json Experimental.Libp2pStreamMounting true && ipfsi 0 config --json Experimental.P2pHttpProxy true ' test_expect_success 'start and connect nodes' ' - iptb start && iptb connect 0 1 + iptb start -wait && iptb connect 0 1 ' test_expect_success 'setup p2p listener on the receiver' ' From 78c43fe68b2c386caf70c7cb0639ac6e127b453d Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Sat, 10 Nov 2018 21:46:46 +0000 Subject: [PATCH 28/40] move p2p http proxy from api to gateway License: MIT Signed-off-by: Ian Preston --- cmd/ipfs/daemon.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 853ddac6e..8c202a48f 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -290,9 +290,9 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment // Start assembling node config ncfg := &core.BuildCfg{ - Repo: repo, - Permanent: true, // It is temporary way to signify that node is permanent - Online: !offline, + Repo: repo, + Permanent: true, // It is temporary way to signify that node is permanent + Online: !offline, DisableEncryptedConnections: unencrypted, ExtraOpts: map[string]bool{ "pubsub": pubsub, @@ -467,9 +467,6 @@ func serveHTTPApi(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, error opts = append(opts, corehttp.RedirectOption("", cfg.Gateway.RootRedirect)) } - if cfg.Experimental.P2pHttpProxy { - opts = append(opts, corehttp.ProxyOption()) - } node, err := cctx.ConstructNode() if err != nil { return nil, fmt.Errorf("serveHTTPApi: ConstructNode() failed: %s", err) @@ -573,6 +570,10 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e corehttp.CommandsROOption(*cctx), } + if cfg.Experimental.P2pHttpProxy { + opts = append(opts, corehttp.ProxyOption()) + } + if len(cfg.Gateway.RootRedirect) > 0 { opts = append(opts, corehttp.RedirectOption("", cfg.Gateway.RootRedirect)) } From fd43f473b4698fe027dd93f3eac96dd47564d243 Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Sat, 10 Nov 2018 22:08:51 +0000 Subject: [PATCH 29/40] switch to new path format in p2p http proxy License: MIT Signed-off-by: Ian Preston --- core/corehttp/proxy.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 5d85d7609..89d0affd6 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -17,7 +17,7 @@ import ( // ProxyOption is an endpoint for proxying a HTTP request to another ipfs peer func ProxyOption() ServeOption { return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { - mux.HandleFunc("/proxy/http/", func(w http.ResponseWriter, request *http.Request) { + mux.HandleFunc("/p2p/", func(w http.ResponseWriter, request *http.Request) { // parse request parsedRequest, err := parseRequest(request) if err != nil { @@ -49,16 +49,27 @@ type proxyRequest struct { } // from the url path parse the peer-ID, name and http path -// /proxy/http/$peer_id/$name/$http_path +// /p2p/$peer_id/http/$http_path +// or +// /p2p/$peer_id/x/$protocol/http/$http_path func parseRequest(request *http.Request) (*proxyRequest, error) { path := request.URL.Path - split := strings.SplitN(path, "/", 6) - if len(split) < 6 { + split := strings.SplitN(path, "/", 5) + if len(split) < 5 { return nil, fmt.Errorf("Invalid request path '%s'", path) } - return &proxyRequest{split[3], protocol.ID(split[4]), split[5]}, nil + if split[3] == "http" { + return &proxyRequest{split[2], protocol.ID("/http"), split[4]}, nil + } + + split = strings.SplitN(path, "/", 7) + if split[3] != "x" || split[5] != "http" { + return nil, fmt.Errorf("Invalid request path '%s'", path) + } + + return &proxyRequest{split[2], protocol.ID("/x/" + split[4] + "/http"), split[6]}, nil } func handleError(w http.ResponseWriter, msg string, err error, code int) { From 47d45c7af86151864ac727fd1162cb0669991bdf Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Sat, 10 Nov 2018 22:24:41 +0000 Subject: [PATCH 30/40] fix tests License: MIT Signed-off-by: Ian Preston --- core/corehttp/proxy_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/corehttp/proxy_test.go b/core/corehttp/proxy_test.go index b8d8e0e8b..c8a2620e9 100644 --- a/core/corehttp/proxy_test.go +++ b/core/corehttp/proxy_test.go @@ -9,7 +9,7 @@ import ( ) func TestParseRequest(t *testing.T) { - url := "http://localhost:5001/proxy/http/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/test-name/path/to/index.txt" + url := "http://localhost:5001/p2p/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/http/path/to/index.txt" req, _ := http.NewRequest("GET", url, strings.NewReader("")) parsed, err := parseRequest(req) @@ -17,12 +17,12 @@ func TestParseRequest(t *testing.T) { t.Error(err) } assert.True(parsed.httpPath == "path/to/index.txt", t, "proxy request path") - assert.True(parsed.name == "test-name", t, "proxy request name") + assert.True(parsed.name == "/http", t, "proxy request name") assert.True(parsed.target == "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", t, "proxy request peer-id") } func TestParseRequestInvalidPath(t *testing.T) { - url := "http://localhost:5001/proxy/http/foobar" + url := "http://localhost:5001/p2p/http/foobar" req, _ := http.NewRequest("GET", url, strings.NewReader("")) _, err := parseRequest(req) @@ -30,5 +30,5 @@ func TestParseRequestInvalidPath(t *testing.T) { t.Fail() } - assert.True(err.Error() == "Invalid request path '/proxy/http/foobar'", t, "fails with invalid path") + assert.True(err.Error() == "Invalid request path '/p2p/http/foobar'", t, "fails with invalid path") } From 19d4d66fc73843a89651c2875c07e9f7b30dcc80 Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Sat, 10 Nov 2018 23:07:56 +0000 Subject: [PATCH 31/40] update p2p http proxy docs License: MIT Signed-off-by: Ian Preston --- docs/experimental-features.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 8f8e9695c..f4fb78fe8 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -393,7 +393,7 @@ Experimental ### In Version -master, 0.4.18 +master, 0.4.19 ### How to enable @@ -415,7 +415,7 @@ On the client, the p2p http proxy needs to be enabled in the config: First, pick a protocol name for your application. Think of the protocol name as a port number, just significantly more user-friendly. In this example, we're -going to use `test`. +going to use `/http`. ***Setup:*** @@ -430,15 +430,15 @@ port `$APP_PORT`. Then, configure the p2p listener by running: ```sh -> ipfs p2p listen --allow-custom-protocol test /ip4/127.0.0.1/tcp/$APP_PORT +> ipfs p2p listen --allow-custom-protocol /http /ip4/127.0.0.1/tcp/$APP_PORT ``` -This will configure IPFS to forward all incoming `test` streams to +This will configure IPFS to forward all incoming `/http` streams to `127.0.0.1:$APP_PORT` (opening a new connection to `127.0.0.1:$APP_PORT` per incoming stream. ***On the "client" node:*** -Next, have your application make a http request to `127.0.0.1:5001/proxy/http/$SERVER_ID/$PROTOCOL/$FORWARDED_PATH`. This +Next, have your application make a http request to `127.0.0.1:8080/p2p/$SERVER_ID/http/$FORWARDED_PATH`. This connection will be forwarded to the service running on `127.0.0.1:$APP_PORT` on the remote machine (which needs to be a http server!) with path `$FORWARDED_PATH`. You can test it with netcat: @@ -449,15 +449,16 @@ the remote machine (which needs to be a http server!) with path `$FORWARDED_PATH ***On "client" node:*** ```sh -> curl http://localhost:5001/proxy/http/$SERVER_ID/test/ +> curl http://localhost:8080/p2p/$SERVER_ID/http/ ``` You should now see the resulting http response: IPFS rocks! +### Custom protocol names +We also support use of protocol names of the form /x/$NAME/http where $NAME doesn't contain any "/"'s ### Road to being a real feature - [ ] Needs p2p streams to graduate from experiments -- [ ] Decide how to handle protocol names with /'s in them - [ ] Needs more people to use and report on how well it works / fits use cases - [ ] More documentation From d8cab7998ed258d357fcd4b9d545d5b090bc4905 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 12 Nov 2018 16:10:19 -0800 Subject: [PATCH 32/40] update tests License: MIT Signed-off-by: Steven Allen --- test/sharness/t0184-http-proxy-over-p2p.sh | 28 ++++++++++++---------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index 64aac3ed4..abd638f5f 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -4,6 +4,8 @@ test_description="Test http proxy over p2p" . lib/test-lib.sh WEB_SERVE_PORT=5099 +IPFS_GATEWAY_PORT=5199 +SENDER_GATEWAY="http://127.0.0.1:$IPFS_GATEWAY_PORT" function show_logs() { @@ -41,8 +43,8 @@ function teardown_remote_server() { function curl_check_response_code() { local expected_status_code=$1 - local path_stub=${2:-http/$RECEIVER_ID/test/index.txt} - local status_code=$(curl -s --write-out %{http_code} --output /dev/null $SENDER_URL/$path_stub) + local path_stub=${2:-p2p/$RECEIVER_ID/http/index.txt} + local status_code=$(curl -s --write-out %{http_code} --output /dev/null $SENDER_GATEWAY/$path_stub) if [[ "$status_code" -ne "$expected_status_code" ]]; then @@ -61,12 +63,12 @@ function curl_send_proxy_request_and_check_response() { # make a request to SENDER_IPFS via the proxy endpoint # CONTENT_PATH="retrieved-file" - STATUS_CODE=$(curl -s -o $CONTENT_PATH --write-out %{http_code} $SENDER_URL/$RECEIVER_ID/test/index.txt) + STATUS_CODE="$(curl -s -o $CONTENT_PATH --write-out %{http_code} $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt)" # # check status code # - if [[ $STATUS_CODE -ne $expected_status_code ]]; + if [[ "$STATUS_CODE" -ne "$expected_status_code" ]]; then echo -e "Found status-code "$STATUS_CODE", expected "$expected_status_code return 1 @@ -75,7 +77,7 @@ function curl_send_proxy_request_and_check_response() { # # check content # - RESPONSE_CONTENT=$(tail -n 1 $CONTENT_PATH) + RESPONSE_CONTENT="$(tail -n 1 $CONTENT_PATH)" if [[ "$RESPONSE_CONTENT" == "$expected_content" ]]; then return 0 @@ -93,11 +95,11 @@ function curl_send_multipart_form_request() { # # send multipart form request # - STATUS_CODE=$(curl -v -F file=@$FILE_PATH $SENDER_URL/$RECEIVER_ID/test/index.txt) + STATUS_CODE="$(curl -v -F file=@$FILE_PATH $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt)" # # check status code # - if [[ $STATUS_CODE -ne $expected_status_code ]]; + if [[ "$STATUS_CODE" -ne "$expected_status_code" ]]; then echo -e "Found status-code "$STATUS_CODE", expected "$expected_status_code return 1 @@ -128,6 +130,7 @@ test_expect_success 'configure nodes' ' ipfsi 0 config --json Experimental.Libp2pStreamMounting true && ipfsi 1 config --json Experimental.Libp2pStreamMounting true && ipfsi 0 config --json Experimental.P2pHttpProxy true + ipfsi 0 config --json Addresses.Gateway "[\"/ip4/127.0.0.1/tcp/$IPFS_GATEWAY_PORT\"]" ' test_expect_success 'start and connect nodes' ' @@ -135,12 +138,11 @@ test_expect_success 'start and connect nodes' ' ' test_expect_success 'setup p2p listener on the receiver' ' - ipfsi 1 p2p listen --allow-custom-protocol test /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT + ipfsi 1 p2p listen --allow-custom-protocol /http /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT ' -test_expect_success 'setup environment variables' ' - RECEIVER_ID="$(iptb get id 1)" && - SENDER_URL="$(sed "s|^/ip[46]/\([^/]*\)/tcp/\([0-9]*\)\$|http://\\1:\\2/proxy/http|" < "$(iptb get path 0)/api")" +test_expect_success 'setup environment' ' + RECEIVER_ID="$(iptb attr get 1 id)" ' test_expect_success 'handle proxy http request propogates error response from remote' ' @@ -160,11 +162,11 @@ test_expect_success 'handle proxy http request ' ' teardown_remote_server test_expect_success 'handle proxy http request invalid request' ' - curl_check_response_code 400 DERPDERPDERP + curl_check_response_code 400 p2p/DERPDERPDERP ' test_expect_success 'handle proxy http request unknown proxy peer ' ' - curl_check_response_code 502 unknown_peer/test/index.txt + curl_check_response_code 502 p2p/unknown_peer/http/index.txt ' test_expect_success 'handle multipart/form-data http request' ' From fe8ffde4c2b8adc1829e0f3a7d6b307e78f3ea2b Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 12 Nov 2018 16:18:52 -0800 Subject: [PATCH 33/40] add some additional tests for custom protocols License: MIT Signed-off-by: Steven Allen --- test/sharness/t0184-http-proxy-over-p2p.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index abd638f5f..65f0cf4c0 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -138,7 +138,8 @@ test_expect_success 'start and connect nodes' ' ' test_expect_success 'setup p2p listener on the receiver' ' - ipfsi 1 p2p listen --allow-custom-protocol /http /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT + ipfsi 1 p2p listen --allow-custom-protocol /http /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT && + ipfsi 1 p2p listen /x/custom/http /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT ' test_expect_success 'setup environment' ' @@ -169,6 +170,22 @@ test_expect_success 'handle proxy http request unknown proxy peer ' ' curl_check_response_code 502 p2p/unknown_peer/http/index.txt ' +test_expect_success 'handle proxy http request to custom protocol' ' + serve_http_once "THE WOODS ARE LOVELY DARK AND DEEP" && + curl_check_response_code 200 p2p/$RECEIVER_ID/x/custom/http/index.txt +' +teardown_remote_server + +test_expect_success 'handle proxy http request to missing protocol' ' + serve_http_once "THE WOODS ARE LOVELY DARK AND DEEP" && + curl_check_response_code 502 p2p/$RECEIVER_ID/x/missing/http/index.txt +' +teardown_remote_server + +test_expect_success 'handle proxy http request missing the /http' ' + curl_check_response_code 400 p2p/$RECEIVER_ID/x/custom/index.txt +' + test_expect_success 'handle multipart/form-data http request' ' serve_http_once "OK" && curl_send_multipart_form_request From b720d1f0b58321ff209131ef8de04ce53ea093e6 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 12 Nov 2018 16:21:27 -0800 Subject: [PATCH 34/40] uses the global PeerHost and don't expose the P2P one License: MIT Signed-off-by: Steven Allen --- core/corehttp/proxy.go | 2 +- p2p/local.go | 2 +- p2p/p2p.go | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 89d0affd6..6c3b793d2 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -33,7 +33,7 @@ func ProxyOption() ServeOption { return } - rt := p2phttp.NewTransport(ipfsNode.P2P.PeerHost, p2phttp.ProtocolOption(parsedRequest.name)) + rt := p2phttp.NewTransport(ipfsNode.PeerHost, p2phttp.ProtocolOption(parsedRequest.name)) proxy := httputil.NewSingleHostReverseProxy(target) proxy.Transport = rt proxy.ServeHTTP(w, request) diff --git a/p2p/local.go b/p2p/local.go index 076af3bc4..47c7a312b 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -55,7 +55,7 @@ func (l *localListener) dial(ctx context.Context) (net.Stream, error) { cctx, cancel := context.WithTimeout(ctx, time.Second*30) //TODO: configurable? defer cancel() - return l.p2p.PeerHost.NewStream(cctx, l.peer, l.proto) + return l.p2p.peerHost.NewStream(cctx, l.peer, l.proto) } func (l *localListener) acceptConns() { diff --git a/p2p/p2p.go b/p2p/p2p.go index 2d8ee8ca4..efc87f722 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -16,23 +16,23 @@ type P2P struct { Streams *StreamRegistry identity peer.ID - PeerHost p2phost.Host + peerHost p2phost.Host peerstore pstore.Peerstore } // NewP2P creates new P2P struct -func NewP2P(identity peer.ID, PeerHost p2phost.Host, peerstore pstore.Peerstore) *P2P { +func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) *P2P { return &P2P{ identity: identity, - PeerHost: PeerHost, + peerHost: peerHost, peerstore: peerstore, ListenersLocal: newListenersLocal(), - ListenersP2P: newListenersP2P(PeerHost), + ListenersP2P: newListenersP2P(peerHost), Streams: &StreamRegistry{ Streams: map[uint64]*Stream{}, - ConnManager: PeerHost.ConnManager(), + ConnManager: peerHost.ConnManager(), conns: map[peer.ID]int{}, }, } @@ -41,7 +41,7 @@ func NewP2P(identity peer.ID, PeerHost p2phost.Host, peerstore pstore.Peerstore) // CheckProtoExists checks whether a proto handler is registered to // mux handler func (p2p *P2P) CheckProtoExists(proto string) bool { - protos := p2p.PeerHost.Mux().Protocols() + protos := p2p.peerHost.Mux().Protocols() for _, p := range protos { if p != proto { From bf184376ca5f390f9387292cae6c35ff0f30d09d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 12 Nov 2018 16:23:32 -0800 Subject: [PATCH 35/40] p2p http proxy: go fmt License: MIT Signed-off-by: Steven Allen --- cmd/ipfs/daemon.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 8c202a48f..1bf68ab37 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -290,9 +290,9 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment // Start assembling node config ncfg := &core.BuildCfg{ - Repo: repo, - Permanent: true, // It is temporary way to signify that node is permanent - Online: !offline, + Repo: repo, + Permanent: true, // It is temporary way to signify that node is permanent + Online: !offline, DisableEncryptedConnections: unencrypted, ExtraOpts: map[string]bool{ "pubsub": pubsub, From 3b2ce4a85daef99c0fd56173316950b82b9d147b Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 15 Nov 2018 21:13:04 +0000 Subject: [PATCH 36/40] Apply suggestions from code review License: MIT Signed-off-by: Ian Preston Co-Authored-By: ianopolous --- core/corehttp/proxy.go | 6 +++--- core/corehttp/proxy_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/corehttp/proxy.go b/core/corehttp/proxy.go index 6c3b793d2..c9d74fa92 100644 --- a/core/corehttp/proxy.go +++ b/core/corehttp/proxy.go @@ -21,7 +21,7 @@ func ProxyOption() ServeOption { // parse request parsedRequest, err := parseRequest(request) if err != nil { - handleError(w, "Failed to parse request", err, 400) + handleError(w, "failed to parse request", err, 400) return } @@ -29,7 +29,7 @@ func ProxyOption() ServeOption { request.URL.Path = parsedRequest.httpPath target, err := url.Parse(fmt.Sprintf("libp2p://%s", parsedRequest.target)) if err != nil { - handleError(w, "Failed to parse url", err, 400) + handleError(w, "failed to parse url", err, 400) return } @@ -75,5 +75,5 @@ func parseRequest(request *http.Request) (*proxyRequest, error) { func handleError(w http.ResponseWriter, msg string, err error, code int) { w.WriteHeader(code) fmt.Fprintf(w, "%s: %s\n", msg, err) - log.Warningf("server error: %s: %s", err) + log.Warningf("http proxy error: %s: %s", err) } diff --git a/core/corehttp/proxy_test.go b/core/corehttp/proxy_test.go index c8a2620e9..b328fcf2d 100644 --- a/core/corehttp/proxy_test.go +++ b/core/corehttp/proxy_test.go @@ -14,7 +14,7 @@ func TestParseRequest(t *testing.T) { parsed, err := parseRequest(req) if err != nil { - t.Error(err) + t.Fatal(err) } assert.True(parsed.httpPath == "path/to/index.txt", t, "proxy request path") assert.True(parsed.name == "/http", t, "proxy request name") From 9956630e0c4ca6620ceb74fc7b76d4dc5b0f3834 Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Thu, 15 Nov 2018 22:13:35 +0000 Subject: [PATCH 37/40] add more p2p http proxy tests License: MIT Signed-off-by: Ian Preston --- core/corehttp/proxy_test.go | 53 ++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/core/corehttp/proxy_test.go b/core/corehttp/proxy_test.go index b328fcf2d..653142f3d 100644 --- a/core/corehttp/proxy_test.go +++ b/core/corehttp/proxy_test.go @@ -6,29 +6,50 @@ import ( "testing" "github.com/ipfs/go-ipfs/thirdparty/assert" + + protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) -func TestParseRequest(t *testing.T) { - url := "http://localhost:5001/p2p/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/http/path/to/index.txt" - req, _ := http.NewRequest("GET", url, strings.NewReader("")) +type TestCase struct { + urlprefix string + target string + name string + path string +} - parsed, err := parseRequest(req) - if err != nil { - t.Fatal(err) +var validtestCases = []TestCase{ + {"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/http", "path/to/index.txt"}, + {"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/x/custom/http", "path/to/index.txt"}, + {"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/x/custom/http", "http/path/to/index.txt"}, +} + +func TestParseRequest(t *testing.T) { + for _, tc := range validtestCases { + url := tc.urlprefix + "/p2p/" + tc.target + tc.name + "/" + tc.path + req, _ := http.NewRequest("GET", url, strings.NewReader("")) + + parsed, err := parseRequest(req) + if err != nil { + t.Fatal(err) + } + assert.True(parsed.httpPath == tc.path, t, "proxy request path") + assert.True(parsed.name == protocol.ID(tc.name), t, "proxy request name") + assert.True(parsed.target == tc.target, t, "proxy request peer-id") } - assert.True(parsed.httpPath == "path/to/index.txt", t, "proxy request path") - assert.True(parsed.name == "/http", t, "proxy request name") - assert.True(parsed.target == "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", t, "proxy request peer-id") +} + +var invalidtestCases = []string{ + "http://localhost:5001/p2p/http/foobar", } func TestParseRequestInvalidPath(t *testing.T) { - url := "http://localhost:5001/p2p/http/foobar" - req, _ := http.NewRequest("GET", url, strings.NewReader("")) + for _, tc := range invalidtestCases { + url := tc + req, _ := http.NewRequest("GET", url, strings.NewReader("")) - _, err := parseRequest(req) - if err == nil { - t.Fail() + _, err := parseRequest(req) + if err == nil { + t.Fail() + } } - - assert.True(err.Error() == "Invalid request path '/p2p/http/foobar'", t, "fails with invalid path") } From 7e541079c7ec6373b7e424ca2eea1f0e2c7abf24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 16 Nov 2018 07:40:10 +0000 Subject: [PATCH 38/40] Update test/sharness/t0184-http-proxy-over-p2p.sh License: MIT Signed-off-by: Ian Preston Co-Authored-By: ianopolous --- test/sharness/t0184-http-proxy-over-p2p.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index 65f0cf4c0..a1b908df3 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -32,7 +32,9 @@ function serve_http_once() { local status_code=${2:-"200 OK"} local length=$((1 + ${#body})) REMOTE_SERVER_LOG="server.log" + rm $REMOTE_SERVER_LOG echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" | nc -l $WEB_SERVE_PORT 2>&1 > $REMOTE_SERVER_LOG & + test_wait_for_file 30 100ms $REMOTE_SERVER_LOG REMOTE_SERVER_PID=$! } From 869498499213ee9795ef8ad86f0f49b3019a643f Mon Sep 17 00:00:00 2001 From: Dr Ian Preston Date: Wed, 21 Nov 2018 00:30:08 +0000 Subject: [PATCH 39/40] Add another test case for invalid p2p http proxy path License: MIT Signed-off-by: Ian Preston --- core/corehttp/proxy_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/corehttp/proxy_test.go b/core/corehttp/proxy_test.go index 653142f3d..786dcf3d9 100644 --- a/core/corehttp/proxy_test.go +++ b/core/corehttp/proxy_test.go @@ -40,6 +40,7 @@ func TestParseRequest(t *testing.T) { var invalidtestCases = []string{ "http://localhost:5001/p2p/http/foobar", + "http://localhost:5001/p2p/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/x/custom/foobar", } func TestParseRequestInvalidPath(t *testing.T) { From 9a443adada9e1cebc61985504c2b4a991d21ddde Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 28 Nov 2018 22:08:07 -0800 Subject: [PATCH 40/40] p2p proxy tests: make robust against timing Instead of repeatedly starting the netcat server, start it once and wait for it to fully start. Then, feed responses in using a fifo. License: MIT Signed-off-by: Steven Allen --- test/sharness/t0184-http-proxy-over-p2p.sh | 68 +++++++++++++--------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/test/sharness/t0184-http-proxy-over-p2p.sh b/test/sharness/t0184-http-proxy-over-p2p.sh index a1b908df3..15de5eac1 100755 --- a/test/sharness/t0184-http-proxy-over-p2p.sh +++ b/test/sharness/t0184-http-proxy-over-p2p.sh @@ -24,23 +24,32 @@ function show_logs() { cat $REMOTE_SERVER_LOG } -function serve_http_once() { - # - # one shot http server (via nc) with static body - # - local body=$1 - local status_code=${2:-"200 OK"} - local length=$((1 + ${#body})) +function start_http_server() { REMOTE_SERVER_LOG="server.log" - rm $REMOTE_SERVER_LOG - echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" | nc -l $WEB_SERVE_PORT 2>&1 > $REMOTE_SERVER_LOG & - test_wait_for_file 30 100ms $REMOTE_SERVER_LOG + rm -f $REMOTE_SERVER_LOG server_stdin + + mkfifo server_stdin + nc -k -l 127.0.0.1 $WEB_SERVE_PORT 2>&1 > $REMOTE_SERVER_LOG < server_stdin & REMOTE_SERVER_PID=$! + exec 7>server_stdin + rm server_stdin + + while ! nc -z 127.0.0.1 $WEB_SERVE_PORT; do + go-sleep 100ms + done } function teardown_remote_server() { - kill -9 $REMOTE_SERVER_PID > /dev/null 2>&1 - sleep 5 + exec 7<&- + kill $REMOTE_SERVER_PID > /dev/null 2>&1 + wait $REMOTE_SERVER_PID || true +} + +function serve_content() { + local body=$1 + local status_code=${2:-"200 OK"} + local length=$((1 + ${#body})) + echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" >&7 } function curl_check_response_code() { @@ -97,7 +106,7 @@ function curl_send_multipart_form_request() { # # send multipart form request # - STATUS_CODE="$(curl -v -F file=@$FILE_PATH $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt)" + STATUS_CODE="$(curl -o /dev/null -s -F file=@$FILE_PATH --write-out %{http_code} $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt)" # # check status code # @@ -148,21 +157,23 @@ test_expect_success 'setup environment' ' RECEIVER_ID="$(iptb attr get 1 id)" ' -test_expect_success 'handle proxy http request propogates error response from remote' ' - serve_http_once "SORRY GUYS, I LOST IT" "404 Not Found" && - curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT" -' -teardown_remote_server - test_expect_success 'handle proxy http request sends bad-gateway when remote server not available ' ' curl_send_proxy_request_and_check_response 502 "" ' +test_expect_success 'start http server' ' + start_http_server +' + +test_expect_success 'handle proxy http request propogates error response from remote' ' + serve_content "SORRY GUYS, I LOST IT" "404 Not Found" && + curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT" +' + test_expect_success 'handle proxy http request ' ' - serve_http_once "THE WOODS ARE LOVELY DARK AND DEEP" && + serve_content "THE WOODS ARE LOVELY DARK AND DEEP" && curl_send_proxy_request_and_check_response 200 "THE WOODS ARE LOVELY DARK AND DEEP" ' -teardown_remote_server test_expect_success 'handle proxy http request invalid request' ' curl_check_response_code 400 p2p/DERPDERPDERP @@ -173,26 +184,27 @@ test_expect_success 'handle proxy http request unknown proxy peer ' ' ' test_expect_success 'handle proxy http request to custom protocol' ' - serve_http_once "THE WOODS ARE LOVELY DARK AND DEEP" && + serve_content "THE WOODS ARE LOVELY DARK AND DEEP" && curl_check_response_code 200 p2p/$RECEIVER_ID/x/custom/http/index.txt ' -teardown_remote_server test_expect_success 'handle proxy http request to missing protocol' ' - serve_http_once "THE WOODS ARE LOVELY DARK AND DEEP" && + serve_content "THE WOODS ARE LOVELY DARK AND DEEP" && curl_check_response_code 502 p2p/$RECEIVER_ID/x/missing/http/index.txt ' -teardown_remote_server test_expect_success 'handle proxy http request missing the /http' ' curl_check_response_code 400 p2p/$RECEIVER_ID/x/custom/index.txt ' test_expect_success 'handle multipart/form-data http request' ' - serve_http_once "OK" && - curl_send_multipart_form_request + serve_content "OK" && + curl_send_multipart_form_request 200 +' + +test_expect_success 'stop http server' ' + teardown_remote_server ' -teardown_remote_server test_expect_success 'stop nodes' ' iptb stop