coosld/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2026-02-26 04:47:45 +08:00
Code Issues Actions 13 Packages Projects Releases Wiki Activity

Merge pull request #5526 from cboddy/feat/http_proxy_over_p2p

Browse Source 
[http_proxy_over_p2p]
This commit is contained in:
Steven Allen 2018-11-29 20:15:32 -08:00 committed by GitHub
commit 2d94a3ffc8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 440 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/ipfs/daemon.go
View File

@ -570,6 +570,10 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e
corehttp.CommandsROOption(*cctx),
}
if cfg.Experimental.P2pHttpProxy {
opts = append(opts, corehttp.ProxyOption())
}
if len(cfg.Gateway.RootRedirect) > 0 {
opts = append(opts, corehttp.RedirectOption("", cfg.Gateway.RootRedirect))
}

79
core/corehttp/proxy.go Normal file
View File

@ -0,0 +1,79 @@
package corehttp
import (
"fmt"
"net"
"net/http"
"net/http/httputil"
"net/url"
"strings"
core "github.com/ipfs/go-ipfs/core"
protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol"
p2phttp "gx/ipfs/QmcLYfmHLsaVRKGMZQovwEYhHAjWtRjg1Lij3pnzw5UkRD/go-libp2p-http"
)
// ProxyOption is an endpoint for proxying a HTTP request to another ipfs peer
func ProxyOption() ServeOption {
return func(ipfsNode *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
mux.HandleFunc("/p2p/", func(w http.ResponseWriter, request *http.Request) {
// parse request
parsedRequest, err := parseRequest(request)
if err != nil {
handleError(w, "failed to parse request", err, 400)
return
}
request.Host = "" // Let URL's Host take precedence.
request.URL.Path = parsedRequest.httpPath
target, err := url.Parse(fmt.Sprintf("libp2p://%s", parsedRequest.target))
if err != nil {
handleError(w, "failed to parse url", err, 400)
return
}
rt := p2phttp.NewTransport(ipfsNode.PeerHost, p2phttp.ProtocolOption(parsedRequest.name))
proxy := httputil.NewSingleHostReverseProxy(target)
proxy.Transport = rt
proxy.ServeHTTP(w, request)
})
return mux, nil
}
}
type proxyRequest struct {
target string
name protocol.ID
httpPath string // path to send to the proxy-host
}
// from the url path parse the peer-ID, name and http path
// /p2p/$peer_id/http/$http_path
// or
// /p2p/$peer_id/x/$protocol/http/$http_path
func parseRequest(request *http.Request) (*proxyRequest, error) {
path := request.URL.Path
split := strings.SplitN(path, "/", 5)
if len(split) < 5 {
return nil, fmt.Errorf("Invalid request path '%s'", path)
}
if split[3] == "http" {
return &proxyRequest{split[2], protocol.ID("/http"), split[4]}, nil
}
split = strings.SplitN(path, "/", 7)
if split[3] != "x" || split[5] != "http" {
return nil, fmt.Errorf("Invalid request path '%s'", path)
}
return &proxyRequest{split[2], protocol.ID("/x/" + split[4] + "/http"), split[6]}, nil
}
func handleError(w http.ResponseWriter, msg string, err error, code int) {
w.WriteHeader(code)
fmt.Fprintf(w, "%s: %s\n", msg, err)
log.Warningf("http proxy error: %s: %s", err)
}

56
core/corehttp/proxy_test.go Normal file
View File

@ -0,0 +1,56 @@
package corehttp
import (
"net/http"
"strings"
"testing"
"github.com/ipfs/go-ipfs/thirdparty/assert"
protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol"
)
type TestCase struct {
urlprefix string
target string
name string
path string
}
var validtestCases = []TestCase{
{"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/http", "path/to/index.txt"},
{"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/x/custom/http", "path/to/index.txt"},
{"http://localhost:5001", "QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT", "/x/custom/http", "http/path/to/index.txt"},
}
func TestParseRequest(t *testing.T) {
for _, tc := range validtestCases {
url := tc.urlprefix + "/p2p/" + tc.target + tc.name + "/" + tc.path
req, _ := http.NewRequest("GET", url, strings.NewReader(""))
parsed, err := parseRequest(req)
if err != nil {
t.Fatal(err)
}
assert.True(parsed.httpPath == tc.path, t, "proxy request path")
assert.True(parsed.name == protocol.ID(tc.name), t, "proxy request name")
assert.True(parsed.target == tc.target, t, "proxy request peer-id")
}
}
var invalidtestCases = []string{
"http://localhost:5001/p2p/http/foobar",
"http://localhost:5001/p2p/QmT8JtU54XSmC38xSb1XHFSMm775VuTeajg7LWWWTAwzxT/x/custom/foobar",
}
func TestParseRequestInvalidPath(t *testing.T) {
for _, tc := range invalidtestCases {
url := tc
req, _ := http.NewRequest("GET", url, strings.NewReader(""))
_, err := parseRequest(req)
if err == nil {
t.Fail()
}
}
}

82
docs/experimental-features.md
View File

@ -21,6 +21,7 @@ the above issue.
- [BadgerDB datastore](#badger-datastore)
- [Private Networks](#private-networks)
- [ipfs p2p](#ipfs-p2p)
- [p2p http proxy](#p2p-http-proxy)
- [Circuit Relay](#circuit-relay)
- [Plugins](#plugins)
- [Directory Sharding / HAMT](#directory-sharding-hamt)
@ -382,6 +383,87 @@ with `ssh [user]@127.0.0.1 -p 2222`.
---
## p2p http proxy
Allows proxying of HTTP requests over p2p streams. This allows serving any standard http app over p2p streams.
### State
Experimental
### In Version
master, 0.4.19
### How to enable
The `p2p` command needs to be enabled in config:
```sh
> ipfs config --json Experimental.Libp2pStreamMounting true
```
On the client, the p2p http proxy needs to be enabled in the config:
```sh
> ipfs config --json Experimental.P2pHttpProxy true
```
### How to use
**Netcat example:**
First, pick a protocol name for your application. Think of the protocol name as
a port number, just significantly more user-friendly. In this example, we're
going to use `/http`.
***Setup:***
1. A "server" node with peer ID `$SERVER_ID`
2. A "client" node.
***On the "server" node:***
First, start your application and have it listen for TCP connections on
port `$APP_PORT`.
Then, configure the p2p listener by running:
```sh
> ipfs p2p listen --allow-custom-protocol /http /ip4/127.0.0.1/tcp/$APP_PORT
```
This will configure IPFS to forward all incoming `/http` streams to
`127.0.0.1:$APP_PORT` (opening a new connection to `127.0.0.1:$APP_PORT` per incoming stream.
***On the "client" node:***
Next, have your application make a http request to `127.0.0.1:8080/p2p/$SERVER_ID/http/$FORWARDED_PATH`. This
connection will be forwarded to the service running on `127.0.0.1:$APP_PORT` on
the remote machine (which needs to be a http server!) with path `$FORWARDED_PATH`. You can test it with netcat:
***On "server" node:***
```sh
> echo -e "HTTP/1.1 200\nContent-length: 11\n\nIPFS rocks!" | nc -l -p $APP_PORT
```
***On "client" node:***
```sh
> curl http://localhost:8080/p2p/$SERVER_ID/http/
```
You should now see the resulting http response: IPFS rocks!
### Custom protocol names
We also support use of protocol names of the form /x/$NAME/http where $NAME doesn't contain any "/"'s
### Road to being a real feature
- [ ] Needs p2p streams to graduate from experiments
- [ ] Needs more people to use and report on how well it works / fits use cases
- [ ] More documentation
---
## Circuit Relay
Allows peers to connect through an intermediate relay node when there

6
package.json
View File

@ -592,6 +592,12 @@
"hash": "QmTqLBwme9BusYWdACqL62NFb8WV2Q72gXLsQVfC7vmCr4",
"name": "iptb-plugins",
"version": "1.0.5"
},
{
"author": "hsanjuan",
"hash": "QmcLYfmHLsaVRKGMZQovwEYhHAjWtRjg1Lij3pnzw5UkRD",
"name": "go-libp2p-http",
"version": "1.1.8"
}
],
"gxVersion": "0.10.0",

213
test/sharness/t0184-http-proxy-over-p2p.sh Executable file
View File

@ -0,0 +1,213 @@
#!/usr/bin/env bash
test_description="Test http proxy over p2p"
. lib/test-lib.sh
WEB_SERVE_PORT=5099
IPFS_GATEWAY_PORT=5199
SENDER_GATEWAY="http://127.0.0.1:$IPFS_GATEWAY_PORT"
function show_logs() {
echo "*****************"
echo " RECEIVER LOG "
echo "*****************"
iptb logs 1
echo "*****************"
echo " SENDER LOG "
echo "*****************"
iptb logs 0
echo "*****************"
echo "REMOTE_SERVER LOG"
echo $REMOTE_SERVER_LOG
echo "*****************"
cat $REMOTE_SERVER_LOG
}
function start_http_server() {
REMOTE_SERVER_LOG="server.log"
rm -f $REMOTE_SERVER_LOG server_stdin
mkfifo server_stdin
nc -k -l 127.0.0.1 $WEB_SERVE_PORT 2>&1 > $REMOTE_SERVER_LOG < server_stdin &
REMOTE_SERVER_PID=$!
exec 7>server_stdin
rm server_stdin
while ! nc -z 127.0.0.1 $WEB_SERVE_PORT; do
go-sleep 100ms
done
}
function teardown_remote_server() {
exec 7<&-
kill $REMOTE_SERVER_PID > /dev/null 2>&1
wait $REMOTE_SERVER_PID || true
}
function serve_content() {
local body=$1
local status_code=${2:-"200 OK"}
local length=$((1 + ${#body}))
echo -e "HTTP/1.1 $status_code\nContent-length: $length\n\n$body" >&7
}
function curl_check_response_code() {
local expected_status_code=$1
local path_stub=${2:-p2p/$RECEIVER_ID/http/index.txt}
local status_code=$(curl -s --write-out %{http_code} --output /dev/null $SENDER_GATEWAY/$path_stub)
if [[ "$status_code" -ne "$expected_status_code" ]];
then
echo "Found status-code "$status_code", expected "$expected_status_code
return 1
fi
return 0
}
function curl_send_proxy_request_and_check_response() {
local expected_status_code=$1
local expected_content=$2
#
# make a request to SENDER_IPFS via the proxy endpoint
#
CONTENT_PATH="retrieved-file"
STATUS_CODE="$(curl -s -o $CONTENT_PATH --write-out %{http_code} $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt)"
#
# check status code
#
if [[ "$STATUS_CODE" -ne "$expected_status_code" ]];
then
echo -e "Found status-code "$STATUS_CODE", expected "$expected_status_code
return 1
fi
#
# check content
#
RESPONSE_CONTENT="$(tail -n 1 $CONTENT_PATH)"
if [[ "$RESPONSE_CONTENT" == "$expected_content" ]];
then
return 0
else
echo -e "Found response content:\n'"$RESPONSE_CONTENT"'\nthat differs from expected content:\n'"$expected_content"'"
return 1
fi
}
function curl_send_multipart_form_request() {
local expected_status_code=$1
local FILE_PATH="uploaded-file"
FILE_CONTENT="curl will send a multipart-form POST request when sending a file which is handy"
echo $FILE_CONTENT > $FILE_PATH
#
# send multipart form request
#
STATUS_CODE="$(curl -o /dev/null -s -F file=@$FILE_PATH --write-out %{http_code} $SENDER_GATEWAY/p2p/$RECEIVER_ID/http/index.txt)"
#
# check status code
#
if [[ "$STATUS_CODE" -ne "$expected_status_code" ]];
then
echo -e "Found status-code "$STATUS_CODE", expected "$expected_status_code
return 1
fi
#
# check request method
#
if ! grep "POST /index.txt" $REMOTE_SERVER_LOG > /dev/null;
then
echo "Remote server request method/resource path was incorrect"
show_logs
return 1
fi
#
# check request is multipart-form
#
if ! grep "Content-Type: multipart/form-data;" $REMOTE_SERVER_LOG > /dev/null;
then
echo "Request content-type was not multipart/form-data"
show_logs
return 1
fi
return 0
}
test_expect_success 'configure nodes' '
iptb testbed create -type localipfs -count 2 -force -init &&
ipfsi 0 config --json Experimental.Libp2pStreamMounting true &&
ipfsi 1 config --json Experimental.Libp2pStreamMounting true &&
ipfsi 0 config --json Experimental.P2pHttpProxy true
ipfsi 0 config --json Addresses.Gateway "[\"/ip4/127.0.0.1/tcp/$IPFS_GATEWAY_PORT\"]"
'
test_expect_success 'start and connect nodes' '
iptb start -wait && iptb connect 0 1
'
test_expect_success 'setup p2p listener on the receiver' '
ipfsi 1 p2p listen --allow-custom-protocol /http /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT &&
ipfsi 1 p2p listen /x/custom/http /ip4/127.0.0.1/tcp/$WEB_SERVE_PORT
'
test_expect_success 'setup environment' '
RECEIVER_ID="$(iptb attr get 1 id)"
'
test_expect_success 'handle proxy http request sends bad-gateway when remote server not available ' '
curl_send_proxy_request_and_check_response 502 ""
'
test_expect_success 'start http server' '
start_http_server
'
test_expect_success 'handle proxy http request propogates error response from remote' '
serve_content "SORRY GUYS, I LOST IT" "404 Not Found" &&
curl_send_proxy_request_and_check_response 404 "SORRY GUYS, I LOST IT"
'
test_expect_success 'handle proxy http request ' '
serve_content "THE WOODS ARE LOVELY DARK AND DEEP" &&
curl_send_proxy_request_and_check_response 200 "THE WOODS ARE LOVELY DARK AND DEEP"
'
test_expect_success 'handle proxy http request invalid request' '
curl_check_response_code 400 p2p/DERPDERPDERP
'
test_expect_success 'handle proxy http request unknown proxy peer ' '
curl_check_response_code 502 p2p/unknown_peer/http/index.txt
'
test_expect_success 'handle proxy http request to custom protocol' '
serve_content "THE WOODS ARE LOVELY DARK AND DEEP" &&
curl_check_response_code 200 p2p/$RECEIVER_ID/x/custom/http/index.txt
'
test_expect_success 'handle proxy http request to missing protocol' '
serve_content "THE WOODS ARE LOVELY DARK AND DEEP" &&
curl_check_response_code 502 p2p/$RECEIVER_ID/x/missing/http/index.txt
'
test_expect_success 'handle proxy http request missing the /http' '
curl_check_response_code 400 p2p/$RECEIVER_ID/x/custom/index.txt
'
test_expect_success 'handle multipart/form-data http request' '
serve_content "OK" &&
curl_send_multipart_form_request 200
'
test_expect_success 'stop http server' '
teardown_remote_server
'
test_expect_success 'stop nodes' '
iptb stop
'
test_done