init docs: go generated welcome dir + files

updated sharness hashes

assets/assets.go

@@ -0,0 +1,14 @@
+//go:generate doc2go -in=init-doc/readme -out=readme.go -package=assets
+//go:generate doc2go -in=init-doc/help -out=help.go -package=assets
+//go:generate doc2go -in=init-doc/contact -out=contact.go -package=assets
+//go:generate doc2go -in=init-doc/security-notes -out=security-notes.go -package=assets
+//go:generate doc2go -in=init-doc/quick-start -out=quick-start.go -package=assets
+package assets
+
+var Init_dir = map[string]string{
+	"readme":         Init_doc_readme,
+	"help":           Init_doc_help,
+	"contact":        Init_doc_contact,
+	"security-notes": Init_doc_security_notes,
+	"quick-start":    Init_doc_quick_start,
+}

assets/contact.go

@@ -0,0 +1,8 @@
+package assets
+var Init_doc_contact = `Come hang out in our IRC chat room if you have any questions.
+
+Contact the ipfs dev team:
+- Bugs: https://github.com/jbenet/go-ipfs/issues
+- Help: irc.freenode.org/#ipfs
+- Email: dev@ipfs.io
+`

assets/help.go

@@ -0,0 +1,9 @@
+package assets
+var Init_doc_help = `Some helpful resources for finding your way around ipfs:
+
+- quick-start: a quick show of various ipfs features.
+- ipfs commands: a list of all commands
+- ipfs --help: every command describes itself
+- https://github.com/jbenet/go-ipfs -- the src repository
+- #ipfs on irc.freenode.org -- the community irc channel
+`

assets/init-doc/docs/index

@@ -0,0 +1 @@
+Index

assets/init-doc/tour/0.0-intro

@@ -0,0 +1,34 @@
+WIP
+
+# 0.0 - Introduction
+
+Welcome to IPFS! This tour will guide you through a few of the
+features of this tool, and the most common commands. Then, it will
+immerse you into the world of merkledags and the amazing things
+you can do with them.
+
+
+This tour has many parts, and can be taken in different sequences.
+Different people learn different ways, so choose your own adventure:
+
+  To start with the concepts, try:
+  - The Merkle DAG
+  - Data Structures on the Merkle DAG
+  - Representing Files with unixfs
+  - add, cat, ls, refs
+  ...
+
+
+  To start with the examples, try:
+  - add, cat, ls, refs
+  - Representing Files with unixfs
+  - Data Structures on the Merkle DAG
+  - The Merkle DAG
+  ...
+
+
+  To start with the network, try:
+  - IPFS Nodes
+  - Running the daemon
+  - The Swarm
+  - The Web

assets/quick-start.go

@@ -0,0 +1,114 @@
+package assets
+var Init_doc_quick_start = `# 0.1 - Quick Start
+
+This is a set of short examples with minmal explanation. It is meant as
+a "quick start". Soon, we'll write a longer tour :-)
+
+
+Add a file to ipfs:
+
+  echo "hello world" >hello
+  ipfs add hello
+
+
+View it:
+
+  ipfs cat <the-hash-you-got-here>
+
+
+Try a directory:
+
+  mkdir foo
+  mkdir foo/bar
+  echo "baz" > foo/baz
+  echo "baz" > foo/bar/baz
+  ipfs add -r foo
+
+
+View things:
+
+  ipfs ls <the-hash-here>
+  ipfs ls <the-hash-here>/bar
+  ipfs cat <the-hash-here>/baz
+  ipfs cat <the-hash-here>/bar/baz
+  ipfs cat <the-hash-here>/bar
+  ipfs ls <the-hash-here>/baz
+
+
+References:
+
+  ipfs refs <the-hash-here>
+  ipfs refs -r <the-hash-here>
+  ipfs refs --help
+
+
+Get:
+
+  ipfs get <the-hash-here> foo2
+  diff foo foo2
+
+
+Objects:
+
+  ipfs object get <the-hash-here>
+  ipfs object get <the-hash-here>/foo2
+  ipfs object --help
+
+
+Pin + GC:
+
+  ipfs pin -r <the-hash-here>
+  ipfs gc
+  ipfs ls <the-hash-here>
+  ipfs unpin -r <the-hash-here>
+  ipfs gc
+
+
+Daemon:
+
+  ipfs daemon  (in another terminal)
+  ipfs id
+
+
+Network:
+
+  (must be online)
+  ipfs swarm peers
+  ipfs id
+  ipfs cat <hash-of-remote-object>
+
+
+Mount:
+
+  (warning: fuse is finicky!)
+  ipfs mount
+  cd /ipfs/<
+
+
+Tool:
+
+  ipfs version
+  ipfs update
+  ipfs commands
+  ipfs config --help
+  open http://localhost:5001/webui
+
+
+Browse:
+
+  webui:
+
+    http://localhost:5001/webui
+
+  video:
+
+    http://localhost:5001/ipfs/QmVc6zuAneKJzicnJpfrqCH9gSy6bz54JhcypfJYhGUFQu/play#/ipfs/QmTKZgRNwDNZwHtJSjCp6r5FYefzpULfy37JvMt9DwvXse
+
+  images:
+
+    http://localhost:5001/ipfs/QmZpc3HvfjEXvLWGQPWbHk3AjD5j8NEN4gmFN8Jmrd5g83/cs
+
+  markdown renderer app:
+
+    http://localhost:5001/ipfs/QmX7M9CiYXjVeFnkfVGf3y5ixTZ2ACeSGyL1vBJY1HvQPp/mdown
+`

assets/readme.go

@@ -0,0 +1,28 @@
+package assets
+var Init_doc_readme = `Hello and Welcome to IPFS!
+
+██╗██████╗ ███████╗███████╗
+██║██╔══██╗██╔════╝██╔════╝
+██║██████╔╝█████╗  ███████╗
+██║██╔═══╝ ██╔══╝  ╚════██║
+██║██║     ██║     ███████║
+╚═╝╚═╝     ╚═╝     ╚══════╝
+
+If you're seeing this, you have successfully installed
+IPFS and are now interfacing with the ipfs merkledag!
+
+ -------------------------------------------------------
+| Warning:                                              |
+|   This is alpha software. use at your own discretion! |
+|   Much is missing or lacking polish. There are bugs.  |
+|   Not yet secure. Read the security notes for more.   |
+ -------------------------------------------------------
+
+Check out some of the other files in this directory:
+
+  ./about
+  ./help
+  ./quick-start     <-- usage examples
+  ./readme          <-- this file
+  ./security-notes
+`

assets/security-notes.go

@@ -0,0 +1,23 @@
+package assets
+var Init_doc_security_notes = `                    IPFS Alpha Security Notes
+
+We try hard to ensure our system is safe and robust, but all software
+has bugs, especially new software. This distribution is meant to be an
+alpha preview, don't use it for anything mission critical.
+
+Please note the following:
+
+- This is alpha software and has not been audited. It is our goal
+  to conduct a proper security audit once we close in on a 1.0 release.
+
+- ipfs is a networked program, and may have serious undiscovered
+  vulnerabilities. It is written in Go, and we do not execute any
+  user provided data. But please point any problems out to us in a
+  github issue, or email security@ipfs.io privately.
+
+- ipfs uses encryption for all communication, but it's NOT PROVEN SECURE
+  YET!  It may be totally broken. For now, the code is included to make
+  sure we benchmark our operations with encryption in mind. In the future,
+  there will be an "unsafe" mode for high performance intranet apps.
+  If this is a blocking feature for you, please contact us.
+`

cmd/ipfs-gateway-fs/main.go

@@ -44,7 +44,7 @@ func run() error {
 	}
 
 	if !fsrepo.IsInitialized(repoPath) {
-		conf, err := config.Init(*nBitsForKeypair)
+		conf, err := config.Init(os.Stdout, *nBitsForKeypair)
 		if err != nil {
 			return err
 		}

cmd/ipfs/daemon.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"fmt"
+	"bytes"
 
 	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
 	cmds "github.com/jbenet/go-ipfs/commands"
@@ -48,8 +48,10 @@ the daemon.
 }
 
 func daemonFunc(req cmds.Request, res cmds.Response) {
+	var out bytes.Buffer
+	res.SetOutput(&out)
+	writef(&out, "Initializing daemon...\n")
 
-	fmt.Println("Initializing daemon...")
 	// first, whether user has provided the initialization flag. we may be
 	// running in an uninitialized state.
 	initialize, _, err := req.Option(initOptionKwd).Bool()
@@ -57,6 +59,7 @@ func daemonFunc(req cmds.Request, res cmds.Response) {
 		res.SetError(err, cmds.ErrNormal)
 		return
 	}
+
 	if initialize {
 
 		// now, FileExists is our best method of detecting whether IPFS is
@@ -64,7 +67,7 @@ func daemonFunc(req cmds.Request, res cmds.Response) {
 		// `IsInitialized` where the quality of the signal can be improved over
 		// time, and many call-sites can benefit.
 		if !util.FileExists(req.Context().ConfigRoot) {
-			err := initWithDefaults(req.Context().ConfigRoot)
+			err := initWithDefaults(&out, req.Context().ConfigRoot)
 			if err != nil {
 				res.SetError(debugerror.Wrap(err), cmds.ErrNormal)
 				return
@@ -149,8 +152,8 @@ func daemonFunc(req cmds.Request, res cmds.Response) {
 			res.SetError(err, cmds.ErrNormal)
 			return
 		}
-		fmt.Printf("IPFS mounted at: %s\n", fsdir)
-		fmt.Printf("IPNS mounted at: %s\n", nsdir)
+		writef(&out, "IPFS mounted at: %s\n", fsdir)
+		writef(&out, "IPNS mounted at: %s\n", nsdir)
 	}
 
 	var rootRedirect corehttp.ServeOption

cmd/ipfs/init.go

@@ -3,14 +3,17 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"io"
 
 	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
+	assets "github.com/jbenet/go-ipfs/assets"
 	cmds "github.com/jbenet/go-ipfs/commands"
 	core "github.com/jbenet/go-ipfs/core"
 	coreunix "github.com/jbenet/go-ipfs/core/coreunix"
 	ipns "github.com/jbenet/go-ipfs/fuse/ipns"
 	config "github.com/jbenet/go-ipfs/repo/config"
 	fsrepo "github.com/jbenet/go-ipfs/repo/fsrepo"
+	uio "github.com/jbenet/go-ipfs/unixfs/io"
 	u "github.com/jbenet/go-ipfs/util"
 	debugerror "github.com/jbenet/go-ipfs/util/debugerror"
 )
@@ -50,12 +53,15 @@ var initCmd = &cmds.Command{
 			nBitsForKeypair = nBitsForKeypairDefault
 		}
 
-		output, err := doInit(req.Context().ConfigRoot, force, nBitsForKeypair)
-		if err != nil {
-			res.SetError(err, cmds.ErrNormal)
-			return
-		}
-		res.SetOutput(output)
+		rpipe, wpipe := io.Pipe()
+		go func() {
+			defer wpipe.Close()
+			if err := doInit(wpipe, req.Context().ConfigRoot, force, nBitsForKeypair); err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+		}()
+		res.SetOutput(rpipe)
 	},
 }
 
@@ -64,58 +70,45 @@ Reinitializing would overwrite your keys.
 (use -f to force overwrite)
 `)
 
-var welcomeMsg = `Hello and Welcome to IPFS!
-
-██╗██████╗ ███████╗███████╗
-██║██╔══██╗██╔════╝██╔════╝
-██║██████╔╝█████╗  ███████╗
-██║██╔═══╝ ██╔══╝  ╚════██║
-██║██║     ██║     ███████║
-╚═╝╚═╝     ╚═╝     ╚══════╝
-
-If you're seeing this, you have successfully installed
-IPFS and are now interfacing with the ipfs merkledag!
-
-`
-
-func initWithDefaults(repoRoot string) error {
-	_, err := doInit(repoRoot, false, nBitsForKeypairDefault)
+func initWithDefaults(out io.Writer, repoRoot string) error {
+	err := doInit(out, repoRoot, false, nBitsForKeypairDefault)
 	return debugerror.Wrap(err)
 }
 
-func doInit(repoRoot string, force bool, nBitsForKeypair int) (interface{}, error) {
-	u.POut("initializing ipfs node at %s\n", repoRoot)
+func writef(out io.Writer, format string, ifs ...interface{}) error {
+	_, err := out.Write([]byte(fmt.Sprintf(format, ifs...)))
+	return err
+}
+
+func doInit(out io.Writer, repoRoot string, force bool, nBitsForKeypair int) error {
+	if err := writef(out, "initializing ipfs node at %s\n", repoRoot); err != nil {
+		return err
+	}
 
 	if fsrepo.IsInitialized(repoRoot) && !force {
-		return nil, errRepoExists
+		return errRepoExists
 	}
-	conf, err := config.Init(nBitsForKeypair)
+	conf, err := config.Init(out, nBitsForKeypair)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	if fsrepo.IsInitialized(repoRoot) {
 		if err := fsrepo.Remove(repoRoot); err != nil {
-			return nil, err
+			return err
 		}
 	}
 	if err := fsrepo.Init(repoRoot, conf); err != nil {
-		return nil, err
+		return err
 	}
 
-	err = addTheWelcomeFile(repoRoot)
-	if err != nil {
-		return nil, err
+	if err := addDefaultAssets(out, repoRoot); err != nil {
+		return err
 	}
-	err = initializeIpnsKeyspace(repoRoot)
-	if err != nil {
-		return nil, err
-	}
-	return nil, nil
+
+	return initializeIpnsKeyspace(repoRoot)
 }
 
-// addTheWelcomeFile adds a file containing the welcome message to the newly
-// minted node.
-func addTheWelcomeFile(repoRoot string) error {
+func addDefaultAssets(out io.Writer, repoRoot string) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	r := fsrepo.At(repoRoot)
@@ -128,14 +121,35 @@ func addTheWelcomeFile(repoRoot string) error {
 	}
 	defer nd.Close()
 
-	// Set up default file
-	reader := bytes.NewBufferString(welcomeMsg)
-	k, err := coreunix.Add(nd, reader)
-	if err != nil {
-		return fmt.Errorf("failed to write test file: %s", err)
+	dirb := uio.NewDirectory(nd.DAG)
+
+	// add every file in the assets pkg
+	for fname, file := range assets.Init_dir {
+		buf := bytes.NewBufferString(file)
+		s, err := coreunix.Add(nd, buf)
+		if err != nil {
+			return err
+		}
+		k := u.B58KeyDecode(s)
+		if err := dirb.AddChild(fname, k); err != nil {
+			return err
+		}
 	}
-	fmt.Printf("\nto get started, enter: ipfs cat %s\n", k)
-	return nil
+
+	dir := dirb.GetNode()
+	dkey, err := nd.DAG.Add(dir)
+	if err != nil {
+		return err
+	}
+	if err := nd.Pinning.Pin(dir, true); err != nil {
+		return err
+	}
+	if err := nd.Pinning.Flush(); err != nil {
+		return err
+	}
+
+	writef(out, "to get started, enter:\n")
+	return writef(out, "\n\tipfs cat /ipfs/%s/readme\n\n", dkey)
 }
 
 func initializeIpnsKeyspace(repoRoot string) error {

repo/config/init.go

@@ -3,19 +3,20 @@ package config
 import (
 	"encoding/base64"
 	"fmt"
+	"io"
 
 	ci "github.com/jbenet/go-ipfs/p2p/crypto"
 	peer "github.com/jbenet/go-ipfs/p2p/peer"
 	errors "github.com/jbenet/go-ipfs/util/debugerror"
 )
 
-func Init(nBitsForKeypair int) (*Config, error) {
+func Init(out io.Writer, nBitsForKeypair int) (*Config, error) {
 	ds, err := datastoreConfig()
 	if err != nil {
 		return nil, err
 	}
 
-	identity, err := identityConfig(nBitsForKeypair)
+	identity, err := identityConfig(out, nBitsForKeypair)
 	if err != nil {
 		return nil, err
 	}
@@ -71,19 +72,19 @@ func datastoreConfig() (*Datastore, error) {
 }
 
 // identityConfig initializes a new identity.
-func identityConfig(nbits int) (Identity, error) {
+func identityConfig(out io.Writer, nbits int) (Identity, error) {
 	// TODO guard higher up
 	ident := Identity{}
 	if nbits < 1024 {
 		return ident, errors.New("Bitsize less than 1024 is considered unsafe.")
 	}
 
-	fmt.Printf("generating %v-bit RSA keypair...", nbits)
+	out.Write([]byte(fmt.Sprintf("generating %v-bit RSA keypair...", nbits)))
 	sk, pk, err := ci.GenerateKeyPair(ci.RSA, nbits)
 	if err != nil {
 		return ident, err
 	}
-	fmt.Printf("done\n")
+	out.Write([]byte(fmt.Sprintf("done\n")))
 
 	// currently storing key unencrypted. in the future we need to encrypt it.
 	// TODO(security)

test/sharness/lib/test-lib.sh

@@ -83,16 +83,16 @@ test_init_ipfs() {
 test_launch_ipfs_daemon() {
 
 	test_expect_success "'ipfs daemon' succeeds" '
-		ipfs daemon >actual 2>daemon_err &
+		ipfs daemon >actual_daemon 2>daemon_err &
 	'
 
 	test_expect_success "'ipfs daemon' output looks good" '
 		IPFS_PID=$! &&
-		echo "Initializing daemon..." >expected &&
-		echo "API server listening on /ip4/127.0.0.1/tcp/5001" >>expected &&
-		test_cmp_repeat_10_sec expected actual ||
+		echo "API server listening on /ip4/127.0.0.1/tcp/5001" >expected_daemon &&
+		test_cmp_repeat_10_sec expected_daemon actual_daemon ||
 		fsh cat daemon_err
 	'
+
 }
 
 test_mount_ipfs() {

test/sharness/t0020-init.sh

@@ -21,24 +21,26 @@ test_expect_success ".go-ipfs/ has been created" '
 '
 
 test_expect_success "ipfs config succeeds" '
-	echo leveldb >expected &&
-	ipfs config Datastore.Type >actual &&
-	test_cmp expected actual
+	echo leveldb >expected_config &&
+	ipfs config Datastore.Type >actual_config &&
+	test_cmp expected_config actual_config
 '
 
 test_expect_success "ipfs peer id looks good" '
 	PEERID=$(ipfs config Identity.PeerID) &&
-	echo $PEERID | tr -dC "[:alnum:]" | wc -c | tr -d " " >actual &&
-	echo "46" >expected &&
-	test_cmp expected actual
+	echo $PEERID | tr -dC "[:alnum:]" | wc -c | tr -d " " >actual_peerid &&
+	echo "46" >expected_peerid &&
+	test_cmp expected_peerid actual_peerid
 '
 
 test_expect_success "ipfs init output looks good" '
-	STARTHASH="QmTTFXiXoixwT53tcGPu419udsHEHYu6AHrQC8HAKdJYaZ" &&
+	STARTHASH="QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT" &&
+	STARTFILE="ipfs cat /ipfs/$STARTHASH/readme"
 	echo "initializing ipfs node at $IPFS_PATH" >expected &&
 	echo "generating 4096-bit RSA keypair...done" >>expected &&
 	echo "peer identity: $PEERID" >>expected &&
-	printf "\\n%s\\n" "to get started, enter: ipfs cat $STARTHASH" >>expected &&
+	echo "to get started, enter:" >>expected &&
+	printf "\\n\\t$STARTFILE\\n\\n" >>expected &&
 	test_cmp expected actual_init
 '
 

test/sharness/t0080-repo.sh

@@ -42,10 +42,12 @@ test_expect_success "'ipfs pin rm' succeeds" '
 '
 
 test_expect_success "file no longer pinned" '
-	# we expect the welcome file to show up here
-	echo QmTTFXiXoixwT53tcGPu419udsHEHYu6AHrQC8HAKdJYaZ >expected2
-	ipfs pin ls -type=recursive >actual2
-	test_cmp expected2 actual2
+	# we expect the welcome files to show up here
+	echo QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT >expected2
+	ipfs refs -r QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT >>expected2
+	cat expected2 | sort >expected_sorted2
+	ipfs pin ls -type=recursive | sort >actual2
+	test_cmp expected_sorted2 actual2
 '
 
 test_expect_success "recursively pin afile" '
@@ -84,10 +86,12 @@ test_expect_success "'ipfs repo gc' removes file" '
 '
 
 test_expect_success "'ipfs refs local' no longer shows file" '
-	echo QmTTFXiXoixwT53tcGPu419udsHEHYu6AHrQC8HAKdJYaZ >expected8
-	echo QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn >>expected8
-	ipfs refs local >actual8
-	test_cmp expected8 actual8
+	echo QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn >expected8
+	echo QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT >>expected8
+	ipfs refs -r QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT >>expected8
+	cat expected8 | sort >expected_sorted8
+	ipfs refs local | sort >actual8
+	test_cmp expected_sorted8 actual8
 '
 
 test_expect_success "adding multiblock random file succeeds" '
@@ -96,9 +100,11 @@ test_expect_success "adding multiblock random file succeeds" '
 '
 
 test_expect_success "'ipfs pin ls -type=indirect' is correct" '
-	ipfs refs "$MBLOCKHASH" | sort >refsout
+	ipfs refs "$MBLOCKHASH" >refsout
+	ipfs refs -r "QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT" >>refsout
+	cat refsout | sort >refsout_sorted
 	ipfs pin ls -type=indirect | sort >indirectpins
-	test_cmp refsout indirectpins
+	test_cmp refsout_sorted indirectpins
 '
 
 test_expect_success "pin something directly" '
@@ -123,7 +129,8 @@ test_expect_success "'ipfs pin ls -type=direct' is correct" '
 
 test_expect_success "'ipfs pin ls -type=recursive' is correct" '
 	echo "$MBLOCKHASH" >rp_expected
-	echo QmTTFXiXoixwT53tcGPu419udsHEHYu6AHrQC8HAKdJYaZ >>rp_expected
+	echo QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT >>rp_expected
+	ipfs refs -r "QmPXME1oRtoT627YKaDPDQ3PwA8tdP9rWuAAweLzqSwAWT" >>rp_expected
 	cat rp_expected | sort >rp_exp_sorted
 	ipfs pin ls -type=recursive | sort >rp_actual
 	test_cmp rp_exp_sorted rp_actual

unixfs/io/dirbuilder.go

@@ -0,0 +1,38 @@
+package io
+
+import (
+	mdag "github.com/jbenet/go-ipfs/merkledag"
+	format "github.com/jbenet/go-ipfs/unixfs"
+	u "github.com/jbenet/go-ipfs/util"
+)
+
+type directoryBuilder struct {
+	dserv   mdag.DAGService
+	dirnode *mdag.Node
+}
+
+func NewDirectory(dserv mdag.DAGService) *directoryBuilder {
+	db := new(directoryBuilder)
+	db.dserv = dserv
+	db.dirnode = new(mdag.Node)
+	db.dirnode.Data = format.FolderPBData()
+	return db
+}
+
+func (d *directoryBuilder) AddChild(name string, k u.Key) error {
+	cnode, err := d.dserv.Get(k)
+	if err != nil {
+		return err
+	}
+
+	err = d.dirnode.AddNodeLinkClean(name, cnode)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *directoryBuilder) GetNode() *mdag.Node {
+	return d.dirnode
+}