Merge pull request #67 from JackalLabs/marston/ipfs-folders-again

Implementing a new Proof Type for Raw IPFS Nodes

api/file_handler.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"net/http"
 	"strconv"
-	"strings"
 	"time"
 
 	cid "github.com/ipfs/go-cid"
@@ -60,6 +59,16 @@ func PostFileHandler(fio *file_system.FileSystem, prover *proofs.Prover, wl *wal
 			return
 		}
 
+		proofTypeString := req.Form.Get("type")
+		if len(proofTypeString) == 0 {
+			proofTypeString = "0"
+		}
+		proofType, err := strconv.ParseInt(proofTypeString, 10, 64)
+		if err != nil {
+			handleErr(fmt.Errorf("cannot parse proof type: %w", err), w, http.StatusBadRequest)
+			return
+		}
+
 		file, fh, err := req.FormFile("file") // Retrieve the file from form data
 		if err != nil {
 			handleErr(fmt.Errorf("cannot get file from form: %w", err), w, http.StatusBadRequest)
@@ -103,7 +112,7 @@ func PostFileHandler(fio *file_system.FileSystem, prover *proofs.Prover, wl *wal
 			}
 		}
 
-		size, c, err := fio.WriteFile(file, merkle, sender, startBlock, wl.AccAddress(), chunkSize)
+		size, c, err := fio.WriteFile(file, merkle, sender, startBlock, wl.AccAddress(), chunkSize, proofType)
 		if err != nil {
 			handleErr(fmt.Errorf("failed to write file to disk: %w", err), w, http.StatusInternalServerError)
 			return
@@ -139,19 +148,24 @@ func PostIPFSFolder(f *file_system.FileSystem) func(http.ResponseWriter, *http.R
 		}
 		defer req.Body.Close()
 
-		cidList := strings.Split(string(body), ",")
-
-		childCIDs := make([]cid.Cid, len(cidList))
+		var cidList map[string]string
 
-		fmt.Println(cidList)
+		err = json.Unmarshal(body, &cidList)
+		if err != nil {
+			if err != nil {
+				http.Error(w, "Error parsing request body", http.StatusInternalServerError)
+				return
+			}
+		}
 
-		for i, s := range cidList {
+		childCIDs := make(map[string]cid.Cid)
+		for key, s := range cidList {
 			c, err := cid.Parse(s)
 			if err != nil {
 				http.Error(w, fmt.Sprintf("Could not parse %s", s), http.StatusInternalServerError)
 				return
 			}
-			childCIDs[i] = c
+			childCIDs[key] = c
 		}
 
 		root, err := f.CreateIPFSFolder(childCIDs)
@@ -161,7 +175,8 @@ func PostIPFSFolder(f *file_system.FileSystem) func(http.ResponseWriter, *http.R
 		}
 
 		f := types.CidFolderResponse{
-			Cid: root,
+			Cid:  root.Cid().String(),
+			Data: root.RawData(),
 		}
 
 		err = json.NewEncoder(w).Encode(f)

api/types/responses.go

@@ -65,7 +65,8 @@ type CidResponse struct {
 }
 
 type CidFolderResponse struct {
-	Cid string `json:"cid"`
+	Cid  string `json:"cid"`
+	Data []byte `json:"data"`
 }
 
 type CidMapResponse struct {

file_system/file_system.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/ipfs/boxo/ipld/merkledag"
+
 	"github.com/ipfs/boxo/ipld/unixfs"
 
 	"github.com/dgraph-io/badger/v4"
@@ -73,7 +75,7 @@ func BuildTree(buf io.Reader, chunkSize int64) ([]byte, []byte, [][]byte, int, e
 	return r, exportedTree, chunks, size, nil
 }
 
-func (f *FileSystem) WriteFile(reader io.Reader, merkle []byte, owner string, start int64, address string, chunkSize int64) (size int, cid string, err error) {
+func (f *FileSystem) WriteFile(reader io.Reader, merkle []byte, owner string, start int64, address string, chunkSize int64, proofType int64) (size int, cid string, err error) {
 	log.Info().Msg(fmt.Sprintf("Writing %x to disk", merkle))
 	root, exportedTree, chunks, s, err := BuildTree(reader, chunkSize)
 	if err != nil {
@@ -91,9 +93,24 @@ func (f *FileSystem) WriteFile(reader io.Reader, merkle []byte, owner string, st
 	}
 	buf := bytes.NewBuffer(b)
 
-	n, err := f.ipfs.AddFile(context.Background(), buf, nil)
-	if err != nil {
-		return 0, "", err
+	var n ipldFormat.Node
+	if proofType == 1 {
+		folderNode := unixfs.EmptyDirNode()
+		err := folderNode.UnmarshalJSON(buf.Bytes())
+		if err != nil {
+			return 0, "", err
+		}
+
+		err = f.ipfs.Add(context.Background(), folderNode)
+		if err != nil {
+			return 0, "", err
+		}
+		n = folderNode
+	} else {
+		n, err = f.ipfs.AddFile(context.Background(), buf, nil)
+		if err != nil {
+			return 0, "", err
+		}
 	}
 
 	err = f.db.Update(func(txn *badger.Txn) error {
@@ -116,31 +133,39 @@ func (f *FileSystem) WriteFile(reader io.Reader, merkle []byte, owner string, st
 	return size, n.Cid().String(), nil
 }
 
-func (f *FileSystem) CreateIPFSFolder(childCIDs []cid.Cid) (cidRes string, err error) {
+func (f *FileSystem) CreateIPFSFolder(childCIDs map[string]cid.Cid) (node ipldFormat.Node, err error) {
+	n, err := f.GenIPFSFolderData(childCIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	// Add the folder node to the DAG service
+	err = f.ipfs.Add(context.Background(), n)
+	if err != nil {
+		return nil, err
+	}
+
+	return n, nil
+}
+
+func (f *FileSystem) GenIPFSFolderData(childCIDs map[string]cid.Cid) (node ipldFormat.Node, err error) {
 	folderNode := unixfs.EmptyDirNode()
 
-	for i, childCID := range childCIDs {
+	for key, childCID := range childCIDs {
 		// Create a link
-		linkName := fmt.Sprintf("%d", i)
 		link := &ipldFormat.Link{
-			Name: linkName,
+			Name: key,
 			Cid:  childCID,
 		}
 
 		// Add the link to the folder node
-		err := folderNode.AddRawLink(linkName, link)
+		err := folderNode.AddRawLink(key, link)
 		if err != nil {
-			return "", err
+			return nil, err
 		}
 	}
 
-	// Add the folder node to the DAG service
-	err = f.ipfs.Add(context.Background(), folderNode)
-	if err != nil {
-		return "", err
-	}
-
-	return folderNode.Cid().String(), nil
+	return folderNode, nil
 }
 
 func (f *FileSystem) DeleteFile(merkle []byte, owner string, start int64) error {
@@ -281,7 +306,7 @@ func (f *FileSystem) Dump() (map[string]string, error) {
 	return files, err
 }
 
-func (f *FileSystem) GetFileTreeByChunk(merkle []byte, owner string, start int64, chunkToLoad int, chunkSize int) (*merkletree.MerkleTree, []byte, error) {
+func (f *FileSystem) GetFileTreeByChunk(merkle []byte, owner string, start int64, chunkToLoad int, chunkSize int, proofType int64) (*merkletree.MerkleTree, []byte, error) {
 	tree := treeKey(merkle, owner, start)
 
 	var newTree merkletree.MerkleTree
@@ -330,9 +355,22 @@ func (f *FileSystem) GetFileTreeByChunk(merkle []byte, owner string, start int64
 		return nil, nil, fmt.Errorf("failed to decode cid: %s | %w", fcid, err)
 	}
 
-	chunkOut, err := f.ipfs.GetFileChunk(context.Background(), c, chunkToLoad, chunkSize)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to get chunk from unixfs %w", err)
+	var chunkOut []byte
+	if proofType == 1 {
+		n, err := f.ipfs.Get(context.Background(), c)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to get node chunk for cid: %s | %w", fcid, err)
+		}
+		data, err := n.(*merkledag.ProtoNode).MarshalJSON()
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to cast proto node: %s | %w", fcid, err)
+		}
+		chunkOut = data
+	} else {
+		chunkOut, err = f.ipfs.GetFileChunk(context.Background(), c, chunkToLoad, chunkSize)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to get chunk from unixfs %w", err)
+		}
 	}
 
 	if chunkOut == nil {

file_system/file_system_test.go

@@ -81,7 +81,7 @@ func BenchmarkFileWrites(b *testing.B) {
 				root, _, _, _, err := BuildTree(buf2, 10240)
 				require.NoError(b, err)
 
-				_, _, err = f.WriteFile(buf, root, "file_owner", 0, "myself", 10240)
+				_, _, err = f.WriteFile(buf, root, "file_owner", 0, "myself", 10240, 0)
 				require.NoError(b, err)
 
 			}
@@ -115,7 +115,7 @@ func TestWriteFile(t *testing.T) {
 	m, err := hex.DecodeString(merkle)
 	require.NoError(t, err)
 
-	size, _, err := f.WriteFile(buf, m, "file_owner", 0, "myself", 1024)
+	size, _, err := f.WriteFile(buf, m, "file_owner", 0, "myself", 1024, 0)
 
 	require.NoError(t, err)
 
@@ -165,7 +165,7 @@ func TestWriteFileWithDomain(t *testing.T) {
 	m, err := hex.DecodeString(merkle)
 	require.NoError(t, err)
 
-	size, _, err := f.WriteFile(buf, m, "file_owner", 0, "myself", 1024)
+	size, _, err := f.WriteFile(buf, m, "file_owner", 0, "myself", 1024, 0)
 
 	require.NoError(t, err)
 
@@ -228,7 +228,7 @@ func TestWriteAndProveFiles(t *testing.T) {
 	owner := "file_owner"
 	var start int64 = 0
 
-	_, _, err = f.WriteFile(b2, root, owner, start, "myself", chunkSize)
+	_, _, err = f.WriteFile(b2, root, owner, start, "myself", chunkSize, 0)
 	require.NoError(t, err)
 
 	ms, _, _, err := f.ListFiles()
@@ -239,7 +239,7 @@ func TestWriteAndProveFiles(t *testing.T) {
 	totalBlocks := size / int(chunkSize)
 	for i := 0; i < totalBlocks; i++ {
 
-		p, c, err := proofs.GenProof(f, root, owner, start, i, int(chunkSize))
+		p, c, err := proofs.GenProof(f, root, owner, start, i, int(chunkSize), 0)
 		require.NoError(t, err)
 
 		h := sha256.New()

ipfs/ipfs.go

@@ -5,13 +5,15 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/libp2p/go-libp2p"
 	"github.com/libp2p/go-libp2p/core/host"
+	libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls"
 
 	ipfslite "github.com/hsanjuan/ipfs-lite"
 	"github.com/ipfs/boxo/blockstore"
-	datastore "github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore"
 	"github.com/libp2p/go-libp2p/core/crypto"
-	multiaddr "github.com/multiformats/go-multiaddr"
+	"github.com/multiformats/go-multiaddr"
 )
 
 func MakeIPFS(ctx context.Context, ds datastore.Batching, bs blockstore.Blockstore, port int, customDomain string) (*ipfslite.Peer, host.Host, error) {
@@ -20,16 +22,22 @@ func MakeIPFS(ctx context.Context, ds datastore.Batching, bs blockstore.Blocksto
 		return nil, nil, err
 	}
 
+	defaultPort, err := multiaddr.NewMultiaddr("/ip4/0.0.0.0/tcp/4001")
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to make ipv4 ipfs address | %w", err)
+	}
+
 	listen, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", port))
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to make ipv4 ipfs address | %w", err)
 	}
+
 	listen6, err := multiaddr.NewMultiaddr(fmt.Sprintf("/ip6/::/tcp/%d", port))
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to make ipv6 ipfs address | %w", err)
 	}
 
-	m := []multiaddr.Multiaddr{listen, listen6}
+	m := []multiaddr.Multiaddr{listen, listen6, defaultPort}
 
 	if !strings.Contains(customDomain, "example.com") && len(customDomain) > 2 {
 		if !strings.HasPrefix(customDomain, "/") {
@@ -42,13 +50,17 @@ func MakeIPFS(ctx context.Context, ds datastore.Batching, bs blockstore.Blocksto
 		m = append(m, domainListener)
 	}
 
+	opts := libp2p.ChainOptions(
+		libp2p.Security(libp2ptls.ID, libp2ptls.New),
+	)
+
 	h, dht, err := ipfslite.SetupLibp2p(
 		ctx,
 		priv,
 		nil,
 		m,
 		ds,
-		ipfslite.Libp2pOptionsExtra...,
+		append(ipfslite.Libp2pOptionsExtra, opts)...,
 	)
 	if err != nil {
 		return nil, h, err

network/downloads.go

@@ -13,7 +13,7 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-func DownloadFile(f *file_system.FileSystem, merkle []byte, owner string, start int64, wallet *wallet.Wallet, fileSize int64, myUrl string, chunkSize int64) error {
+func DownloadFile(f *file_system.FileSystem, merkle []byte, owner string, start int64, wallet *wallet.Wallet, fileSize int64, myUrl string, chunkSize int64, proofType int64) error {
 	queryParams := &types.QueryFindFile{
 		Merkle: merkle,
 	}
@@ -37,7 +37,7 @@ func DownloadFile(f *file_system.FileSystem, merkle []byte, owner string, start
 			continue
 		}
 
-		size, err := DownloadFileFromURL(f, url, merkle, owner, start, wallet.AccAddress(), chunkSize)
+		size, err := DownloadFileFromURL(f, url, merkle, owner, start, wallet.AccAddress(), chunkSize, proofType)
 		if err != nil {
 			log.Info().Msg(fmt.Sprintf("Couldn't get %x from %s, trying again...", merkle, url))
 			continue
@@ -58,7 +58,7 @@ func DownloadFile(f *file_system.FileSystem, merkle []byte, owner string, start
 	return nil
 }
 
-func DownloadFileFromURL(f *file_system.FileSystem, url string, merkle []byte, owner string, start int64, address string, chunkSize int64) (int, error) {
+func DownloadFileFromURL(f *file_system.FileSystem, url string, merkle []byte, owner string, start int64, address string, chunkSize int64, proofType int64) (int, error) {
 	log.Info().Msg(fmt.Sprintf("Downloading %x from %s...", merkle, url))
 	cli := http.Client{}
 	req, err := http.NewRequest("GET", fmt.Sprintf("%s/download/%x", url, merkle), nil)
@@ -93,7 +93,7 @@ func DownloadFileFromURL(f *file_system.FileSystem, url string, merkle []byte, o
 
 	reader := bytes.NewReader(buff.Bytes())
 
-	size, _, err := f.WriteFile(reader, merkle, owner, start, address, chunkSize)
+	size, _, err := f.WriteFile(reader, merkle, owner, start, address, chunkSize, proofType)
 	if err != nil {
 		return 0, err
 	}

proofs/proofs.go

@@ -51,8 +51,8 @@ func GenerateMerkleProof(tree *merkletree.MerkleTree, index int, item []byte) (b
 // GenProof generates a proof from an arbitrary file on the file system
 //
 // returns proof, item and error
-func GenProof(io FileSystem, merkle []byte, owner string, start int64, block int, chunkSize int) ([]byte, []byte, error) {
-	tree, chunk, err := io.GetFileTreeByChunk(merkle, owner, start, block, chunkSize)
+func GenProof(io FileSystem, merkle []byte, owner string, start int64, block int, chunkSize int, proofType int64) ([]byte, []byte, error) {
+	tree, chunk, err := io.GetFileTreeByChunk(merkle, owner, start, block, chunkSize, proofType)
 	if err != nil {
 		e := fmt.Errorf("cannot get chunk for %x at %d | %w", merkle, block, err)
 		log.Error().Err(e)
@@ -143,7 +143,7 @@ func (p *Prover) GenerateProof(merkle []byte, owner string, start int64, blockHe
 
 	log.Debug().Msg(fmt.Sprintf("Getting file tree by chunk for %x", merkle))
 
-	proof, item, err := GenProof(p.io, merkle, owner, start, block, p.chunkSize)
+	proof, item, err := GenProof(p.io, merkle, owner, start, block, p.chunkSize, file.ProofType)
 
 	return proof, item, newProof.ChunkToProve, err
 }

proofs/types.go

@@ -24,7 +24,7 @@ type Prover struct {
 type FileSystem interface {
 	DeleteFile([]byte, string, int64) error
 	ProcessFiles(func([]byte, string, int64)) error
-	GetFileTreeByChunk([]byte, string, int64, int, int) (*merkletree.MerkleTree, []byte, error)
+	GetFileTreeByChunk([]byte, string, int64, int, int, int64) (*merkletree.MerkleTree, []byte, error)
 }
 
 func (p *Prover) Inc() {

recycle/recycle.go

@@ -207,7 +207,7 @@ func (r *RecycleDepot) activateFile(openFile types.UnifiedFile) (size int, cid s
 		openFile.Owner,
 		openFile.Start,
 		"",
-		r.chunkSize)
+		r.chunkSize, 0)
 	if err != nil {
 		return 0, "", fmt.Errorf("could not write file | %w", err)
 	}