Merge pull request #109 from covalenthq/develop

IPFS-Pinner-DTM-RC:v1.1.0

Dockerfile

@@ -17,7 +17,7 @@ RUN apk del git && rm -rf /var/cache/apk/* /root/.npm /tmp/*
 HEALTHCHECK --interval=10s --timeout=5s CMD wget --no-verbose --tries=1 --spider localhost:3001/health
 
 ENTRYPOINT [ "/bin/bash", "-l", "-c" ]
-CMD [ "./ipfs-server -port 3001 -w3-agent-key $W3_AGENT_KEY -w3-delegation-file $W3_DELEGATION_FILE" ]
+CMD [ "./ipfs-server -port 3001 -w3-agent-key $W3_AGENT_KEY -w3-delegation-file $W3_DELEGATION_FILE --enable-gc" ]
 
 # ipfs-pinner API server;
 EXPOSE 3001

README.md

@@ -66,6 +66,9 @@ NOTE: If you want more control over CLI params, you can run the server binary (a
 
 NOTE: If you get some error when running this, check if the diagnostic is there in [known issues](#known-issues)
 
+NOTE: If you want to run the server with garbage collection enabled, you can run the server binary with the `--enable-gc` flag:
+Garbage Collector can be configured through IPFS config file `~/.ipfs/config`. More information can be found [here](https://github.com/ipfs/kubo/blob/master/docs/config.md#datastore)
+
 ipfs-pinner can be run as a server and allows two functionalities currently - `/get` and `/upload`
 
 ### Upload a file

binary/main.go

@@ -18,11 +18,11 @@ var WEB3_JWT = "WEB3_JWT"
 var UPLOAD_FILE = "./main.go" // uploading current file itself
 
 func main() {
+	ctx := context.Background()
 	clientCreateReq := client.NewClientRequest(core.Web3Storage)
 	// check if cid compute true works with car uploads
-	nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, []string{"https://w3s.link/ipfs/%s"}).CidVersion(1).CidComputeOnly(false)
-	node := pinner.NewPinnerNode(*nodeCreateReq)
-	ctx := context.Background()
+	nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, []string{"https://w3s.link/ipfs/%s"}, false).CidVersion(1).CidComputeOnly(false)
+	node := pinner.NewPinnerNode(ctx, *nodeCreateReq)
 	//upload(ctx, node)
 	core.Version()
 	download(ctx, node)

core/ipfs.go

@@ -15,7 +15,7 @@ import (
 )
 
 // returns a go-ipfs node backend CoreAPI instance
-func CreateIpfsNode(cidComputeOnly bool) (*core.IpfsNode, error) {
+func CreateIpfsNode(ctx context.Context, cidComputeOnly bool) (*core.IpfsNode, error) {
 	cfg := core.BuildCfg{
 		Online:    !cidComputeOnly, // networking
 		Permanent: !cidComputeOnly, // data persists across restarts?
@@ -50,7 +50,7 @@ func CreateIpfsNode(cidComputeOnly bool) (*core.IpfsNode, error) {
 	}
 
 	var nnode *core.IpfsNode
-	if nnode, err = core.NewNode(context.Background(), &cfg); err != nil {
+	if nnode, err = core.NewNode(ctx, &cfg); err != nil {
 		return nil, err
 	}
 

coreapi/api.go

@@ -15,4 +15,5 @@ type CoreExtensionAPI interface {
 
 type GarbageCollectAPI interface {
 	GarbageCollect(ctx context.Context)
+	InitPeriodicGC(ctx context.Context) <-chan error
 }

coreapi/impl.go

@@ -43,3 +43,12 @@ func (gci *garbageCollectorImpl) GarbageCollect(ctx context.Context) {
 		log.Println("error getting garbage collector: %w", err)
 	}
 }
+
+func (gci *garbageCollectorImpl) InitPeriodicGC(ctx context.Context) <-chan error {
+	errc := make(chan error)
+	go func() {
+		errc <- corerepo.PeriodicGC(ctx, gci.node)
+		close(errc)
+	}()
+	return errc
+}

pinclient/client_create_request.go

@@ -16,6 +16,7 @@ type ClientCreateRequest struct {
 	W3_AgentKey            string
 	W3_AgentDid            did.DID
 	W3_DelegationProofPath string
+	GC_Enable              bool
 
 	httpClient *http.Client
 }
@@ -56,3 +57,8 @@ func (r ClientCreateRequest) HttpClient(client http.Client) ClientCreateRequest
 	r.httpClient = &client
 	return r
 }
+
+func (r ClientCreateRequest) GcEnable(gcEnable bool) ClientCreateRequest {
+	r.GC_Enable = gcEnable
+	return r
+}

pinner.go

@@ -5,6 +5,7 @@
 package pinner
 
 import (
+	"context"
 	"log"
 
 	car "github.com/covalenthq/ipfs-pinner/car"
@@ -22,14 +23,18 @@ type pinnerNode struct {
 	pinApiClient  pinclient.PinServiceAPI
 }
 
-func NewPinnerNode(req PinnerNodeCreateRequest) PinnerNode {
+func NewPinnerNode(ctx context.Context, req PinnerNodeCreateRequest) PinnerNode {
 	node := pinnerNode{}
-	ipfsNode, err := core.CreateIpfsNode(req.cidComputeOnly)
+	ipfsNode, err := core.CreateIpfsNode(ctx, req.cidComputeOnly)
 	if err != nil {
 		log.Fatal("error initializing ipfs node: ", err)
 	}
 
 	node.ipfsCore = coreapi.NewCoreExtensionApi(ipfsNode)
+	if req.enableGC {
+		log.Print("enabling garbage collection....")
+		node.ipfsCore.GC().InitPeriodicGC(ctx)
+	}
 
 	//SETUP W3UP
 	log.Print("setting up w3up for uploads....")

pinner_create_request.go

@@ -9,14 +9,16 @@ type PinnerNodeCreateRequest struct {
 	cidComputeOnly    bool
 	cidVersion        int
 	ipfsFetchUrls     []string
+	enableGC          bool
 }
 
-func NewNodeRequest(clientRequest pinclient.ClientCreateRequest, ipfsFetchUrls []string) *PinnerNodeCreateRequest {
+func NewNodeRequest(clientRequest pinclient.ClientCreateRequest, ipfsFetchUrls []string, enableGC bool) *PinnerNodeCreateRequest {
 	request := new(PinnerNodeCreateRequest)
 	request.cidVersion = 0
 	request.cidComputeOnly = true
 	request.pinServiceRequest = clientRequest
 	request.ipfsFetchUrls = ipfsFetchUrls
+	request.enableGC = enableGC
 	return request
 }
 

server/main.go

@@ -45,6 +45,11 @@ type Config struct {
 	w3AgentDid          did.DID
 	delegationProofPath string
 	ipfsGatewayUrls     []string
+	enableGC            bool
+}
+
+func NewConfig(portNumber int, w3AgentKey string, w3AgentDid did.DID, delegationProofPath string, ipfsGatewayUrls []string, enableGC bool) *Config {
+	return &Config{portNumber, w3AgentKey, w3AgentDid, delegationProofPath, ipfsGatewayUrls, enableGC}
 }
 
 var (
@@ -62,6 +67,8 @@ func main() {
 
 	ipfsGatewayUrls := flag.String("ipfs-gateway-urls", "https://w3s.link/ipfs/%s,https://dweb.link/ipfs/%s,https://ipfs.io/ipfs/%s", "comma separated list of ipfs gateway urls")
 
+	enableGC := flag.Bool("enable-gc", false, "enable garbage collection")
+
 	flag.Parse()
 	core.Version()
 
@@ -85,17 +92,25 @@ func main() {
 
 	log.Printf("agent did: %s", agentSigner.DID().DID().String())
 
-	setUpAndRunServer(Config{*portNumber, *w3AgentKey, agentSigner.DID().DID(), *w3DelegationFile, strings.Split(*ipfsGatewayUrls, ",")})
+	config := NewConfig(*portNumber, *w3AgentKey, agentSigner.DID().DID(), *w3DelegationFile, strings.Split(*ipfsGatewayUrls, ","), *enableGC)
+
+	setUpAndRunServer(*config)
 }
 
 func setUpAndRunServer(config Config) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 	mux := http.NewServeMux()
 	httpState := NewState()
 
-	clientCreateReq := client.NewClientRequest(core.Web3Storage).W3AgentKey(config.w3AgentKey).W3AgentDid(config.w3AgentDid).DelegationProofPath(config.delegationProofPath)
+	clientCreateReq := client.NewClientRequest(core.Web3Storage).
+		W3AgentKey(config.w3AgentKey).
+		W3AgentDid(config.w3AgentDid).
+		DelegationProofPath(config.delegationProofPath).
+		GcEnable(config.enableGC)
 	// check if cid compute true works with car uploads
-	nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, config.ipfsGatewayUrls).CidVersion(1).CidComputeOnly(false)
-	node := pinner.NewPinnerNode(*nodeCreateReq)
+	nodeCreateReq := pinner.NewNodeRequest(clientCreateReq, config.ipfsGatewayUrls, config.enableGC).CidVersion(1).CidComputeOnly(false)
+	node := pinner.NewPinnerNode(ctx, *nodeCreateReq)
 
 	mux.Handle("/upload", recoveryWrapper(uploadHttpHandler(node)))
 	mux.Handle("/get", recoveryWrapper(downloadHttpHandler(node)))