Adds secret support (#14)

Also:

1. Fix bug with updating a function
2. Fix bug listing more than 10 functions
3. Add log support for functions
4. Proxy invoke function using service discovery hostname

Resolves #1

Signed-off-by: Edward Wilde <[email protected]>

.gitignore

@@ -1,3 +1,4 @@
 .idea
 examples/hellogoworld2/
 dist
+faas-fargate

Dockerfile

@@ -11,7 +11,7 @@ COPY . .
 
 RUN curl -sL https://github.com/alexellis/license-check/releases/download/0.2.2/license-check > /usr/bin/license-check \
     && chmod +x /usr/bin/license-check
-RUN license-check -path ./ --verbose=false "Edward Wilde" "OpenFaaS Project"
+RUN license-check -path ./ --verbose=false "Edward Wilde" "OpenFaaS Project" "Alex Ellis"
 RUN goimports -l -d $(find . -type f -name '*.go' -not -path "./vendor/*") \
     && VERSION=$(git describe --all --exact-match `git rev-parse HEAD` | grep tags | sed 's/tags\///') \
     && GIT_COMMIT_SHA=$(git rev-list -1 HEAD) \

README.md

@@ -37,6 +37,7 @@ All configuration is managed using environment variables
 | `read_timeout`                    | HTTP timeout for reading the payload from the client caller (in seconds).                      | `8`                      |   no     |
 | `image_pull_policy`               | Image pull policy for deployed functions (`Always`, `IfNotPresent`, `Never`)                   | `Always`                 |   no     |
 | `LOG_LEVEL`                       | Logging level either: `trace, debug, info, warn, error, fatal, panic`.                         | `info`                   |   no     |
+| `AWS_DEFAULT_REGION`              | AWS region faas-fargate is running in.                                                         | `us-east-1`              |   no     |
 
 ## Overview
 ![diagram of the openfaas on fargate architecture](./docs/architecture.png "Openfaas for fargate overview")

aws/api.go

@@ -1,6 +1,38 @@
 package aws
 
-import "github.com/aws/aws-sdk-go/service/ecs"
+import (
+	"os"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/secretsmanager"
+	"github.com/aws/aws-sdk-go/service/servicediscovery"
+)
+
+var (
+	cloudwatchClient *cloudwatchlogs.CloudWatchLogs
+	ecsClient        *ecs.ECS
+	ec2Client        *ec2.EC2
+	iamClient        *iam.IAM
+	secretsClient    *secretsmanager.SecretsManager
+	discoveryClient  *servicediscovery.ServiceDiscovery
+)
+
+func init() {
+	session := session.Must(session.NewSession())
+	logLevel := awsLogLevel()
+
+	cloudwatchClient = cloudwatchlogs.New(session, aws.NewConfig().WithLogLevel(logLevel))
+	ecsClient = ecs.New(session, aws.NewConfig().WithLogLevel(logLevel))
+	ec2Client = ec2.New(session, aws.NewConfig().WithLogLevel(logLevel))
+	iamClient = iam.New(session, aws.NewConfig().WithLogLevel(logLevel))
+	secretsClient = secretsmanager.New(session, aws.NewConfig().WithLogLevel(logLevel))
+	discoveryClient = servicediscovery.New(session, aws.NewConfig().WithLogLevel(logLevel))
+}
 
 // KeyValuePairGetValue searches the array of values and returns the matching name or nil if none are found.
 func KeyValuePairGetValue(name string, values []*ecs.KeyValuePair) (*string, bool) {
@@ -12,3 +44,13 @@ func KeyValuePairGetValue(name string, values []*ecs.KeyValuePair) (*string, boo
 
 	return nil, false
 }
+
+func awsLogLevel() aws.LogLevelType {
+	lvl := os.Getenv("LOG_LEVEL")
+
+	if lvl == "trace" {
+		return aws.LogDebugWithRequestErrors
+	}
+
+	return aws.LogOff
+}

aws/autonaming.go

@@ -14,18 +14,18 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-const namespace = "openfaas.local"
+const dnsNamespace = "openfaas.local"
 
 var once = &sync.Once{}
 var namespaceID *string
 
-func deleteServiceRegistration(discovery *servicediscovery.ServiceDiscovery, serviceName string, vpcID string) error {
-	namespaceID, err := ensureDNSNamespaceExists(discovery, vpcID)
+func deleteServiceRegistration(serviceName string, vpcID string) error {
+	namespaceID, err := ensureDNSNamespaceExists(vpcID)
 	if err != nil {
 		return fmt.Errorf("error ensuring dns namespace existing. %v", err)
 	}
 
-	listResults, err := discovery.ListServices(&servicediscovery.ListServicesInput{
+	listResults, err := discoveryClient.ListServices(&servicediscovery.ListServicesInput{
 		Filters: []*servicediscovery.ServiceFilter{
 			{
 				Name: aws.String("NAMESPACE_ID"),
@@ -49,7 +49,7 @@ func deleteServiceRegistration(discovery *servicediscovery.ServiceDiscovery, ser
 	}
 
 	log.Infof("Listing service instances for %s", serviceID)
-	instances, err := discovery.ListInstances(&servicediscovery.ListInstancesInput{
+	instances, err := discoveryClient.ListInstances(&servicediscovery.ListInstancesInput{
 		ServiceId: aws.String(serviceID),
 	})
 	if err != nil {
@@ -60,7 +60,7 @@ func deleteServiceRegistration(discovery *servicediscovery.ServiceDiscovery, ser
 	for _, v := range instances.Instances {
 		log.Infof("De-registering instance %s for service %s", aws.StringValue(v.Id), serviceID)
 
-		_, err = discovery.DeregisterInstance(&servicediscovery.DeregisterInstanceInput{
+		_, err = discoveryClient.DeregisterInstance(&servicediscovery.DeregisterInstanceInput{
 			ServiceId:  aws.String(serviceID),
 			InstanceId: v.Id,
 		})
@@ -75,7 +75,7 @@ func deleteServiceRegistration(discovery *servicediscovery.ServiceDiscovery, ser
 	eb.MaxElapsedTime = time.Second * 30
 
 	err = backoff.Retry(func() error {
-		_, err := discovery.DeleteService(&servicediscovery.DeleteServiceInput{
+		_, err := discoveryClient.DeleteService(&servicediscovery.DeleteServiceInput{
 			Id: aws.String(serviceID),
 		})
 
@@ -95,15 +95,15 @@ func deleteServiceRegistration(discovery *servicediscovery.ServiceDiscovery, ser
 	return nil
 }
 
-func ensureServiceRegistrationExists(discovery *servicediscovery.ServiceDiscovery, serviceName string, vpcID string) (string, error) {
+func ensureServiceRegistrationExists(serviceName string, vpcID string) (string, error) {
 
-	namespaceID, err := ensureDNSNamespaceExists(discovery, vpcID)
+	namespaceID, err := ensureDNSNamespaceExists(vpcID)
 	if err != nil {
 		log.Errorln("error ensuring dns namespace existing. ", err)
 		return "", err
 	}
 
-	listResults, err := discovery.ListServices(&servicediscovery.ListServicesInput{
+	listResults, err := discoveryClient.ListServices(&servicediscovery.ListServicesInput{
 		Filters: []*servicediscovery.ServiceFilter{
 			{
 				Name: aws.String("NAMESPACE_ID"),
@@ -129,7 +129,7 @@ func ensureServiceRegistrationExists(discovery *servicediscovery.ServiceDiscover
 
 	if serviceArn == "" {
 		requestID := uuid.NewV4()
-		createResult, err := discovery.CreateService(&servicediscovery.CreateServiceInput{
+		createResult, err := discoveryClient.CreateService(&servicediscovery.CreateServiceInput{
 			Name:             aws.String(serviceName),
 			CreatorRequestId: aws.String(requestID.String()),
 			Description:      aws.String(fmt.Sprintf("Openfaas auto-naming service for %s", serviceName)),
@@ -158,20 +158,20 @@ func ensureServiceRegistrationExists(discovery *servicediscovery.ServiceDiscover
 	return serviceArn, nil
 }
 
-func ensureDNSNamespaceExists(discovery *servicediscovery.ServiceDiscovery, vpcID string) (id *string, err error) {
+func ensureDNSNamespaceExists(vpcID string) (id *string, err error) {
 	once.Do(func() {
 		var found bool
 
-		id, found, err = findNamespace(discovery)
+		id, found, err = findNamespace()
 		if err != nil {
 			log.Errorln("error finding private dns name. ", err)
 			return
 		}
 
 		if !found {
 			requestID := uuid.NewV4()
-			_, err = discovery.CreatePrivateDnsNamespace(&servicediscovery.CreatePrivateDnsNamespaceInput{
-				Name:             aws.String(namespace),
+			_, err = discoveryClient.CreatePrivateDnsNamespace(&servicediscovery.CreatePrivateDnsNamespaceInput{
+				Name:             aws.String(dnsNamespace),
 				CreatorRequestId: aws.String(requestID.String()),
 				Description:      aws.String("Openfaas private DNS namespace"),
 				Vpc:              aws.String(vpcID),
@@ -182,7 +182,7 @@ func ensureDNSNamespaceExists(discovery *servicediscovery.ServiceDiscovery, vpcI
 				return
 			}
 
-			id, found, err = findNamespace(discovery)
+			id, found, err = findNamespace()
 			if err != nil {
 				log.Errorln("error finding private dns name. ", err)
 				return
@@ -200,9 +200,9 @@ func ensureDNSNamespaceExists(discovery *servicediscovery.ServiceDiscovery, vpcI
 	return namespaceID, err
 }
 
-func findNamespace(discovery *servicediscovery.ServiceDiscovery) (*string, bool, error) {
+func findNamespace() (*string, bool, error) {
 	var listResult *servicediscovery.ListNamespacesOutput
-	listResult, err := discovery.ListNamespaces(&servicediscovery.ListNamespacesInput{})
+	listResult, err := discoveryClient.ListNamespaces(&servicediscovery.ListNamespacesInput{})
 	if err != nil {
 		log.Errorln("error listing namespaces. ", err)
 		return nil, false, err
@@ -211,7 +211,7 @@ func findNamespace(discovery *servicediscovery.ServiceDiscovery) (*string, bool,
 	found := false
 	var id *string
 	for _, item := range listResult.Namespaces {
-		if aws.StringValue(item.Name) == namespace {
+		if aws.StringValue(item.Name) == dnsNamespace {
 			id = item.Id
 			found = true
 			break

aws/cloudwatch.go

@@ -0,0 +1,58 @@
+package aws
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws/awserr"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
+)
+
+func buildLogPolicyStatement(
+	builder *PolicyBuilder,
+	name string) error {
+
+	builder.AddStatement(
+		[]string{
+			"logs:CreateLogStream",
+			"logs:PutLogEvents",
+		},
+		[]string{
+			fmt.Sprintf("arn:aws:logs:*:%s:*", name),
+		})
+
+	return nil
+}
+
+func createLogGroup(functionName string) (string, error) {
+	name := ServiceNameFromFunctionName(functionName)
+	_, err := cloudwatchClient.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
+		LogGroupName: aws.String(name),
+	})
+
+	if err != nil {
+		if awsErr, ok := err.(awserr.Error); ok {
+			if awsErr.Code() == cloudwatchlogs.ErrCodeResourceAlreadyExistsException {
+				return name, nil
+			}
+		}
+
+		return "", fmt.Errorf("error creating log group for %s. %v", functionName, err)
+	}
+
+	return name, nil
+}
+
+func deleteLogGroup(functionName string) error {
+	name := ServiceNameFromFunctionName(functionName)
+	_, err := cloudwatchClient.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{
+		LogGroupName: aws.String(name),
+	})
+
+	if err != nil {
+		return fmt.Errorf("error deleting log group for %s. %v", functionName, err)
+	}
+
+	return nil
+}