Merge pull request #5610 from TheThingsNetwork/feature/5512-load-devi…

…ce-profile-on-import

Load device profile on CSV import

CHANGELOG.md

@@ -12,6 +12,7 @@ For details about compatibility between different releases, see the **Commitment
 ### Added
 
 - New `ListBands` RPC on the `Configuration` service.
+- Support for loading end device template from Device Repository when importing devices using a CSV file.
 
 ### Changed
 

api/api.md

@@ -3352,6 +3352,7 @@ Configuration options for static ADR.
 | ----- | ---- | ----- | ----------- |
 | `format_id` | [`string`](#string) |  | ID of the format. |
 | `data` | [`bytes`](#bytes) |  | Data to convert. |
+| `end_device_version_ids` | [`EndDeviceVersionIdentifiers`](#ttn.lorawan.v3.EndDeviceVersionIdentifiers) |  | End device profile identifiers. |
 
 #### Field Rules
 

api/api.swagger.json

@@ -20344,6 +20344,10 @@
           "type": "string",
           "format": "byte",
           "description": "Data to convert."
+        },
+        "end_device_version_ids": {
+          "$ref": "#/definitions/v3EndDeviceVersionIdentifiers",
+          "description": "End device profile identifiers."
         }
       }
     },

api/end_device.proto

@@ -900,4 +900,7 @@ message ConvertEndDeviceTemplateRequest {
   string format_id = 1 [(validate.rules).string = {pattern: "^[a-z0-9](?:[-]?[a-z0-9]){2,}$", max_len: 36}];
   // Data to convert.
   bytes data = 2;
+
+  // End device profile identifiers.
+  EndDeviceVersionIdentifiers end_device_version_ids = 3;
 }

cmd/ttn-lw-cli/commands/end_device_templates.go

@@ -39,10 +39,22 @@ func templateFormatIDFlags() *pflag.FlagSet {
 }
 
 var (
-	errNoTemplateFormatID             = errors.DefineInvalidArgument("no_template_format_id", "no template format ID set")
-	errEndDeviceMappingNotFound       = errors.DefineNotFound("mapped_end_device_not_found", "end device mapping not found")
-	errNoEndDeviceTemplateJoinEUI     = errors.DefineInvalidArgument("no_end_device_template_join_eui", "no end device template JoinEUI set")
-	errNoEndDeviceTemplateStartDevEUI = errors.DefineInvalidArgument("no_end_device_template_start_dev_eui", "no end device template start DevEUI set")
+	errNoTemplateFormatID = errors.DefineInvalidArgument(
+		"no_template_format_id",
+		"no template format ID set",
+	)
+	errEndDeviceMappingNotFound = errors.DefineNotFound(
+		"mapped_end_device_not_found",
+		"end device mapping not found",
+	)
+	errNoEndDeviceTemplateJoinEUI = errors.DefineInvalidArgument(
+		"no_end_device_template_join_eui",
+		"no end device template JoinEUI set",
+	)
+	errNoEndDeviceTemplateStartDevEUI = errors.DefineInvalidArgument(
+		"no_end_device_template_start_dev_eui",
+		"no end device template start DevEUI set",
+	)
 )
 
 func getTemplateFormatID(flagSet *pflag.FlagSet, args []string) string {
@@ -297,6 +309,7 @@ This command takes end device templates from stdin.`,
 		Short:             "Convert data to an end device template (EXPERIMENTAL)",
 		PersistentPreRunE: preRun(),
 		RunE: func(cmd *cobra.Command, args []string) error {
+			_ = optionalAuth()
 			formatID := getTemplateFormatID(cmd.Flags(), args)
 			if formatID == "" {
 				return errNoTemplateFormatID.New()
@@ -310,10 +323,16 @@ This command takes end device templates from stdin.`,
 			if err != nil {
 				return err
 			}
-			stream, err := ttnpb.NewEndDeviceTemplateConverterClient(dtc).Convert(ctx, &ttnpb.ConvertEndDeviceTemplateRequest{
-				FormatId: formatID,
-				Data:     data,
-			})
+
+			req := &ttnpb.ConvertEndDeviceTemplateRequest{
+				FormatId:            formatID,
+				Data:                data,
+				EndDeviceVersionIds: &ttnpb.EndDeviceVersionIdentifiers{},
+			}
+			if _, err = req.EndDeviceVersionIds.SetFromFlags(cmd.Flags(), "end-device-version-ids"); err != nil {
+				return err
+			}
+			stream, err := ttnpb.NewEndDeviceTemplateConverterClient(dtc).Convert(ctx, req)
 			if err != nil {
 				return err
 			}
@@ -430,7 +449,10 @@ command to assign EUIs to map to end device templates.`,
 				var res ttnpb.EndDeviceTemplate
 				res.EndDevice.SetFields(inputEntry.EndDevice, inputEntry.FieldMask.GetPaths()...)
 				res.EndDevice.SetFields(mappedEntry.EndDevice, mappedEntry.FieldMask.GetPaths()...)
-				res.FieldMask = ttnpb.FieldMask(ttnpb.BottomLevelFields(append(inputEntry.FieldMask.GetPaths(), mappedEntry.FieldMask.GetPaths()...))...)
+				res.FieldMask = ttnpb.FieldMask(
+					ttnpb.BottomLevelFields(
+						append(inputEntry.FieldMask.GetPaths(), mappedEntry.FieldMask.GetPaths()...),
+					)...)
 
 				if err := io.Write(os.Stdout, config.OutputFormat, &res); err != nil {
 					return err
@@ -455,6 +477,11 @@ func init() {
 	endDeviceTemplatesCommand.AddCommand(endDeviceTemplatesListFormats)
 	endDeviceTemplatesFromDataCommand.Flags().AddFlagSet(templateFormatIDFlags())
 	endDeviceTemplatesFromDataCommand.Flags().AddFlagSet(dataFlags("", ""))
+	ttnpb.AddSetFlagsForEndDeviceVersionIdentifiers(
+		endDeviceTemplatesFromDataCommand.Flags(),
+		"end-device-version-ids",
+		false,
+	)
 	endDeviceTemplatesCommand.AddCommand(endDeviceTemplatesFromDataCommand)
 	endDeviceTemplatesMapCommand.Flags().AddFlagSet(dataFlags("input", "input file"))
 	endDeviceTemplatesMapCommand.Flags().AddFlagSet(dataFlags("mapping", "mapping file"))

pkg/devicerepository/mock/mock.go

@@ -0,0 +1,114 @@
+// Copyright © 2022 The Things Network Foundation, The Things Industries B.V.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package mockdr contains the mock of a Device Repository Server.
+package mockdr
+
+import (
+	"context"
+
+	"go.thethings.network/lorawan-stack/v3/pkg/ttnpb"
+	"google.golang.org/grpc"
+)
+
+// New returns a mock of the Device Repository client.
+func New() *MockDR {
+	return &MockDR{}
+}
+
+// MockDR contains the response that it will provide when a client communicates with the server address.
+type MockDR struct {
+	Err                     error
+	ListEndDeviceBrandsResp *ttnpb.ListEndDeviceBrandsResponse
+	EndDeviceBrand          *ttnpb.EndDeviceBrand
+	ListEndDeviceModelsResp *ttnpb.ListEndDeviceModelsResponse
+	EndDeviceModel          *ttnpb.EndDeviceModel
+	EndDeviceTemplate       *ttnpb.EndDeviceTemplate
+	MessagePayloadDecoder   *ttnpb.MessagePayloadDecoder
+	MessagePayloadEncoder   *ttnpb.MessagePayloadEncoder
+}
+
+/// Methods of DR
+
+// ListBrands mock method.
+func (mdr *MockDR) ListBrands(
+	context.Context,
+	*ttnpb.ListEndDeviceBrandsRequest,
+	...grpc.CallOption,
+) (*ttnpb.ListEndDeviceBrandsResponse, error) {
+	return mdr.ListEndDeviceBrandsResp, mdr.Err
+}
+
+// GetBrand mock method.
+func (mdr *MockDR) GetBrand(
+	context.Context,
+	*ttnpb.GetEndDeviceBrandRequest,
+	...grpc.CallOption,
+) (*ttnpb.EndDeviceBrand, error) {
+	return mdr.EndDeviceBrand, mdr.Err
+}
+
+// ListModels mock method.
+func (mdr *MockDR) ListModels(
+	context.Context,
+	*ttnpb.ListEndDeviceModelsRequest,
+	...grpc.CallOption,
+) (*ttnpb.ListEndDeviceModelsResponse, error) {
+	return mdr.ListEndDeviceModelsResp, mdr.Err
+}
+
+// GetModel mock method.
+func (mdr *MockDR) GetModel(
+	context.Context,
+	*ttnpb.GetEndDeviceModelRequest,
+	...grpc.CallOption,
+) (*ttnpb.EndDeviceModel, error) {
+	return mdr.EndDeviceModel, mdr.Err
+}
+
+// GetTemplate mock method.
+func (mdr *MockDR) GetTemplate(
+	context.Context,
+	*ttnpb.GetTemplateRequest,
+	...grpc.CallOption,
+) (*ttnpb.EndDeviceTemplate, error) {
+	return mdr.EndDeviceTemplate, mdr.Err
+}
+
+// GetUplinkDecoder mock method.
+func (mdr *MockDR) GetUplinkDecoder(
+	context.Context,
+	*ttnpb.GetPayloadFormatterRequest,
+	...grpc.CallOption,
+) (*ttnpb.MessagePayloadDecoder, error) {
+	return mdr.MessagePayloadDecoder, mdr.Err
+}
+
+// GetDownlinkDecoder mock method.
+func (mdr *MockDR) GetDownlinkDecoder(
+	context.Context,
+	*ttnpb.GetPayloadFormatterRequest,
+	...grpc.CallOption,
+) (*ttnpb.MessagePayloadDecoder, error) {
+	return mdr.MessagePayloadDecoder, mdr.Err
+}
+
+// GetDownlinkEncoder mock method.
+func (mdr *MockDR) GetDownlinkEncoder(
+	context.Context,
+	*ttnpb.GetPayloadFormatterRequest,
+	...grpc.CallOption,
+) (*ttnpb.MessagePayloadEncoder, error) {
+	return mdr.MessagePayloadEncoder, mdr.Err
+}

pkg/devicetemplateconverter/devicetemplateconverter.go

@@ -50,7 +50,6 @@ func New(c *component.Component, conf *Config) (*DeviceTemplateConverter, error)
 		devicetemplates.TTSJSON,
 		devicetemplates.TTSCSV,
 	)
-
 	converters := make(map[string]devicetemplates.Converter, len(conf.Enabled))
 	for _, id := range conf.Enabled {
 		converter := devicetemplates.GetConverter(id)
@@ -77,7 +76,7 @@ func (dtc *DeviceTemplateConverter) Context() context.Context {
 }
 
 // Roles returns the roles that the Device Template Converter fulfills.
-func (dtc *DeviceTemplateConverter) Roles() []ttnpb.ClusterRole {
+func (*DeviceTemplateConverter) Roles() []ttnpb.ClusterRole {
 	return []ttnpb.ClusterRole{ttnpb.ClusterRole_DEVICE_TEMPLATE_CONVERTER}
 }
 
@@ -88,5 +87,5 @@ func (dtc *DeviceTemplateConverter) RegisterServices(s *grpc.Server) {
 
 // RegisterHandlers registers gRPC handlers.
 func (dtc *DeviceTemplateConverter) RegisterHandlers(s *runtime.ServeMux, conn *grpc.ClientConn) {
-	ttnpb.RegisterEndDeviceTemplateConverterHandler(dtc.Context(), s, conn)
+	ttnpb.RegisterEndDeviceTemplateConverterHandler(dtc.Context(), s, conn) //nolint:errcheck
 }

pkg/devicetemplateconverter/devicetemplateconverter_test.go

@@ -26,6 +26,7 @@ import (
 )
 
 func TestDeviceTemplateConverter(t *testing.T) {
+	t.Parallel()
 	ctx := log.NewContext(test.Context(), test.GetLogger(t))
 
 	conf := &component.Config{}

pkg/devicetemplateconverter/grpc.go

@@ -19,6 +19,8 @@ import (
 	"context"
 
 	pbtypes "github.com/gogo/protobuf/types"
+	"go.thethings.network/lorawan-stack/v3/pkg/devicetemplateconverter/profilefetcher"
+	"go.thethings.network/lorawan-stack/v3/pkg/devicetemplates"
 	"go.thethings.network/lorawan-stack/v3/pkg/errorcontext"
 	"go.thethings.network/lorawan-stack/v3/pkg/ttnpb"
 )
@@ -28,7 +30,10 @@ type endDeviceTemplateConverterServer struct {
 }
 
 // ListFormats implements ttnpb.DeviceTemplateServiceServer.
-func (s *endDeviceTemplateConverterServer) ListFormats(ctx context.Context, _ *pbtypes.Empty) (*ttnpb.EndDeviceTemplateFormats, error) {
+func (s *endDeviceTemplateConverterServer) ListFormats(
+	context.Context,
+	*pbtypes.Empty,
+) (*ttnpb.EndDeviceTemplateFormats, error) {
 	formats := make(map[string]*ttnpb.EndDeviceTemplateFormat, len(s.DTC.converters))
 	for id, converter := range s.DTC.converters {
 		formats[id] = converter.Format()
@@ -39,25 +44,31 @@ func (s *endDeviceTemplateConverterServer) ListFormats(ctx context.Context, _ *p
 }
 
 // Convert implements ttnpb.DeviceTemplateServiceServer.
-func (s *endDeviceTemplateConverterServer) Convert(req *ttnpb.ConvertEndDeviceTemplateRequest, res ttnpb.EndDeviceTemplateConverter_ConvertServer) error {
+func (s *endDeviceTemplateConverterServer) Convert(
+	req *ttnpb.ConvertEndDeviceTemplateRequest,
+	res ttnpb.EndDeviceTemplateConverter_ConvertServer,
+) error {
 	converter, ok := s.DTC.converters[req.FormatId]
 	if !ok {
 		return errNotFound.WithAttributes("id", req.FormatId)
 	}
 	ctx, cancel := errorcontext.New(res.Context())
+	ctx = devicetemplates.NewContextWithProfileIDs(ctx, req.GetEndDeviceVersionIds())
+	ctx = profilefetcher.NewContextWithFetcher(ctx, profilefetcher.NewTemplateFetcher(s.DTC.Component))
 	ch := make(chan *ttnpb.EndDeviceTemplate)
 	go func() {
 		if err := converter.Convert(ctx, bytes.NewReader(req.Data), ch); err != nil {
 			cancel(err)
 		}
 	}()
+
 	for {
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
 		case tmpl, ok := <-ch:
 			if !ok {
-				return nil
+				return ctx.Err()
 			}
 			if err := res.Send(tmpl); err != nil {
 				return err

pkg/devicetemplateconverter/grpc_test.go

@@ -17,6 +17,7 @@ package devicetemplateconverter_test
 import (
 	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"testing"
@@ -33,6 +34,7 @@ import (
 )
 
 func TestConvertEndDeviceTemplate(t *testing.T) {
+	t.Parallel()
 	a := assertions.New(t)
 	ctx := log.NewContext(test.Context(), test.GetLogger(t))
 
@@ -94,7 +96,7 @@ func TestConvertEndDeviceTemplate(t *testing.T) {
 	tmpls := make([]*ttnpb.EndDeviceTemplate, 0, 2)
 	for {
 		tmpl, err := stream.Recv()
-		if err == io.EOF {
+		if errors.Is(err, io.EOF) {
 			break
 		}
 		tmpls = append(tmpls, tmpl)