Skip bitmap presence bits when pack and unpack message

field/bitmap.go

@@ -232,3 +232,25 @@ func (f *Bitmap) UnmarshalJSON(b []byte) error {
 
 	return f.SetBytes(bs)
 }
+
+// IsBitmapPresenceBit checks if the bit at position n in the bitmap is an
+// indicator of the presence of an additional bitmap. For fixed-length bitmaps
+// (when DisableAutoExpand is set in the specification), this method will
+// always return false since additional bitmaps are not applicable.
+func (f *Bitmap) IsBitmapPresenceBit(n int) bool {
+	// there are not presence bits in fixed bitmaps
+	if f.spec.DisableAutoExpand {
+		return false
+	}
+
+	if n <= 0 {
+		return false
+	}
+
+	// check if n is the first bit of a bitmap
+	if n%(f.bitmapLength*8) == 1 {
+		return true
+	}
+
+	return false
+}

field/bitmap_test.go

@@ -68,6 +68,57 @@ func TestFixedBitmap(t *testing.T) {
 	require.False(t, bitmap.IsSet(17))
 }
 
+func TestBitmap_IsBitmapPresenceBit(t *testing.T) {
+	t.Run("no presence bits for fixed bitmap (without auto expand)", func(t *testing.T) {
+		bitmap := NewBitmap(&Spec{
+			Length:            2, // 2 bytes - 16 bits
+			Description:       "Bitmap",
+			Enc:               encoding.BytesToASCIIHex,
+			Pref:              prefix.Hex.Fixed,
+			DisableAutoExpand: true,
+		})
+
+		require.False(t, bitmap.IsBitmapPresenceBit(1))
+		require.False(t, bitmap.IsBitmapPresenceBit(17))
+	})
+
+	t.Run("presence bits for custom length auto expanded bitmap", func(t *testing.T) {
+		bitmap := NewBitmap(&Spec{
+			Length:      2, // 2 bytes - 16 bits
+			Description: "Bitmap",
+			Enc:         encoding.BytesToASCIIHex,
+			Pref:        prefix.Hex.Fixed,
+		})
+
+		require.True(t, bitmap.IsBitmapPresenceBit(1))
+		require.True(t, bitmap.IsBitmapPresenceBit(17))
+		require.True(t, bitmap.IsBitmapPresenceBit(33))
+
+		require.False(t, bitmap.IsBitmapPresenceBit(2))
+		require.False(t, bitmap.IsBitmapPresenceBit(18))
+		require.False(t, bitmap.IsBitmapPresenceBit(34))
+	})
+
+	t.Run("presence bits for default length auto expanded bitmap", func(t *testing.T) {
+		bitmap := NewBitmap(&Spec{
+			Description: "Bitmap",
+			Enc:         encoding.BytesToASCIIHex,
+			Pref:        prefix.Hex.Fixed,
+		})
+
+		// default length is 8 bytes - 64 bits
+		require.True(t, bitmap.IsBitmapPresenceBit(1))
+		require.True(t, bitmap.IsBitmapPresenceBit(65))
+		require.True(t, bitmap.IsBitmapPresenceBit(129))
+		require.True(t, bitmap.IsBitmapPresenceBit(193))
+
+		require.False(t, bitmap.IsBitmapPresenceBit(2))
+		require.False(t, bitmap.IsBitmapPresenceBit(66))
+		require.False(t, bitmap.IsBitmapPresenceBit(130))
+		require.False(t, bitmap.IsBitmapPresenceBit(194))
+	})
+}
+
 func TestHexBitmap(t *testing.T) {
 	t.Run("Read only first bitmap", func(t *testing.T) {
 		bitmap := NewBitmap(&Spec{
@@ -117,6 +168,7 @@ func TestHexBitmap(t *testing.T) {
 
 		require.True(t, bitmap.IsSet(10))
 		require.True(t, bitmap.IsSet(140))
+
 	})
 
 	t.Run("When not enough data to unpack", func(t *testing.T) {

field/composite.go

@@ -395,41 +395,43 @@ func (f *Composite) pack() ([]byte, error) {
 func (f *Composite) packByBitmap() ([]byte, error) {
 	f.Bitmap().Reset()
 
-	// Set bitmap bits for all fields that are present.
-	for id := range f.setSubfields {
-		idInt, err := strconv.Atoi(id)
-		if err != nil {
-			return nil, fmt.Errorf("failed to pack composite: %w", err)
-		}
-
-		f.Bitmap().Set(idInt)
-	}
-
-	// pack bitmap.
-	packed, err := f.Bitmap().Pack()
-	if err != nil {
-		return nil, fmt.Errorf("failed to pack bitmap: %w", err)
-	}
+	var packedFields []byte
 
 	// pack fields
-	for _, i := range f.orderedSpecFieldTags {
+	for _, id := range f.orderedSpecFieldTags {
 		// If this ordered field is not set, continue to the next field.
-		if _, ok := f.setSubfields[i]; !ok {
+		if _, ok := f.setSubfields[id]; !ok {
 			continue
 		}
 
-		field, ok := f.subfields[i]
+		idInt, err := strconv.Atoi(id)
+		if err != nil {
+			return nil, fmt.Errorf("converting id %s to int: %w", id, err)
+		}
+
+		// set bitmap bit for this field
+		f.Bitmap().Set(idInt)
+
+		field, ok := f.subfields[id]
 		if !ok {
-			return nil, fmt.Errorf("failed to pack subfield %s: no specification found", i)
+			return nil, fmt.Errorf("failed to pack subfield %s: no specification found", id)
 		}
+
 		packedField, err := field.Pack()
 		if err != nil {
-			return nil, fmt.Errorf("failed to pack subfield %s (%s): %w", i, field.Spec().Description, err)
+			return nil, fmt.Errorf("failed to pack subfield %s (%s): %w", id, field.Spec().Description, err)
 		}
-		packed = append(packed, packedField...)
+
+		packedFields = append(packedFields, packedField...)
 	}
 
-	return packed, nil
+	// pack bitmap.
+	packedBitmap, err := f.Bitmap().Pack()
+	if err != nil {
+		return nil, fmt.Errorf("packing bitmap: %w", err)
+	}
+
+	return append(packedBitmap, packedFields...), nil
 }
 
 func (f *Composite) packByTag() ([]byte, error) {

message.go

@@ -149,14 +149,20 @@ func (m *Message) pack() ([]byte, error) {
 	for _, id := range ids {
 		// indexes 0 and 1 are for mti and bitmap
 		// regular field number startd from index 2
-		if id < 2 {
+		// do not pack presence bits as well
+		if id < 2 || m.Bitmap().IsBitmapPresenceBit(id) {
 			continue
 		}
 		m.Bitmap().Set(id)
 	}
 
 	// pack fields
 	for _, i := range ids {
+		// do not pack presence bits other than the first one as it's the bitmap itself
+		if i != 1 && m.Bitmap().IsBitmapPresenceBit(i) {
+			continue
+		}
+
 		field, ok := m.fields[i]
 		if !ok {
 			return nil, fmt.Errorf("failed to pack field %d: no specification found", i)
@@ -210,6 +216,11 @@ func (m *Message) unpack(src []byte) error {
 	off += read
 
 	for i := 2; i <= m.Bitmap().Len(); i++ {
+		// skip bitmap presence bits (for default bitmap length of 64 these are bits 1, 65, 129, 193, etc.)
+		if m.Bitmap().IsBitmapPresenceBit(i) {
+			continue
+		}
+
 		if m.Bitmap().IsSet(i) {
 			fl, ok := m.fields[i]
 			if !ok {

message_test.go

@@ -71,6 +71,22 @@ func TestMessage(t *testing.T) {
 				Pref:        prefix.ASCII.Fixed,
 				Pad:         padding.Left('0'),
 			}),
+
+			// this field will be ignored when packing and
+			// unpacking, as bit 65 is a bitmap presence indicator
+			65: field.NewString(&field.Spec{
+				Length:      1,
+				Description: "Settlement Code",
+				Enc:         encoding.ASCII,
+				Pref:        prefix.ASCII.Fixed,
+			}),
+			// this is a field of the third bitmap
+			130: field.NewString(&field.Spec{
+				Length:      1,
+				Description: "Additional Data",
+				Enc:         encoding.ASCII,
+				Pref:        prefix.ASCII.Fixed,
+			}),
 		},
 	}
 
@@ -89,7 +105,8 @@ func TestMessage(t *testing.T) {
 		require.Equal(t, want, string(got))
 
 		message = NewMessage(spec)
-		message.Unpack([]byte(want))
+		err = message.Unpack([]byte(want))
+		require.NoError(t, err)
 
 		s, err := message.GetMTI()
 		require.NoError(t, err)
@@ -108,6 +125,64 @@ func TestMessage(t *testing.T) {
 		require.Equal(t, "100", s)
 	})
 
+	t.Run("Do not pack fields that match the bitmap presence indicator", func(t *testing.T) {
+		message := NewMessage(spec)
+		message.MTI("0100")
+		require.NoError(t, message.Field(65, "1"))
+		require.NoError(t, message.Field(130, "1")) // field of third bitmap
+
+		got, err := message.Pack()
+
+		want := "01008000000000000000800000000000000040000000000000001"
+		require.NoError(t, err)
+		require.NotNil(t, got)
+		require.Equal(t, want, string(got))
+
+		message = NewMessage(spec)
+
+		err = message.Unpack([]byte(want))
+		require.NoError(t, err)
+
+		s, err := message.GetMTI()
+		require.NoError(t, err)
+		require.Equal(t, "0100", s)
+
+		s, err = message.GetString(65)
+		require.NoError(t, err)
+		require.Equal(t, "", s)
+
+		s, err = message.GetString(130)
+		require.NoError(t, err)
+		require.Equal(t, "1", s)
+	})
+
+	t.Run("Does not fail when packing and unpacking message with three bitmaps", func(t *testing.T) {
+		message := NewMessage(spec)
+		message.MTI("0100")
+		require.NoError(t, message.Field(130, "1")) // field of third bitmap
+
+		got, err := message.Pack()
+
+		require.NoError(t, err)
+		require.NotNil(t, got)
+
+		want := "01008000000000000000800000000000000040000000000000001"
+		require.Equal(t, want, string(got))
+
+		message = NewMessage(spec)
+		err = message.Unpack([]byte(want))
+
+		require.NoError(t, err)
+
+		s, err := message.GetMTI()
+		require.NoError(t, err)
+		require.Equal(t, "0100", s)
+
+		s, err = message.GetString(130)
+		require.NoError(t, err)
+		require.Equal(t, "1", s)
+	})
+
 	t.Run("Test unpacking with typed fields", func(t *testing.T) {
 		type TestISOF3Data struct {
 			F1 *field.String