diff --git a/internal/search/api_search.go b/internal/search/api_search.go new file mode 100644 index 00000000..0bf34117 --- /dev/null +++ b/internal/search/api_search.go @@ -0,0 +1,41 @@ +package search + +import ( + "net/http" + + "github.com/moov-io/base/log" + + "github.com/gorilla/mux" +) + +// GET /v2/search + +type Controller interface { + AppendRoutes(router *mux.Router) *mux.Router +} + +func NewController(logger log.Logger, service Service) Controller { + return &controller{ + logger: logger, + service: service, + } +} + +type controller struct { + logger log.Logger + service Service +} + +func (c *controller) AppendRoutes(router *mux.Router) *mux.Router { + router. + Name("Search.v2"). + Methods("GET"). + Path("/v2/search"). + HandlerFunc(c.search) + + return router +} + +func (c *controller) search(w http.ResponseWriter, r *http.Request) { + // TODO(adam): +} diff --git a/pkg/csl_eu/mapper.go b/pkg/csl_eu/mapper.go new file mode 100644 index 00000000..6688b778 --- /dev/null +++ b/pkg/csl_eu/mapper.go @@ -0,0 +1,42 @@ +package csl + +import ( + "strings" + "time" + + "github.com/moov-io/watchman/pkg/csl" + "github.com/moov-io/watchman/pkg/search" +) + +func PtrToEntity(record *csl.EUCSLRecord) search.Entity[csl.EUCSLRecord] { + if record != nil { + return ToEntity(*record) + } + return search.Entity[csl.EUCSLRecord]{} +} + +func ToEntity(record csl.EUCSLRecord) search.Entity[csl.EUCSLRecord] { + out := search.Entity[csl.EUCSLRecord]{ + Source: search.SourceEUCSL, + SourceData: record, + } + + if strings.EqualFold(record.EntitySubjectType, "person") { + out.Type = search.EntityPerson + out.Person = &search.Person{} + + if len(record.NameAliasWholeNames) > 0 { + out.Name = record.NameAliasWholeNames[0] + out.Person.Name = record.NameAliasWholeNames[0] + out.Person.AltNames = record.NameAliasWholeNames[1:] + } + if len(record.BirthDates) > 0 { + tt, err := time.Parse("2006-01-02", record.BirthDates[0]) + if err == nil { + out.Person.BirthDate = &tt + } + } + } + + return out +} diff --git a/pkg/csl_us/mapper.go b/pkg/csl_us/mapper.go new file mode 100644 index 00000000..da3768ef --- /dev/null +++ b/pkg/csl_us/mapper.go @@ -0,0 +1,186 @@ +package mapper + +import ( + "github.com/moov-io/watchman/pkg/csl" + "github.com/moov-io/watchman/pkg/search" +) + +// Entity List – Bureau of Industry and Security +func EL_ToEntity(record csl.EL) search.Entity[csl.EL] { + out := search.Entity[csl.EL]{ + Source: search.SourceUSCSL, + SourceData: record, + } + + out.Name = record.Name + // out.Type = // TODO(adam): + + // record.AlternateNames []string // TODO(adam): + // record.Addresses []string // TODO(adam): + + return out +} + +// Military End User List +func MEU_ToEntity(record csl.MEU) search.Entity[csl.MEU] { + out := search.Entity[csl.MEU]{ + Source: search.SourceUSCSL, + SourceData: record, + } + + out.Name = record.Name + + // Name string `json:"name"` + // Addresses string `json:"addresses"` + + return out +} + +// Sectoral Sanctions Identifications List (SSI) - Treasury Department +func SSI_ToEntity(record csl.SSI) search.Entity[csl.SSI] { + out := search.Entity[csl.SSI]{ + Source: search.SourceUSCSL, + SourceData: record, + } + + out.Name = record.Name + + // Addresses []string `json:"addresses"` + // Remarks []string `json:"remarks"` + // AlternateNames []string `json:"alternateNames"` + + // IDsOnRecord []string `json:"ids"` + + return out +} + +// Unverified List – Bureau of Industry and Security +func UVL_ToEntity(record csl.UVL) search.Entity[csl.UVL] { + out := search.Entity[csl.UVL]{ + Source: search.SourceUSCSL, + SourceData: record, + } + + out.Name = record.Name + // Addresses []string `json:"addresses"` + + return out +} + +// Foreign Sanctions Evaders (FSE) - Treasury Department +func FSE_ToEntity(record csl.FSE) search.Entity[csl.FSE] { + out := search.Entity[csl.FSE]{ + Source: search.SourceUSCSL, + SourceData: record, + } + + out.Name = record.Name + + // Type string `json:"type"` + // Addresses []string `json:"addresses,omitempty"` + // DatesOfBirth string `json:"datesOfBirth"` + // IDs []string `json:"IDs"` + + return out +} + +// Nonproliferation Sanctions (ISN) - State Department +func ISN_ToEntity(record csl.ISN) search.Entity[csl.ISN] { + out := search.Entity[csl.ISN]{ + Source: search.SourceUSCSL, + SourceData: record, + } + + out.Name = record.Name + + // Remarks []string `json:"remarks,omitempty"` + // AlternateNames []string `json:"alternateNames,omitempty"` + + return out +} + +// Palestinian Legislative Council List (PLC) - Treasury Department +func PLC_ToEntity(record csl.PLC) search.Entity[csl.PLC] { + out := search.Entity[csl.PLC]{ + Source: search.SourceUSCSL, + SourceData: record, + } + + out.Name = record.Name + + // Type string `json:"type"` + // Addresses []string `json:"addresses,omitempty"` + // DatesOfBirth string `json:"datesOfBirth"` + // IDs []string `json:"IDs"` + // Remarks []string `json:"remarks,omitempty"` + + return out +} + +// CAPTA (formerly Foreign Financial Institutions Subject to Part 561 - Treasury Department) +func CAP_ToEntity(record csl.CAP) search.Entity[csl.CAP] { + out := search.Entity[csl.CAP]{ + Source: search.SourceUSCSL, + SourceData: record, + } + + out.Name = record.Name + + // Type string `json:"type"` + // Addresses []string `json:"addresses,omitempty"` + // DatesOfBirth string `json:"datesOfBirth"` + // IDs []string `json:"IDs"` + // Remarks []string `json:"remarks,omitempty"` + + return out +} + +// ITAR Debarred (DTC) - State Department +func DTC_ToEntity(record csl.DTC) search.Entity[csl.DTC] { + out := search.Entity[csl.DTC]{ + Source: search.SourceUSCSL, + SourceData: record, + } + + out.Name = record.Name + + // AlternateNames []string `json:"alternateNames,omitempty"` + + return out +} + +// Non-SDN Chinese Military-Industrial Complex Companies List (CMIC) - Treasury Department +func CMIC_ToEntity(record csl.CMIC) search.Entity[csl.CMIC] { + out := search.Entity[csl.CMIC]{ + Source: search.SourceUSCSL, + SourceData: record, + } + + out.Name = record.Name + + // Type string `json:"type"` + // Addresses []string `json:"addresses,omitempty"` + // DatesOfBirth string `json:"datesOfBirth"` + // IDs []string `json:"IDs"` + // Remarks []string `json:"remarks,omitempty"` + + return out +} + +// Non-SDN Menu-Based Sanctions List (NS-MBS List) - Treasury Department +func NS_MBS_ToEntity(record csl.NS_MBS) search.Entity[csl.NS_MBS] { + out := search.Entity[csl.NS_MBS]{ + Source: search.SourceUSCSL, + SourceData: record, + } + + out.Name = record.Name + + // Type string `json:"type"` + // Addresses []string `json:"addresses,omitempty"` + // DatesOfBirth string `json:"datesOfBirth"` + // IDs []string `json:"IDs"` + // Remarks []string `json:"remarks,omitempty"` + + return out +} diff --git a/pkg/ofac/mapper.go b/pkg/ofac/mapper.go new file mode 100644 index 00000000..01e4014b --- /dev/null +++ b/pkg/ofac/mapper.go @@ -0,0 +1,304 @@ +package ofac + +import ( + "regexp" + "strings" + "time" + + "github.com/moov-io/watchman/pkg/search" +) + +func PtrToEntity(sdn *SDN) search.Entity[SDN] { + if sdn != nil { + return ToEntity(*sdn) + } + return search.Entity[SDN]{} +} + +// TODO(adam): Accept Addresses, Alts, Comments + +func ToEntity(sdn SDN) search.Entity[SDN] { + out := search.Entity[SDN]{ + Name: sdn.SDNName, + Source: search.SourceUSOFAC, + SourceData: sdn, + } + + remarks := splitRemarks(sdn.Remarks) + + switch strings.ToLower(strings.TrimSpace(sdn.SDNType)) { + case "-0-", "": + out.Type = search.EntityBusiness + // Set properties + out.Business = &search.Business{ + Name: sdn.SDNName, + } + out.Business.Identifier = makeIdentifiers(remarks, []string{ + "Branch Unit Number", + "Business Number", + "Business Registration Document", + "Business Registration Number", + "Certificate of Incorporation Number", + "Chamber of Commerce Number", + "Chinese Commercial Code", + "Registered Charity No.", + }) + + case "individual": + out.Type = search.EntityPerson + out.Person = &search.Person{ + Name: sdn.SDNName, + Gender: search.Gender(strings.ToLower(firstValue(findMatchingRemarks(remarks, "Gender")))), + } + out.Person.BirthDate = withFirstP(findMatchingRemarks(remarks, "DOB"), func(in remark) *time.Time { + t, err := parseTime(dobPatterns, in.value) + if t.IsZero() || err != nil { + return nil + } + return &t + }) + + // TODO(adam): + // citizen Venezuela + // + // nationality Russia; + // nationality: Eritrean + // + // POB 'Adlun, Lebanon + // Alt. POB: Keren Eritrea + // POB Abadan, Iran + + case "vessel": + out.Type = search.EntityVessel + out.Vessel = &search.Vessel{ + Name: sdn.SDNName, + IMONumber: firstValue(findMatchingRemarks(remarks, "IMO")), + Type: withFirstF(findMatchingRemarks(remarks, "Vessel Type"), func(r remark) search.VesselType { + return search.VesselType(r.value) // TODO(adam): OFAC values are not an enum + }), + Flag: firstValue(findMatchingRemarks(remarks, "Flag")), // TODO(adam): ISO-3166 + // Built *time.Time `json:"built"` + // Model string `json:"model"` + // Tonnage int `json:"tonnage"` // TODO(adam): remove , and ParseInt + MMSI: firstValue(findMatchingRemarks(remarks, "MMSI")), + } + + case "aircraft": + out.Type = search.EntityAircraft + out.Aircraft = &search.Aircraft{ + Name: sdn.SDNName, + // Type AircraftType `json:"type"` + Flag: firstValue(findMatchingRemarks(remarks, "Flag")), // TODO(adam): ISO-3166 + Built: withFirstP(findMatchingRemarks(remarks, "Manufacture Date"), func(in remark) *time.Time { + t, err := parseTime(dobPatterns, in.value) + if t.IsZero() || err != nil { + return nil + } + return &t + }), + // ICAOCode string `json:"icaoCode"` // ICAO aircraft type designator + Model: firstValue(findMatchingRemarks(remarks, "Aircraft Model")), + SerialNumber: withFirstF(findMatchingRemarks(remarks, "Serial Number"), func(r remark) string { + // Trim parens from these remarks + // e.g. "Aircraft Manufacturer's Serial Number (MSN) 1023409321;" + idx := strings.Index(r.value, ")") + if idx > -1 && len(r.value) > idx+1 { + r.value = strings.TrimSpace(r.value[idx+1:]) + } + return r.value + }), + } + + // TODO(adam): + // Aircraft Operator YAS AIR; + // Previous Aircraft Tail Number 2-WGLP + } + + return out +} + +var parenCountryRegex = regexp.MustCompile(`\(([\w\s]+)\)`) + +func makeIdentifier(remarks []string, suffix string) *search.Identifier { + found := findMatchingRemarks(remarks, suffix) + if len(found) == 0 { + return nil + } + + // Often the country is in parenthesis at the end, so let's look for that + // + // Business Number 51566843 (Hong Kong) + country := parenCountryRegex.FindString(found[0].value) + country = strings.TrimPrefix(strings.TrimSuffix(country, ")"), "(") + + return &search.Identifier{ + Name: found[0].fullName, + Country: country, // ISO-3166 // TODO(adam): + Identifier: found[0].value, + } +} + +func makeIdentifiers(remarks []string, needles []string) []search.Identifier { + var out []search.Identifier + for i := range needles { + if id := makeIdentifier(remarks, needles[i]); id != nil { + out = append(out, *id) + } + } + return out +} + +var ( + dobPatterns = []string{ + "02 Jan 2006", // 01 Apr 1950 + "Jan 2006", // Sep 1958 + "2006", // 1928 + } +) + +func parseTime(acceptedLayouts []string, value string) (time.Time, error) { + // We don't currently support ranges for birth dates, so take the first date provided + // Examples include: + // 01 Feb 1958 to 28 Feb 1958 + // circa 1934 + // circa 1979-1982 + value = strings.TrimSpace(strings.ReplaceAll(value, "circa", "")) + + parts := strings.Split(value, "to") + if len(parts) > 1 { + value = parts[0] + } else { + parts = strings.Split(value, "-") + if len(parts) > 1 { + value = parts[0] + } + } + value = strings.TrimSpace(value) + + for i := range acceptedLayouts { + tt, err := time.Parse(acceptedLayouts[i], value) + if !tt.IsZero() && err == nil { + return tt, nil + } + } + return time.Time{}, nil +} + +// TODO(adam): +// Drop "alt. " + +// ContactInfo +// Fax: 0097282858208. +// Fax No. (022) 7363196. +// Fax (356)(25990640) +// FAX 850 2 381 4431/4432 +// Alt. Fax: 9221227700019 +// +// Alt. Telephone: 982188554432 +// Telephone (356)(21241232) +// Telephone + 97165749996 +// Telephone +31 010-4951863 +// Telephone No. (022) 7363030 +// Telephone Number: (971) (4) (3248000). +// PHONE 850 2 18111 8204/8208 +// PHONE 850 2 18111 ext. 8221 +// Phone No. 263-4-486946 +// Phone Number 982188526300 +// +// EMAIL daesong@co.chesin.com. +// Email Address EnExchanger@gmail.com +// Email:adelb@shabakah.net.sa. +// info@sanabel.org.uk (email). +// +// Website Oboronlogistika.ru +// Website http://comitet.su/about/ +// http://www.saraproperties.co.uk (website). + +// a.k.a. 'ABU AHMAD ISHAB'. +// a.k.a. 'ZAMANI, Aziz Shah' + +// GovernmentIDs +// +// Cedula No. 94428531 (Colombia) +// Cedula No. 94487319 (Colombia) issued 31 Oct 1994 +// Birth Certificate Number 32270 (Iran) +// Bosnian Personal ID No. 1005967953038 +// British National Overseas Passport 750200421 (United Kingdom) +// C.I.F. B84758374 (Spain). +// C.R. No. 03-B-1620 +// C.R. No. J10/623/1997 (Romania) +// C.U.I.P. AOIR671020H1374898 (Mexico). +// C.U.I.T. 20-60357110-0 (Argentina) +// C.U.R.P. # HESU430525HBCRMR13 (Mexico) +// CNP (Personal Numerical Code) 7460301380011 (Romania) +// Cartilla de Servicio Militar Nacional 607092 (Mexico). +// Citizen's Card Number 210222198011096648 (China). +// Commercial Registry Number 0411518776478 (Iran) +// Commercial Registry Number CH-020.1.066.499-9 (Switzerland) +// Company ID: No. 59 531 at Commercial Registry of the Civil Court of First Instance at Baabda, Lebanon. +// Company Number 05527424 (United Kingdom) +// Company Number IMO 1991835. +// Credencial electoral 073855815496 (Mexico). +// D-U-N-S Number 33-843-5672 +// D.N.I. 00263695-T (Spain) +// Diplomatic Passport 00000017 (Yemen) issued 27 Oct 2008 expires 26 Oct 2014 +// Driver's License No. 04900377 (Moldova) issued 02 Jul 2004 +// Driver's License No. 07442833 (United States) expires 15 Mar 2016 +// Driver's License No. 1-1-22-07-00030905-3 (Guatemala) expires 2010. +// Driver's License No. M600161650080 (United States) issued 07 Apr 2006 expires 08 Jan 2011. +// Driver's License is issued by the State of Texas. +// Dubai Chamber of Commerce Membership No. 123076 (United Arab Emirates). +// Electoral Registry No. 07385114 (Afghanistan). +// Enterprise Number 0430.033.662 (Belgium). +// Fiscal Code 9896460 (Romania). +// Folio Mercantil No. 10328 (Jalisco) (Mexico). +// Government Gazette Number 00132598 (Russia). +// I.F.E. 05116040222575 (Mexico). +// Identification Number 0-16 Reg 53089 (Guatemala) +// Immigration No. A38839964 (United States). +// Interpol: Red Notice. File No. 2009/3599. March 24, 2009. Orange Notice. File No. 2009/52/OS/CCC. February 10, 2009. +// Italian Fiscal Code BCHMHT69R13Z352T. +// Kenyan ID No. 12773667 +// LE Number 07541863 (Peru). +// Legal Entity Number 851683897 (Netherlands) +// License 1249 (Russia). +// Matricula Mercantil No 0000104026 (Colombia). +// N.I.E. X-1552120-B (Spain). +// NIT # 16215230-1 (Colombia). +// National Foreign ID Number 210602197107153012 (China) +// National ID No. (HWI)040182 (Burma) +// Passport #H0044232 (Iraq). +// Passport 00016161 (Yemen) issued 19 Jun 2012 expires 18 Jun 2018 +// Passport No.: 0310857, Eritrea, Issue Date 21 August 2006, Expire Date 20 August 2008) +// Personal ID Card 00246412491303975500493 (Slovenia) expires 17 Dec 2018 +// Pilot License Number 2326384 +// Public Registration Number 1021801434380. +// Public Registration Number 1041202 (Virgin Islands, British) +// R.F.C. # IES-870805 (Mexico). +// RFC AAIJ810808SX4 (Mexico) +// RIF # J-00317392-4 (Venezuela). +// RTN 01019995013319 (Honduras) +// RUC # 1008619-1-537654 (Panama). +// Refugee ID Card A88000043 (Moldova) issued 16 Dec 2005. +// Registration ID 0000421465 (Poland) +// Registration Number 1027700499903 (Russia) +// Residency Number 003-5506420-0100028 (Costa Rica). +// Romanian C.R. J23/242/2004 (Romania). +// Romanian Permanent Resident CAN 0125477 (Romania) issued 13 Jul 2007. +// Romanian Tax Registration 14637977 (Romania). +// SSN 156-92-9858 (United States) +// SSN 33-3208848-3 (Philippines) +// SWIFT/BIC AFABAFKA +// Stateless Person ID Card CC00200261 (Moldova) issued 09 Sep 2000 +// Stateless Person Passport C000375 (Moldova) issued 09 Sep 2000 +// Tax ID No. 002235933 (Canada). +// Trade License No. 04110179 (United Kingdom). +// Travel Document Number A0003900 (Germany) +// Turkish Identificiation Number 10298480866 (Turkey). +// U.S.A. Passport issued 21 Jun 1992 in Amman, Jordan. +// UAE Identification 784-1968-9720837-5 +// UK Company Number 01019769 (United Kingdom) +// US FEIN 000920912 (United States). +// United Social Credit Code Certificate (USCCC) 91420112711981060J (China) +// V.A.T. Number 0430.033.662 (Belgium) +// VisaNumberID 2024702 (Mexico). diff --git a/pkg/ofac/mapper_test.go b/pkg/ofac/mapper_test.go new file mode 100644 index 00000000..6e23d02c --- /dev/null +++ b/pkg/ofac/mapper_test.go @@ -0,0 +1,133 @@ +package ofac + +import ( + "path/filepath" + "testing" + "time" + + "github.com/moov-io/watchman/pkg/search" + + "github.com/stretchr/testify/require" +) + +func TestMapper__Person(t *testing.T) { + res, err := Read(filepath.Join("..", "..", "test", "testdata", "sdn.csv")) + require.NoError(t, err) + + var sdn *SDN + for i := range res.SDNs { + if res.SDNs[i].EntityID == "15102" { + sdn = res.SDNs[i] + } + } + require.NotNil(t, sdn) + + e := ToEntity(*sdn) + require.Equal(t, "MORENO, Daniel", e.Name) + require.Equal(t, search.EntityPerson, e.Type) + require.Equal(t, search.SourceUSOFAC, e.Source) + + require.NotNil(t, e.Person) + require.Equal(t, "MORENO, Daniel", e.Person.Name) + require.Equal(t, "", string(e.Person.Gender)) + require.Equal(t, "1972-10-12T00:00:00Z", e.Person.BirthDate.Format(time.RFC3339)) + require.Nil(t, e.Person.DeathDate) + require.Len(t, e.Person.GovernmentIDs, 0) + + require.Nil(t, e.Business) + require.Nil(t, e.Organization) + require.Nil(t, e.Aircraft) + require.Nil(t, e.Vessel) + + require.Equal(t, "15102", e.SourceData.EntityID) +} + +func TestMapper__Vessel(t *testing.T) { + res, err := Read(filepath.Join("..", "..", "test", "testdata", "sdn.csv")) + require.NoError(t, err) + + var sdn *SDN + for i := range res.SDNs { + if res.SDNs[i].EntityID == "15036" { + sdn = res.SDNs[i] + } + } + require.NotNil(t, sdn) + + e := ToEntity(*sdn) + require.Equal(t, "ARTAVIL", e.Name) + require.Equal(t, search.EntityVessel, e.Type) + require.Equal(t, search.SourceUSOFAC, e.Source) + + require.Nil(t, e.Person) + require.Nil(t, e.Business) + require.Nil(t, e.Organization) + require.Nil(t, e.Aircraft) + require.NotNil(t, e.Vessel) + + require.Equal(t, "ARTAVIL", e.Vessel.Name) + require.Equal(t, "Malta", e.Vessel.Flag) + require.Equal(t, "9187629", e.Vessel.IMONumber) + require.Equal(t, "572469210", e.Vessel.MMSI) + + require.Equal(t, "15036", e.SourceData.EntityID) +} + +func TestMapper__Aircraft(t *testing.T) { + res, err := Read(filepath.Join("..", "..", "test", "testdata", "sdn.csv")) + require.NoError(t, err) + + var sdn *SDN + for i := range res.SDNs { + if res.SDNs[i].EntityID == "18158" { + sdn = res.SDNs[i] + } + } + require.NotNil(t, sdn) + + e := ToEntity(*sdn) + require.Equal(t, "MSN 550", e.Name) + require.Equal(t, search.EntityAircraft, e.Type) + require.Equal(t, search.SourceUSOFAC, e.Source) + + require.Nil(t, e.Person) + require.Nil(t, e.Business) + require.Nil(t, e.Organization) + require.NotNil(t, e.Aircraft) + require.Nil(t, e.Vessel) + + require.Equal(t, "MSN 550", e.Aircraft.Name) + require.Equal(t, "1995-01-01", e.Aircraft.Built.Format(time.DateOnly)) + require.Equal(t, "Airbus A321-131", e.Aircraft.Model) + require.Equal(t, "550", e.Aircraft.SerialNumber) + + require.Equal(t, "18158", e.SourceData.EntityID) +} + +func TestParseTime(t *testing.T) { + t.Run("DOB", func(t *testing.T) { + tt, _ := parseTime(dobPatterns, "01 Apr 1950") + require.Equal(t, "1950-04-01", tt.Format(time.DateOnly)) + + tt, _ = parseTime(dobPatterns, "01 Feb 1958 to 28 Feb 1958") + require.Equal(t, "1958-02-01", tt.Format(time.DateOnly)) + + tt, _ = parseTime(dobPatterns, "1928") + require.Equal(t, "1928-01-01", tt.Format(time.DateOnly)) + + tt, _ = parseTime(dobPatterns, "1928 to 1930") + require.Equal(t, "1928-01-01", tt.Format(time.DateOnly)) + + tt, _ = parseTime(dobPatterns, "Sep 1958") + require.Equal(t, "1958-09-01", tt.Format(time.DateOnly)) + + tt, _ = parseTime(dobPatterns, "circa 01 Jan 1961") + require.Equal(t, "1961-01-01", tt.Format(time.DateOnly)) + + tt, _ = parseTime(dobPatterns, "circa 1934") + require.Equal(t, "1934-01-01", tt.Format(time.DateOnly)) + + tt, _ = parseTime(dobPatterns, "circa 1979-1982") + require.Equal(t, "1979-01-01", tt.Format(time.DateOnly)) + }) +} diff --git a/pkg/ofac/reader.go b/pkg/ofac/reader.go index 1c9fc3e1..737de1f6 100644 --- a/pkg/ofac/reader.go +++ b/pkg/ofac/reader.go @@ -252,6 +252,73 @@ func splitPrograms(in string) []string { return strings.Split(norm, "; ") } +func splitRemarks(input string) []string { + return strings.Split(input, ";") +} + +type remark struct { + matchedName string + fullName string + value string +} + +func findMatchingRemarks(remarks []string, suffix string) []remark { + var out []remark + if suffix == "" { + return out + } + for i := range remarks { + idx := strings.Index(remarks[i], suffix) + if idx == -1 { + continue // not found + } + + value := remarks[i][idx+len(suffix):] + value = strings.TrimPrefix(value, ":") // identifiers can end with a colon + value = strings.TrimSuffix(value, ";") + value = strings.TrimSuffix(value, ".") + + out = append(out, remark{ + matchedName: suffix, + fullName: remarks[i][:idx+len(suffix)], + value: strings.TrimSpace(value), + }) + } + return out +} + +func findRemarkValues(remarks []string, suffix string) []string { + found := findMatchingRemarks(remarks, suffix) + var out []string + for i := range found { + out = append(out, found[i].value) + } + return out +} + +func firstValue(values []remark) string { + if len(values) == 0 { + return "" + } + return values[0].value +} + +func withFirstF[T any](values []remark, f func(remark) T) T { + if len(values) == 0 { + var zero T + return zero + } + return f(values[0]) +} + +func withFirstP[T any](values []remark, f func(remark) *T) *T { + if len(values) == 0 { + var zero T + return &zero + } + return f(values[0]) +} + var ( digitalCurrencies = []string{ "XBT", // Bitcoin @@ -281,7 +348,7 @@ func readDigitalCurrencyAddresses(remarks string) []DigitalCurrencyAddress { // // alt. Digital Currency Address - XBT 12jVCWW1ZhTLA5yVnroEJswqKwsfiZKsax; // - parts := strings.Split(remarks, ";") + parts := splitRemarks(remarks) for i := range parts { // Check if the currency is in the remark var addressIndex int diff --git a/pkg/ofac/reader_test.go b/pkg/ofac/reader_test.go index d3a60f74..32ca3bf5 100644 --- a/pkg/ofac/reader_test.go +++ b/pkg/ofac/reader_test.go @@ -120,6 +120,51 @@ func TestSDNComments(t *testing.T) { } } +func TestSDN__remarks(t *testing.T) { + // individual + remarks := splitRemarks("DOB 12 Oct 1972; POB Corozal, Belize; Passport 0291622 (Belize); Linked To: D'S SUPERMARKET COMPANY LTD.") + expected := []string{"12 Oct 1972"} + require.ElementsMatch(t, expected, findRemarkValues(remarks, "DOB")) + expected = []string{"0291622 (Belize)"} + require.ElementsMatch(t, expected, findRemarkValues(remarks, "Passport")) + + // Contact info + remarks = splitRemarks("Website www.nitc.co.ir; Email Address info@nitc.co.ir; alt. Email Address administrator@nitc.co.ir; IFCA Determination - Involved in the Shipping Sector; Additional Sanctions Information - Subject to Secondary Sanctions; Telephone (98)(21)(66153220); Telephone (98)(21)(23803202); Telephone (98)(21)(23803303); Telephone (98)(21)(66153224); Telephone (98)(21)(23802230); Telephone (98)(9121115315); Telephone (98)(9128091642); Telephone (98)(9127389031); Fax (98)(21)(22224537); Fax (98)(21)(23803318); Fax (98)(21)(22013392); Fax (98)(21)(22058763).") + expected = []string{"www.nitc.co.ir"} + require.ElementsMatch(t, expected, findRemarkValues(remarks, "Website")) + expected = []string{"info@nitc.co.ir", "administrator@nitc.co.ir"} + require.ElementsMatch(t, expected, findRemarkValues(remarks, "Email Address")) + expected = []string{"(98)(21)(66153220)", "(98)(21)(23803202)", "(98)(21)(23803303)", "(98)(21)(66153224)", "(98)(21)(23802230)", "(98)(9121115315)", "(98)(9128091642)", "(98)(9127389031)"} + require.ElementsMatch(t, expected, findRemarkValues(remarks, "Telephone")) + expected = []string{"(98)(21)(22224537)", "(98)(21)(23803318)", "(98)(21)(22013392)", "(98)(21)(22058763)"} + require.ElementsMatch(t, expected, findRemarkValues(remarks, "Fax")) + + // Vessel + remarks = splitRemarks("Former Vessel Flag Malta; alt. Former Vessel Flag Tuvalu; alt. Former Vessel Flag None Identified; alt. Former Vessel Flag Tanzania; Additional Sanctions Information - Subject to Secondary Sanctions; Vessel Registration Identification IMO 9187629; MMSI 572469210; Linked To: NATIONAL IRANIAN TANKER COMPANY.") + expected = []string{"9187629"} + require.ElementsMatch(t, expected, findRemarkValues(remarks, "Vessel Registration Identification IMO")) + expected = []string{"572469210"} + require.ElementsMatch(t, expected, findRemarkValues(remarks, "MMSI")) + + // Aircraft + remarks = splitRemarks("Aircraft Construction Number (also called L/N or S/N or F/N) 8401; Aircraft Manufacture Date 1992; Aircraft Model IL76-TD; Aircraft Operator YAS AIR; Aircraft Manufacturer's Serial Number (MSN) 1023409321; Linked To: POUYA AIR.") + expected = []string{"1992"} + require.ElementsMatch(t, expected, findRemarkValues(remarks, "Manufacture Date")) + expected = []string{"IL76-TD"} + require.ElementsMatch(t, expected, findRemarkValues(remarks, "Model")) + expected = []string{"(MSN) 1023409321"} + require.ElementsMatch(t, expected, findRemarkValues(remarks, "Serial Number")) + + t.Run("error conditions", func(t *testing.T) { + remarks = splitRemarks("") + require.Len(t, findRemarkValues(remarks, ""), 0) + require.Len(t, findRemarkValues(remarks, "DOB"), 0) + + remarks = splitRemarks(" ; ;;;;; ; ;") + require.Len(t, findRemarkValues(remarks, "DOB"), 0) + }) +} + func TestSDNComments_CryptoCurrencies(t *testing.T) { fd, err := os.CreateTemp("", "sdn-comments") require.NoError(t, err) diff --git a/pkg/search/models.go b/pkg/search/models.go new file mode 100644 index 00000000..861e8673 --- /dev/null +++ b/pkg/search/models.go @@ -0,0 +1,178 @@ +package search + +import "time" + +type Entity[T any] struct { + Name string `json:"name"` + Type EntityType `json:"entityType"` + Source SourceList `json:"sourceList"` + SourceID string `json:"sourceID"` // TODO(adam): + + // TODO(adam): What has opensanctions done to normalize and join this data + // Review https://www.opensanctions.org/reference/ + + Person *Person `json:"person"` + Business *Business `json:"business"` + Organization *Organization `json:"organization"` + Aircraft *Aircraft `json:"aircraft"` + Vessel *Vessel `json:"vessel"` + + CryptoAddresses []CryptoAddress `json:"cryptoAddresses"` + + Addresses []Address `json:"addresses"` + + SourceData T `json:"sourceData"` // Contains all original list data with source list naming +} + +type EntityType string + +var ( + EntityPerson EntityType = "person" + EntityBusiness EntityType = "business" + EntityAircraft EntityType = "aircraft" + EntityVessel EntityType = "vessel" + EntityCryptoAddress EntityType = "crypto-address" // TODO(adam): Does this make sense? +) + +type SourceList string + +var ( + SourceEUCSL SourceList = "eu_csl" + SourceUKCSL SourceList = "uk_csl" + SourceUSCSL SourceList = "us_csl" + SourceUSOFAC SourceList = "us_ofac" +) + +type Person struct { + Name string `json:"name"` + AltNames []string `json:"altNames"` + Gender Gender `json:"gender"` + BirthDate *time.Time `json:"birthDate"` + DeathDate *time.Time `json:"deathDate"` + + GovernmentIDs []GovernmentID `json:"governmentIDs"` +} + +type Gender string + +var ( + GenderUnknown Gender = "unknown" + GenderMale Gender = "male" + GenderFemale Gender = "female" +) + +type ContactInfo struct { + EmailAddresses []string + PhoneNumbers []string + FaxNumbers []string +} + +// TODO(adam): +// +// Website www.tidewaterco.com; +// Email Address info@tidewaterco.com; alt. Email Address info@tidewaterco.ir; +// Telephone: 982188553321; Alt. Telephone: 982188554432; +// Fax: 982188717367; Alt. Fax: 982188708761; +// +// 12803,"TIDEWATER MIDDLE EAST CO.",-0- ,"SDGT] [NPWMD] [IRGC] [IFSR] [IFCA",-0- ,-0- ,-0- ,-0- ,-0- ,-0- ,-0- ," alt. Email Address info@tidewaterco.ir; IFCA Determination - Port Operator; Additional Sanctions Information - Subject to Secondary Sanctions; Business Registration Document # 18745 (Iran); Alt. Fax: 982188708911." + +type GovernmentID struct { + Type GovernmentIDType `json:"type"` + Country string `json:"country"` // ISO-3166 // TODO(adam): + Identifier string `json:"identifier"` +} + +type GovernmentIDType string + +var ( + GovernmentIDPassport GovernmentIDType = "passport" +) + +type Business struct { + Name string `json:"name"` + Created *time.Time `json:"created"` + Dissolved *time.Time `json:"dissolved"` + Identifier []Identifier `json:"identifier"` +} + +// Identifier +// +// TODO(adam): Look at OpenSanctions for tax ID codes +// https://www.opensanctions.org/reference/#schema.Company +type Identifier struct { + Name string `json:"string"` + Country string `json:"country"` // ISO-3166 // TODO(adam): + Identifier string `json:"value"` +} + +// Organization +// +// TODO(adam): https://www.opensanctions.org/reference/#schema.Organization +type Organization struct { + Name string `json:"name"` + Created *time.Time `json:"created"` + Dissolved *time.Time `json:"dissolved"` + Identifier []Identifier `json:"identifier"` +} + +type Aircraft struct { + Name string `json:"name"` + Type AircraftType `json:"type"` + Flag string `json:"flag"` // ISO-3166 // TODO(adam): + Built *time.Time `json:"built"` + ICAOCode string `json:"icaoCode"` // ICAO aircraft type designator + Model string `json:"model"` + SerialNumber string `json:"serialNumber"` +} + +type AircraftType string + +var ( + AircraftTypeUnknown AircraftType = "unknown" + AircraftCargo AircraftType = "cargo" +) + +// Vessel +// +// TODO(adam): https://www.opensanctions.org/reference/#schema.Vessel +type Vessel struct { + Name string `json:"name"` + IMONumber string `json:"imoNumber"` + Type VesselType `json:"type"` + Flag string `json:"flag"` // ISO-3166 // TODO(adam): + Built *time.Time `json:"built"` + Model string `json:"model"` + Tonnage int `json:"tonnage"` + MMSI string `json:"mmsi"` // Maritime Mobile Service Identity +} + +type VesselType string + +var ( + VesselTypeUnknown VesselType = "unknown" + VesselTypeCargo VesselType = "cargo" +) + +type CryptoAddress struct { + Currency string `json:"currency"` + Address string `json:"address"` +} + +// Address is a struct which represents any physical location +// +// TODO(adam): Should probably adopt something like libpostal's naming +// https://github.com/openvenues/libpostal?tab=readme-ov-file#parser-labels +// +// Or OpenSanctions +// https://www.opensanctions.org/reference/#schema.Address +type Address struct { + Line1 string `json:"line1"` + Line2 string `json:"line2"` + City string `json:"city"` + PostalCode string `json:"postalCode"` + State string `json:"state"` + Country string `json:"country"` // ISO-3166 code + + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` +} diff --git a/pkg/search/models_test.go b/pkg/search/models_test.go new file mode 100644 index 00000000..f6a0ca5f --- /dev/null +++ b/pkg/search/models_test.go @@ -0,0 +1,39 @@ +package search + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEntityJSON(t *testing.T) { + type SDN struct { + EntityID string `json:"entityID"` + } + bs, err := json.MarshalIndent(Entity[SDN]{ + SourceData: SDN{ + EntityID: "12345", + }, + }, "", " ") + require.NoError(t, err) + + expected := strings.TrimSpace(`{ + "name": "", + "entityType": "", + "sourceList": "", + "sourceID": "", + "person": null, + "business": null, + "organization": null, + "aircraft": null, + "vessel": null, + "cryptoAddresses": null, + "addresses": null, + "sourceData": { + "entityID": "12345" + } +}`) + require.Equal(t, expected, string(bs)) +}