From 0a9c58689cb331462e5f62781108becb26b0edf6 Mon Sep 17 00:00:00 2001 From: Ji-Ping Shen <5998992+jiping-s@users.noreply.github.com> Date: Mon, 17 Jul 2023 10:58:57 +0300 Subject: [PATCH] feat: Support marshal and unmarshal (#10) * add Makefile * expose all private fields * support marshaling and unmarshaling * upgrade go to 1.20 --- Makefile | 16 +++++ aini.go | 24 ++++---- go.mod | 13 ++++- go.sum | 10 +++- inventory.go | 28 ++++----- marshal.go | 74 +++++++++++++++++++++++ marshal_test.go | 57 ++++++++++++++++++ marshal_test_inventory.json | 113 ++++++++++++++++++++++++++++++++++++ ordered.go | 6 +- parser.go | 12 ++-- vars.go | 38 ++++++------ 11 files changed, 331 insertions(+), 60 deletions(-) create mode 100644 Makefile create mode 100644 marshal.go create mode 100644 marshal_test.go create mode 100644 marshal_test_inventory.json diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f1fcf01 --- /dev/null +++ b/Makefile @@ -0,0 +1,16 @@ +SOURCES ?= $(shell find . -name '*.go') +SOURCES_NONTEST ?= $(shell find . -name '*.go' -not -name '*_test.go') + +.PHONY: test +test: + go test -timeout $${TEST_TIMEOUT:-10s} -v ./... + +# test-all ignores testcache (go clean testcache) +.PHONY: test-all +test-all: + go test -timeout $${TEST_TIMEOUT:-10s} -v -count=1 ./... + +.PHONY: upgrade +upgrade: + rm -f go.sum + go get -u -d ./...; go mod tidy diff --git a/aini.go b/aini.go index 71ca15c..c2652f9 100644 --- a/aini.go +++ b/aini.go @@ -4,7 +4,7 @@ import ( "bufio" "bytes" "io" - "io/ioutil" + "os" "path" "sort" "strings" @@ -25,15 +25,15 @@ type Group struct { Children map[string]*Group Parents map[string]*Group - directParents map[string]*Group + DirectParents map[string]*Group // Vars set in inventory - inventoryVars map[string]string + InventoryVars map[string]string // Vars set in group_vars - fileVars map[string]string + FileVars map[string]string // Projection of all parent inventory variables - allInventoryVars map[string]string + AllInventoryVars map[string]string // Projection of all parent group_vars variables - allFileVars map[string]string + AllFileVars map[string]string } // Host represents ansible host @@ -43,16 +43,16 @@ type Host struct { Vars map[string]string Groups map[string]*Group - directGroups map[string]*Group + DirectGroups map[string]*Group // Vars set in inventory - inventoryVars map[string]string + InventoryVars map[string]string // Vars set in host_vars - fileVars map[string]string + FileVars map[string]string } // ParseFile parses Inventory represented as a file func ParseFile(f string) (*InventoryData, error) { - bs, err := ioutil.ReadFile(f) + bs, err := os.ReadFile(f) if err != nil { return &InventoryData{}, err } @@ -143,7 +143,7 @@ func hostMapToLower(hosts map[string]*Host, keysOnly bool) map[string]*Host { func (inventory *InventoryData) GroupsToLower() { inventory.Groups = groupMapToLower(inventory.Groups, false) for _, host := range inventory.Hosts { - host.directGroups = groupMapToLower(host.directGroups, true) + host.DirectGroups = groupMapToLower(host.DirectGroups, true) host.Groups = groupMapToLower(host.Groups, true) } } @@ -162,7 +162,7 @@ func groupMapToLower(groups map[string]*Group, keysOnly bool) map[string]*Group groupname = strings.ToLower(groupname) if !keysOnly { group.Name = groupname - group.directParents = groupMapToLower(group.directParents, true) + group.DirectParents = groupMapToLower(group.DirectParents, true) group.Parents = groupMapToLower(group.Parents, true) group.Children = groupMapToLower(group.Children, true) } diff --git a/go.mod b/go.mod index 32eefbe..143f3ff 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,16 @@ module github.com/relex/aini -go 1.13 +go 1.20 require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 - github.com/stretchr/testify v1.7.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + github.com/samber/lo v1.38.1 + github.com/stretchr/testify v1.7.0 + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/davecgh/go-spew v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 59db3a0..e0caf8e 100644 --- a/go.sum +++ b/go.sum @@ -4,11 +4,15 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= +github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/inventory.go b/inventory.go index 3e25102..44716b2 100644 --- a/inventory.go +++ b/inventory.go @@ -23,11 +23,11 @@ func (inventory *InventoryData) Reconcile() { allGroup := inventory.getOrCreateGroup("all") ungroupedGroup := inventory.getOrCreateGroup("ungrouped") - ungroupedGroup.directParents[allGroup.Name] = allGroup + ungroupedGroup.DirectParents[allGroup.Name] = allGroup // First, ensure that inventory.Groups contains all the groups for _, host := range inventory.Hosts { - for _, group := range host.directGroups { + for _, group := range host.DirectGroups { inventory.Groups[group.Name] = group for _, ancestor := range group.ListParentGroupsOrdered() { inventory.Groups[ancestor.Name] = ancestor @@ -37,7 +37,7 @@ func (inventory *InventoryData) Reconcile() { // Calculate intergroup relationships for _, group := range inventory.Groups { - group.directParents[allGroup.Name] = allGroup + group.DirectParents[allGroup.Name] = allGroup for _, ancestor := range group.ListParentGroupsOrdered() { group.Parents[ancestor.Name] = ancestor ancestor.Children[group.Name] = group @@ -47,7 +47,7 @@ func (inventory *InventoryData) Reconcile() { // Now set hosts for groups and groups for hosts for _, host := range inventory.Hosts { host.Groups[allGroup.Name] = allGroup - for _, group := range host.directGroups { + for _, group := range host.DirectGroups { group.Hosts[host.Name] = host host.Groups[group.Name] = group for _, parent := range group.Parents { @@ -64,7 +64,7 @@ func (inventory *InventoryData) Reconcile() { func (host *Host) clearData() { host.Groups = make(map[string]*Group) host.Vars = make(map[string]string) - for _, group := range host.directGroups { + for _, group := range host.DirectGroups { group.clearData(make(map[string]struct{}, len(host.Groups))) } } @@ -77,10 +77,10 @@ func (group *Group) clearData(visited map[string]struct{}) { group.Parents = make(map[string]*Group) group.Children = make(map[string]*Group) group.Vars = make(map[string]string) - group.allInventoryVars = nil - group.allFileVars = nil + group.AllInventoryVars = nil + group.AllFileVars = nil visited[group.Name] = struct{}{} - for _, parent := range group.directParents { + for _, parent := range group.DirectParents { parent.clearData(visited) } } @@ -97,9 +97,9 @@ func (inventory *InventoryData) getOrCreateGroup(groupName string) *Group { Children: make(map[string]*Group), Parents: make(map[string]*Group), - directParents: make(map[string]*Group), - inventoryVars: make(map[string]string), - fileVars: make(map[string]string), + DirectParents: make(map[string]*Group), + InventoryVars: make(map[string]string), + FileVars: make(map[string]string), } inventory.Groups[groupName] = g return g @@ -116,9 +116,9 @@ func (inventory *InventoryData) getOrCreateHost(hostName string) *Host { Groups: make(map[string]*Group), Vars: make(map[string]string), - directGroups: make(map[string]*Group), - inventoryVars: make(map[string]string), - fileVars: make(map[string]string), + DirectGroups: make(map[string]*Group), + InventoryVars: make(map[string]string), + FileVars: make(map[string]string), } inventory.Hosts[hostName] = h return h diff --git a/marshal.go b/marshal.go new file mode 100644 index 0000000..297d14a --- /dev/null +++ b/marshal.go @@ -0,0 +1,74 @@ +package aini + +import ( + "encoding/json" + + "github.com/samber/lo" + "golang.org/x/exp/maps" +) + +type alwaysNil interface{} // to hold place for Group and Host references; must be nil in serialized form + +func (group *Group) MarshalJSON() ([]byte, error) { + type groupWithoutCustomMarshal Group + + return json.Marshal(&struct { + groupWithoutCustomMarshal + Hosts map[string]alwaysNil + Children map[string]alwaysNil + Parents map[string]alwaysNil + DirectParents map[string]alwaysNil + }{ + groupWithoutCustomMarshal: groupWithoutCustomMarshal(*group), + Hosts: makeNilValueMap(group.Hosts), + Children: makeNilValueMap(group.Children), + Parents: makeNilValueMap(group.Parents), + DirectParents: makeNilValueMap(group.DirectParents), + }) +} + +func (host *Host) MarshalJSON() ([]byte, error) { + type hostWithoutCustomMarshal Host + + return json.Marshal(&struct { + hostWithoutCustomMarshal + Groups map[string]alwaysNil + DirectGroups map[string]alwaysNil + }{ + hostWithoutCustomMarshal: hostWithoutCustomMarshal(*host), + Groups: makeNilValueMap(host.Groups), + DirectGroups: makeNilValueMap(host.DirectGroups), + }) +} + +func makeNilValueMap[K comparable, V any](m map[K]*V) map[K]alwaysNil { + return lo.MapValues(m, func(_ *V, _ K) alwaysNil { return nil }) +} + +func (inventory *InventoryData) UnmarshalJSON(data []byte) error { + type inventoryWithoutCustomUnmarshal InventoryData + var rawInventory inventoryWithoutCustomUnmarshal + if err := json.Unmarshal(data, &rawInventory); err != nil { + return err + } + // rawInventory's Groups and Hosts should now contain all properties, + // except child group maps and host maps are filled with original keys and null values + + // reassign child groups and hosts to reference rawInventory.Hosts and .Groups + + for _, group := range rawInventory.Groups { + group.Hosts = lo.PickByKeys(rawInventory.Hosts, maps.Keys(group.Hosts)) + group.Children = lo.PickByKeys(rawInventory.Groups, maps.Keys(group.Children)) + group.Parents = lo.PickByKeys(rawInventory.Groups, maps.Keys(group.Parents)) + group.DirectParents = lo.PickByKeys(rawInventory.Groups, maps.Keys(group.DirectParents)) + } + + for _, host := range rawInventory.Hosts { + host.Groups = lo.PickByKeys(rawInventory.Groups, maps.Keys(host.Groups)) + host.DirectGroups = lo.PickByKeys(rawInventory.Groups, maps.Keys(host.DirectGroups)) + } + + inventory.Groups = rawInventory.Groups + inventory.Hosts = rawInventory.Hosts + return nil +} diff --git a/marshal_test.go b/marshal_test.go new file mode 100644 index 0000000..f7b4099 --- /dev/null +++ b/marshal_test.go @@ -0,0 +1,57 @@ +package aini + +import ( + _ "embed" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +const minMarshalInventory = `[Animals] +ET + +[Animals:children] +Cats + +[Cats] +Lion +` + +//go:embed marshal_test_inventory.json +var minMarshalJSON string + +func TestMarshalJSON(t *testing.T) { + v, err := ParseString(minMarshalInventory) + assert.Nil(t, err) + + j, err := json.MarshalIndent(v, "", " ") + assert.Nil(t, err) + assert.Equal(t, minMarshalJSON, string(j)) + + t.Run("unmarshal", func(t *testing.T) { + var v2 InventoryData + assert.Nil(t, json.Unmarshal(j, &v2)) + assert.Equal(t, v.Hosts["Lion"], v2.Hosts["Lion"]) + assert.Equal(t, v.Groups["Cats"], v2.Groups["Cats"]) + }) +} + +func TestMarshalWithVars(t *testing.T) { + v, err := ParseFile("test_data/inventory") + assert.Nil(t, err) + + v.HostsToLower() + v.GroupsToLower() + v.AddVarsLowerCased("test_data") + + j, err := json.MarshalIndent(v, "", " ") + assert.Nil(t, err) + + t.Run("unmarshal", func(t *testing.T) { + var v2 InventoryData + assert.Nil(t, json.Unmarshal(j, &v2)) + assert.Equal(t, v.Hosts["host1"], v2.Hosts["host1"]) + assert.Equal(t, v.Groups["tomcat"], v2.Groups["tomcat"]) + }) +} diff --git a/marshal_test_inventory.json b/marshal_test_inventory.json new file mode 100644 index 0000000..a686d78 --- /dev/null +++ b/marshal_test_inventory.json @@ -0,0 +1,113 @@ +{ + "Groups": { + "Animals": { + "Name": "Animals", + "Vars": {}, + "InventoryVars": {}, + "FileVars": {}, + "AllInventoryVars": {}, + "AllFileVars": {}, + "Hosts": { + "ET": null, + "Lion": null + }, + "Children": { + "Cats": null + }, + "Parents": { + "all": null + }, + "DirectParents": { + "all": null + } + }, + "Cats": { + "Name": "Cats", + "Vars": {}, + "InventoryVars": {}, + "FileVars": {}, + "AllInventoryVars": {}, + "AllFileVars": {}, + "Hosts": { + "Lion": null + }, + "Children": {}, + "Parents": { + "Animals": null, + "all": null + }, + "DirectParents": { + "Animals": null, + "all": null + } + }, + "all": { + "Name": "all", + "Vars": {}, + "InventoryVars": {}, + "FileVars": {}, + "AllInventoryVars": {}, + "AllFileVars": {}, + "Hosts": { + "ET": null, + "Lion": null + }, + "Children": { + "Animals": null, + "Cats": null, + "ungrouped": null + }, + "Parents": {}, + "DirectParents": { + "all": null + } + }, + "ungrouped": { + "Name": "ungrouped", + "Vars": {}, + "InventoryVars": {}, + "FileVars": {}, + "AllInventoryVars": {}, + "AllFileVars": {}, + "Hosts": {}, + "Children": {}, + "Parents": { + "all": null + }, + "DirectParents": { + "all": null + } + } + }, + "Hosts": { + "ET": { + "Name": "ET", + "Port": 22, + "Vars": {}, + "InventoryVars": {}, + "FileVars": {}, + "Groups": { + "Animals": null, + "all": null + }, + "DirectGroups": { + "Animals": null + } + }, + "Lion": { + "Name": "Lion", + "Port": 22, + "Vars": {}, + "InventoryVars": {}, + "FileVars": {}, + "Groups": { + "Animals": null, + "Cats": null, + "all": null + }, + "DirectGroups": { + "Cats": null + } + } + } +} \ No newline at end of file diff --git a/ordered.go b/ordered.go index 029a40e..d90cd4f 100644 --- a/ordered.go +++ b/ordered.go @@ -44,13 +44,13 @@ func (group *Group) MatchGroupsOrdered(pattern string) ([]*Group, error) { // ListGroupsOrdered returns all ancestor groups of a given host in level order func (host *Host) ListGroupsOrdered() []*Group { - return listAncestorsOrdered(host.directGroups, nil, true) + return listAncestorsOrdered(host.DirectGroups, nil, true) } // ListParentGroupsOrdered returns all ancestor groups of a given group in level order func (group *Group) ListParentGroupsOrdered() []*Group { visited := map[string]struct{}{group.Name: {}} - return listAncestorsOrdered(group.directParents, visited, group.Name != "all") + return listAncestorsOrdered(group.DirectParents, visited, group.Name != "all") } // listAncestorsOrdered returns all ancestor groups of a given group map in level order @@ -74,7 +74,7 @@ func listAncestorsOrdered(groups map[string]*Group, visited map[string]struct{}, continue } visited[group.Name] = struct{}{} - parentList := GroupMapListValues(group.directParents) + parentList := GroupMapListValues(group.DirectParents) result = append(result, group) queue = append(queue, parentList...) } diff --git a/parser.go b/parser.go index 0dc46d5..f5fb821 100644 --- a/parser.go +++ b/parser.go @@ -71,10 +71,10 @@ func (inventory *InventoryData) parse(reader *bufio.Reader) error { return err } for _, host := range hosts { - host.directGroups[activeGroup.Name] = activeGroup + host.DirectGroups[activeGroup.Name] = activeGroup inventory.Hosts[host.Name] = host if activeGroup.Name != "ungrouped" { - delete(host.directGroups, "ungrouped") + delete(host.DirectGroups, "ungrouped") } } } @@ -85,7 +85,7 @@ func (inventory *InventoryData) parse(reader *bufio.Reader) error { } groupName := parsed[0] newGroup := inventory.getOrCreateGroup(groupName) - newGroup.directParents[activeGroup.Name] = activeGroup + newGroup.DirectParents[activeGroup.Name] = activeGroup inventory.Groups[line] = newGroup } if activeState == varsState { @@ -93,7 +93,7 @@ func (inventory *InventoryData) parse(reader *bufio.Reader) error { if err != nil { return err } - activeGroup.inventoryVars[k] = v + activeGroup.InventoryVars[k] = v } } inventory.Groups[activeGroup.Name] = activeGroup @@ -128,8 +128,8 @@ func (inventory *InventoryData) getHosts(line string, group *Group) (map[string] host := inventory.getOrCreateHost(hostname) host.Port = port - host.directGroups[group.Name] = group - addValues(host.inventoryVars, vars) + host.DirectGroups[group.Name] = group + addValues(host.InventoryVars, vars) result[host.Name] = host } diff --git a/vars.go b/vars.go index 17ef223..b68a9d6 100644 --- a/vars.go +++ b/vars.go @@ -41,11 +41,11 @@ type fileVarsGetter interface { } func (host *Host) getFileVars() map[string]string { - return host.fileVars + return host.FileVars } func (group *Group) getFileVars() map[string]string { - return group.fileVars + return group.FileVars } func (inventory InventoryData) getHostsMap() map[string]fileVarsGetter { @@ -148,8 +148,8 @@ func (inventory *InventoryData) reconcileVars() { 4. inventory host_vars/* */ for _, group := range inventory.Groups { - group.allInventoryVars = nil - group.allFileVars = nil + group.AllInventoryVars = nil + group.AllFileVars = nil } for _, group := range inventory.Groups { group.Vars = make(map[string]string) @@ -157,39 +157,39 @@ func (inventory *InventoryData) reconcileVars() { group.populateFileVars() // At this point we already "populated" all parent's inventory and file vars // So it's fine to build Vars right away, without needing the second pass - group.Vars = copyStringMap(group.allInventoryVars) - addValues(group.Vars, group.allFileVars) + group.Vars = copyStringMap(group.AllInventoryVars) + addValues(group.Vars, group.AllFileVars) } for _, host := range inventory.Hosts { host.Vars = make(map[string]string) - for _, group := range GroupMapListValues(host.directGroups) { + for _, group := range GroupMapListValues(host.DirectGroups) { addValues(host.Vars, group.Vars) } - addValues(host.Vars, host.inventoryVars) - addValues(host.Vars, host.fileVars) + addValues(host.Vars, host.InventoryVars) + addValues(host.Vars, host.FileVars) } } func (group *Group) populateInventoryVars() { - if group.allInventoryVars != nil { + if group.AllInventoryVars != nil { return } - group.allInventoryVars = make(map[string]string) - for _, parent := range GroupMapListValues(group.directParents) { + group.AllInventoryVars = make(map[string]string) + for _, parent := range GroupMapListValues(group.DirectParents) { parent.populateInventoryVars() - addValues(group.allInventoryVars, parent.allInventoryVars) + addValues(group.AllInventoryVars, parent.AllInventoryVars) } - addValues(group.allInventoryVars, group.inventoryVars) + addValues(group.AllInventoryVars, group.InventoryVars) } func (group *Group) populateFileVars() { - if group.allFileVars != nil { + if group.AllFileVars != nil { return } - group.allFileVars = make(map[string]string) - for _, parent := range GroupMapListValues(group.directParents) { + group.AllFileVars = make(map[string]string) + for _, parent := range GroupMapListValues(group.DirectParents) { parent.populateFileVars() - addValues(group.allFileVars, parent.allFileVars) + addValues(group.AllFileVars, parent.AllFileVars) } - addValues(group.allFileVars, group.fileVars) + addValues(group.AllFileVars, group.FileVars) }