From 81da508945bc3dda5e49f034a5fbb6c6953e3d71 Mon Sep 17 00:00:00 2001 From: awilliams Date: Tue, 17 Jun 2014 11:29:49 +0200 Subject: [PATCH] refactored to avoid reaching API batch request limit; go fmtd --- api/api.go | 194 +++++++++++++++------------------ api/linode.go | 289 ++++++++++++++++++++++++++++++-------------------- inventory.go | 47 ++++---- main.go | 156 +++++++++++++++------------ 4 files changed, 373 insertions(+), 313 deletions(-) diff --git a/api/api.go b/api/api.go index 324ed5a..4839722 100644 --- a/api/api.go +++ b/api/api.go @@ -1,141 +1,119 @@ package api import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "net/url" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" ) const ( - API_URL = "https://api.linode.com/" + ApiUrl = "https://api.linode.com/" + MaxBatchRequests = 25 ) type queryParams map[string]string type apiAction struct { - method string - params queryParams + method string + params queryParams } func (self *apiAction) Set(key, value string) { - self.params[key] = value + self.params[key] = value } func (self apiAction) values() queryParams { - self.params["api_action"] = self.method - return self.params + self.params["api_action"] = self.method + return self.params } type apiRequest struct { - apiKey string - baseUrl *url.URL - actions []*apiAction + apiKey string + baseUrl *url.URL + actions []*apiAction } func NewApiRequest(apiKey string) (*apiRequest, error) { - apiUrl, err := url.Parse(API_URL) - if err != nil { - return nil, err - } - var actions []*apiAction - return &apiRequest{apiKey: apiKey, baseUrl: apiUrl, actions: actions}, nil + apiUrl, err := url.Parse(ApiUrl) + if err != nil { + return nil, err + } + var actions []*apiAction + return &apiRequest{apiKey: apiKey, baseUrl: apiUrl, actions: actions}, nil } func (self *apiRequest) AddAction(method string) *apiAction { - action := &apiAction{method: method, params: make(queryParams)} - self.actions = append(self.actions, action) - return action + action := &apiAction{method: method, params: make(queryParams)} + self.actions = append(self.actions, action) + return action } -func (self apiRequest) URL() string { - params := make(url.Values) - params.Set("api_key", self.apiKey) - - if len(self.actions) == 1 { - for key, value := range self.actions[0].values() { - params.Set(key, value) - } - } else if len(self.actions) > 1 { - params.Set("api_action", "batch") - var requestArray []queryParams - for _, action := range self.actions { - requestArray = append(requestArray, action.values()) - } - b, err := json.Marshal(requestArray) - if err != nil { - log.Fatal(err) - } - params.Set("api_requestArray", string(b)) - } - self.baseUrl.RawQuery = params.Encode() - return self.baseUrl.String() +func (self apiRequest) URL() []string { + actionBatches := make([][]*apiAction, (len(self.actions)/MaxBatchRequests)+1) + for j, action := range self.actions { + i := j / MaxBatchRequests + actionBatches[i] = append(actionBatches[i], action) + } + + var urls []string + for _, actions := range actionBatches { + params := make(url.Values) + params.Set("api_key", self.apiKey) + params.Set("api_action", "batch") + var requestArray []queryParams + for _, action := range actions { + requestArray = append(requestArray, action.values()) + } + b, _ := json.Marshal(requestArray) + params.Set("api_requestArray", string(b)) + u := self.baseUrl + u.RawQuery = params.Encode() + urls = append(urls, u.String()) + } + return urls } -func (self apiRequest) GetJson(data interface{}) error { - resp, err := http.Get(self.URL()) - if err != nil { - return err - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - // the linode API does not use HTTP status codes to indicate errors, - // rather it embeds in the JSON document the errors. When there is an error - // the foramt of the `DATA` element changes as well, which would cause the json decode to fail. - // - // Here we first parse the json to see if it contains errors, then re-parse with the provided - // json structure - if linodeErr := checkForLinodeError(bytes.NewReader(body)); linodeErr != nil { - return linodeErr - } - - decoder := json.NewDecoder(bytes.NewReader(body)) - err = decoder.Decode(data) - if err != nil { - return err - } - - return nil +type apiResponse struct { + Action string `json:"ACTION"` + Errors []struct { + Code int `json:"ERRORCODE"` + Message string `json:"ERRORMESSAGE"` + } `json:"ERRORARRAY,omitempty"` + Data json.RawMessage `json:"DATA,omitempty"` } -type errorJson struct { - Errors []struct { - Code int `json:"ERRORCODE"` - Message string `json:"ERRORMESSAGE"` - } `json:"ERRORARRAY,omitempty"` -} - -func checkForLinodeError(body *bytes.Reader) error { - data := new(errorJson) - decoder := json.NewDecoder(body) - err := decoder.Decode(&data) - if err != nil { - // this is not actually an error, since there is not always an error present in the JSON - return nil - } - if len(data.Errors) > 0 { - var buf bytes.Buffer - buf.WriteString("Api Error!\n") - for _, e := range data.Errors { - buf.WriteString(fmt.Sprintf("[Code: %d] %s\n", e.Code, e.Message)) - } - return fmt.Errorf(buf.String()) - } - return nil -} - -func (self *apiRequest) GoString() string { - s, err := url.QueryUnescape(self.URL()) - if err != nil { - return "" - } - return s +func (self apiRequest) GetJson() ([]json.RawMessage, []error) { + var datas []json.RawMessage + var errs []error + + for _, url := range self.URL() { + resp, err := http.Get(url) + if err != nil { + return nil, []error{err} + } + defer resp.Body.Close() + + decoder := json.NewDecoder(resp.Body) + + var apiResponses []apiResponse + err = decoder.Decode(&apiResponses) + if err != nil { + return nil, []error{err} + } + + for _, apiResp := range apiResponses { + // Check for 'ERROR' attribute for any values, which would indicate an error + if len(apiResp.Errors) > 0 { + for _, e := range apiResp.Errors { + errs = append(errs, errors.New(fmt.Sprintf("[Code: %d] %s\n", e.Code, e.Message))) + } + continue + } + datas = append(datas, apiResp.Data) + } + } + + return datas, errs } diff --git a/api/linode.go b/api/linode.go index c762bdd..4688e8e 100644 --- a/api/linode.go +++ b/api/linode.go @@ -1,142 +1,205 @@ package api import ( - "strconv" - "sort" + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" ) -type Linode struct { - Id int `json:"LINODEID"` - Status int `json:"STATUS"` - Label string `json:"LABEL"` - DisplayGroup string `json:"LPM_DISPLAYGROUP"` - Ram int `json:"TOTALRAM"` - Ips []LinodeIp +func FetchLinodesWithIps(apiKey string) (Linodes, error) { + api, err := NewApiRequest(apiKey) + if err != nil { + return nil, err + } + + linodes, err := FetchLinodeList(*api) + if err != nil { + return nil, err + } + + linodeIps, err := FetchLinodeIpList(*api, linodes.Ids()) + if err != nil { + return nil, err + } + + // associate ips with linodes + for _, linodeDisplayGroup := range linodes { + for _, linode := range linodeDisplayGroup { + if ips, ok := linodeIps[linode.Id]; ok { + sortLinodeIps(ips) + linode.Ips = ips + } + } + sortLinodes(linodeDisplayGroup) + } + + return linodes, nil } -func (self Linode) PublicIp() string { - var ip string - for _, linodeIp := range self.Ips { - if linodeIp.Public == 1 { - ip = linodeIp.Ip - break - } - } - return ip +// map of Linodes by their display group +type Linodes map[string][]*Linode + +func (nodes Linodes) Ids() []int { + ids := []int{} + for _, linodeDisplayGroup := range nodes { + for _, linode := range linodeDisplayGroup { + ids = append(ids, linode.Id) + } + } + return ids } -func (self Linode) PrivateIp() string { - var ip string - for _, linodeIp := range self.Ips { - if linodeIp.Public == 0 { - ip = linodeIp.Ip - break - } - } - return ip +func (nodes Linodes) Size() int { + s := 0 + for _, grp := range nodes { + s += len(grp) + } + return s } -type sortedLinodeIps []LinodeIp +type Linode struct { + Id int `json:"LINODEID"` + Status int `json:"STATUS"` + Label string `json:"LABEL"` + DisplayGroup string `json:"LPM_DISPLAYGROUP"` + Ram int `json:"TOTALRAM"` + Ips []*LinodeIp +} -func (self sortedLinodeIps) Len() int { - return len(self) +func (node *Linode) PublicIp() string { + var ip string + for _, linodeIp := range node.Ips { + if linodeIp.Public == 1 { + ip = linodeIp.Ip + break + } + } + return ip } -func (self sortedLinodeIps) Swap(i, j int) { - self[i], self[j] = self[j], self[i] + +func (node *Linode) PrivateIp() string { + var ip string + for _, linodeIp := range node.Ips { + if linodeIp.Public == 0 { + ip = linodeIp.Ip + break + } + } + return ip } -// Public first -func (self sortedLinodeIps) Less(i, j int) bool { - return self[i].Public > self[j].Public + +func (node *Linode) IsRunning() bool { + return node.Status == 1 } -func (self Linode) SortIps() { - sort.Sort(sortedLinodeIps(self.Ips)) +func FetchLinodeList(api apiRequest) (Linodes, error) { + api.AddAction("linode.list") + + datas, errs := api.GetJson() + if len(errs) > 0 { + errMsg := make([]string, len(errs)) + for i, err := range errs { + errMsg[i] = err.Error() + } + return nil, errors.New(strings.Join(errMsg, "\n")) + } + + var err error + if len(datas) != 1 { + return nil, fmt.Errorf("unexpected numbers of results") + } + var ls []Linode + err = json.Unmarshal(datas[0], &ls) + if err != nil { + return nil, err + } + linodes := make(Linodes, len(ls)) + for _, linode := range ls { + l := linode + linodes[linode.DisplayGroup] = append(linodes[linode.DisplayGroup], &l) + } + + return linodes, nil } -type Linodes map[int]*Linode +// map of LinodeIps by their Linode.Id +type LinodeIps map[int][]*LinodeIp -func (self Linodes) FilterByDisplayGroup(group string) Linodes { - for id, linode := range self { - if linode.Status != 1 || (linode.DisplayGroup != "" && linode.DisplayGroup != group) { - delete(self, id) - } - } - return self +type LinodeIp struct { + LinodeId int `json:"LINODEID"` + Public int `json:"ISPUBLIC"` + Ip string `json:"IPADDRESS"` } -func (self Linodes) FilterByStatus(status int) Linodes { - for id, linode := range self { - if linode.Status != status { - delete(self, id) - } - } - return self +func FetchLinodeIpList(api apiRequest, linodeIds []int) (LinodeIps, error) { + apiMethod := "linode.ip.list" + // one batch request for all linode Ids + for _, linodeId := range linodeIds { + action := api.AddAction(apiMethod) + action.Set("LinodeID", strconv.Itoa(linodeId)) + } + + datas, errs := api.GetJson() + if len(errs) > 0 { + errMsg := make([]string, len(errs)) + for i, err := range errs { + errMsg[i] = err.Error() + } + return nil, errors.New(strings.Join(errMsg, "\n")) + } + + var err error + linodeIps := make(LinodeIps, len(datas)) + for _, rawJson := range datas { + var ipList []LinodeIp + err = json.Unmarshal(rawJson, &ipList) + if err != nil { + return nil, err + } + for _, linodeIp := range ipList { + i := linodeIp + linodeIps[linodeIp.LinodeId] = append(linodeIps[linodeIp.LinodeId], &i) + } + } + + return linodeIps, nil } -func LinodeList(apiKey string) (Linodes, error) { - method := "linode.list" - apiRequest, err := NewApiRequest(apiKey) - if err != nil { - return nil, err - } - apiRequest.AddAction(method) +// Sort functions - var data struct { - Linodes []Linode `json:"DATA,omitempty"` - } - err = apiRequest.GetJson(&data) - if err != nil { - return nil, err - } +type sortedLinodeIps []*LinodeIp - linodes := make(Linodes) - for _, linode := range data.Linodes { - l := linode - linodes[linode.Id] = &l - } +func (sorted sortedLinodeIps) Len() int { + return len(sorted) +} +func (sorted sortedLinodeIps) Swap(i, j int) { + sorted[i], sorted[j] = sorted[j], sorted[i] +} - return linodes, nil +// Sort by public IPs first +func (sorted sortedLinodeIps) Less(i, j int) bool { + return sorted[i].Public > sorted[j].Public +} +func sortLinodeIps(ips []*LinodeIp) { + sort.Sort(sortedLinodeIps(ips)) } -type LinodeIp struct { - LinodeId int `json:"LINODEID"` - Ip string `json:"IPADDRESS"` - Public int `json:"ISPUBLIC"` -} - -// first fetch the list of linodes, -// then use a batch request to list all the ips associated with those linodes -func LinodeListWithIps(apiKey string) (Linodes, error) { - linodes, err := LinodeList(apiKey) - if err != nil { - return nil, err - } - - method := "linode.ip.list" - apiRequest, err := NewApiRequest(apiKey) - if err != nil { - return nil, err - } - for _, linode := range linodes { - action := apiRequest.AddAction(method) - action.Set("LinodeID", strconv.Itoa(linode.Id)) - } - - var data []struct { - LinodeIps []LinodeIp `json:"DATA"` - } - err = apiRequest.GetJson(&data) - if err != nil { - return nil, err - } - - for _, ipList := range data { - for _, linodeIp := range ipList.LinodeIps { - if linode, ok := linodes[linodeIp.LinodeId]; ok { - linode.Ips = append(linode.Ips, linodeIp) - } - } - } - - return linodes, nil +type sortedLinodes []*Linode + +func (sorted sortedLinodes) Len() int { + return len(sorted) +} +func (sorted sortedLinodes) Swap(i, j int) { + sorted[i], sorted[j] = sorted[j], sorted[i] +} + +// Sort by Label +func (sorted sortedLinodes) Less(i, j int) bool { + return sorted[i].Label < sorted[j].Label +} +func sortLinodes(linodes []*Linode) { + sort.Sort(sortedLinodes(linodes)) } diff --git a/inventory.go b/inventory.go index eddb0c2..756f3b8 100644 --- a/inventory.go +++ b/inventory.go @@ -3,35 +3,36 @@ package main // Provides output for use as an Ansible inventory plugin import ( - "encoding/json" - "github.com/awilliams/linode-inventory/api" + "encoding/json" + + "github.com/awilliams/linode-inventory/api" ) type HostMeta map[string]string type Inventory struct { - Meta map[string]map[string]HostMeta `json:"_meta"` - Hosts []string `json:"hosts"` + Meta map[string]map[string]HostMeta `json:"_meta"` + Hosts []string `json:"hosts"` } -func (self Inventory) toJson() ([]byte, error) { - return json.MarshalIndent(self, " ", " ") +func (i Inventory) toJSON() ([]byte, error) { + return json.MarshalIndent(i, " ", " ") } -func makeInventory(linodes api.Linodes) Inventory { - meta := make(map[string]map[string]HostMeta) - hostvars := make(map[string]HostMeta) - meta["hostvars"] = hostvars - - inventory := Inventory{Hosts: []string{}, Meta: meta} - for _, linode := range linodes { - inventory.Hosts = append(inventory.Hosts, linode.Label) - hostmeta := make(HostMeta) - hostmeta["ansible_ssh_host"] = linode.PublicIp() - hostmeta["host_label"] = linode.Label - hostmeta["host_display_group"] = linode.DisplayGroup - hostmeta["host_private_ip"] = linode.PrivateIp() - hostvars[linode.Label] = hostmeta - } - return inventory -} \ No newline at end of file +func makeInventory(linodes []*api.Linode) Inventory { + meta := make(map[string]map[string]HostMeta) + hostvars := make(map[string]HostMeta) + meta["hostvars"] = hostvars + + inventory := Inventory{Hosts: []string{}, Meta: meta} + for _, linode := range linodes { + inventory.Hosts = append(inventory.Hosts, linode.Label) + hostmeta := make(HostMeta) + hostmeta["ansible_ssh_host"] = linode.PublicIp() + hostmeta["host_label"] = linode.Label + hostmeta["host_display_group"] = linode.DisplayGroup + hostmeta["host_private_ip"] = linode.PrivateIp() + hostvars[linode.Label] = hostmeta + } + return inventory +} diff --git a/main.go b/main.go index 412023a..c2bad9b 100644 --- a/main.go +++ b/main.go @@ -1,89 +1,107 @@ package main import ( - "code.google.com/p/gcfg" - "fmt" - "github.com/awilliams/linode-inventory/api" - "log" - "os" - "path/filepath" + "fmt" + "log" + "os" + "path/filepath" + + "code.google.com/p/gcfg" + "github.com/awilliams/linode-inventory/api" ) -const CONFIG_PATH = "linode-inventory.ini" +const ConfigPath = "linode-inventory.ini" type Configuration struct { - ApiKey string `gcfg:"api-key"` - DisplayGroup string `gcfg:"display-group"` + APIKey string `gcfg:"api-key"` + DisplayGroup string `gcfg:"display-group"` } func getConfig() (*Configuration, error) { - // first check directory where the executable is located - dir, err := filepath.Abs(filepath.Dir(os.Args[0])) - if err != nil { - return nil, err - } - path := dir + "/" + CONFIG_PATH - if _, err := os.Stat(path); os.IsNotExist(err) { - // fallback to PWD. This is usefull when using `go run` - path = CONFIG_PATH - } + // first check directory where the executable is located + dir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + return nil, err + } + path := dir + "/" + ConfigPath + if _, err := os.Stat(path); os.IsNotExist(err) { + // fallback to PWD. This is usefull when using `go run` + path = ConfigPath + } - var config struct { - Linode Configuration - } + var config struct { + Linode Configuration + } - err = gcfg.ReadFileInto(&config, path) - if err != nil { - return nil, err - } + err = gcfg.ReadFileInto(&config, path) + if err != nil { + return nil, err + } - return &config.Linode, nil + return &config.Linode, nil } -func getLinodes(config *Configuration) (api.Linodes, error) { - linodes, err := api.LinodeListWithIps(config.ApiKey) - if err != nil { - return nil, err - } - // 1 == running - linodes = linodes.FilterByStatus(1) - // only apply DisplayGroup filter when using ansible feature - if config.DisplayGroup != "" { - linodes = linodes.FilterByDisplayGroup(config.DisplayGroup) - } - - return linodes, nil +func getLinodes(config *Configuration) ([]*api.Linode, error) { + nodeMap, err := api.FetchLinodesWithIps(config.APIKey) + if err != nil { + return nil, err + } + + var linodes []*api.Linode + // function to add nodes to the slice + addNodes := func(nodes []*api.Linode) { + for _, node := range nodes { + // Status 1 == running + if node.Status == 1 { + linodes = append(linodes, node) + } + } + } + + if config.DisplayGroup != "" { + nodes, ok := nodeMap[config.DisplayGroup] + if !ok { + return nil, fmt.Errorf("display group '%s' not found", config.DisplayGroup) + } + addNodes(nodes) + } else { + for _, nodes := range nodeMap { + addNodes(nodes) + } + } + + return linodes, nil } func main() { - config, err := getConfig() - if err != nil { - log.Fatal(err) - } + config, err := getConfig() + if err != nil { + log.Fatal(err) + } + + // --list and --host are called from Ansible + // see: http://docs.ansible.com/developing_inventory.html + if len(os.Args) > 1 && os.Args[1][0:2] == "--" { + switch os.Args[1] { + case "--list": + linodes, err := getLinodes(config) + if err != nil { + log.Fatal(err) + } - // --list and --host are called from Ansible - // see: http://docs.ansible.com/developing_inventory.html - if len(os.Args) > 1 && os.Args[1][0:2] == "--" { - switch os.Args[1] { - case "--list": - linodes, err := getLinodes(config) - if err != nil { - log.Fatal(err) - } - - inventory := makeInventory(linodes) - inventoryJson, err := inventory.toJson() - if err != nil { - log.Fatal(err) - } - os.Stdout.Write(inventoryJson) - case "--host": - // empty hash - fmt.Fprint(os.Stdout, "{}") - default: - fmt.Fprintf(os.Stderr, "Unrecognized flag: %v\nUsage: linode-inventory --list or --host\n", os.Args[1]) - } - } else { - fmt.Fprint(os.Stderr, "Usage: linode-inventory --list or --host\n") - } + inventory := makeInventory(linodes) + inventoryJSON, err := inventory.toJSON() + if err != nil { + log.Fatal(err) + } + os.Stdout.Write(inventoryJSON) + case "--host": + // empty hash + fmt.Fprint(os.Stdout, "{}") + default: + fmt.Fprintf(os.Stderr, "Unrecognized flag: %v\nUsage: linode-inventory --list or --host\n", os.Args[1]) + } + } else { + fmt.Fprint(os.Stderr, "Usage: linode-inventory --list or --host\n") + } }