Skip to content

Commit

Permalink
replace URL query with URL payload
Browse files Browse the repository at this point in the history
  • Loading branch information
@dmitsh
dmitsh committed Oct 18, 2024
1 parent 31c08a9 commit 0edc13b
Show file tree
Hide file tree
Showing 18 changed files with 239 additions and 227 deletions.
2 changes: 0 additions & 2 deletions pkg/common/const.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@ const (
EngineTest = "test"

KeyUID = "uid"
KeyProvider = "provider"
KeyEngine = "engine"
KeyTopoConfigPath = "topology_config_path"
KeyTopoConfigmapName = "topology_configmap_name"
KeyTopoConfigmapNamespace = "topology_configmap_namespace"
Expand Down
93 changes: 37 additions & 56 deletions pkg/common/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (e *HTTPError) Error() string {
}

type Provider interface {
GetCredentials(*Credentials) (interface{}, error)
GetCredentials(map[string]string) (interface{}, error)
GetComputeInstances(context.Context, Engine) ([]ComputeInstances, error)
GenerateTopologyConfig(context.Context, interface{}, int, []ComputeInstances) (*Vertex, error)
}
Expand All @@ -67,69 +67,58 @@ type Engine interface {
GenerateOutput(context.Context, *Vertex, map[string]string) ([]byte, error)
}

type Payload struct {
Nodes []ComputeInstances `json:"nodes"`
Creds *Credentials `json:"creds,omitempty"` // access credentials
type TopologyRequest struct {
Provider provider `json:"provider"`
Engine engine `json:"engine"`
Nodes []ComputeInstances `json:"nodes"`
}

type ComputeInstances struct {
Region string `json:"region"`
Instances map[string]string `json:"instances"` // <instance ID>:<node name> map
type provider struct {
Name string `json:"name"`
Creds map[string]string `json:"creds"` // access credentials
}

type Credentials struct {
AWS *AWSCredentials `yaml:"aws,omitempty" json:"aws,omitempty"` // AWS credentials
OCI *OCICredentials `yaml:"oci,omitempty" json:"oci,omitempty"` // OCI credentials
type engine struct {
Name string `json:"name"`
Params map[string]string `json:"params"` // access credentials
}

type AWSCredentials struct {
AccessKeyId string `yaml:"access_key_id" json:"access_key_id"`
SecretAccessKey string `yaml:"secret_access_key" json:"secret_access_key"`
Token string `yaml:"token,omitempty" json:"token,omitempty"` // token is optional
type ComputeInstances struct {
Region string `json:"region"`
Instances map[string]string `json:"instances"` // <instance ID>:<node name> map
}

type OCICredentials struct {
TenancyID string `yaml:"tenancy_id" json:"tenancy_id"`
UserID string `yaml:"user_id" json:"user_id"`
Region string `yaml:"region" json:"region"`
Fingerprint string `yaml:"fingerprint" json:"fingerprint"`
PrivateKey string `yaml:"private_key" json:"private_key"`
Passphrase string `yaml:"passphrase,omitempty" json:"passphrase,omitempty"` // passphrase is optional
func NewTopologyRequest(prv string, creds map[string]string, eng string, params map[string]string) *TopologyRequest {
return &TopologyRequest{
Provider: provider{
Name: prv,
Creds: creds,
},
Engine: engine{
Name: eng,
Params: params,
},
}
}

func (p *Payload) String() string {
func (p *TopologyRequest) String() string {
var sb strings.Builder

sb.WriteString(fmt.Sprintf("Payload:\n Nodes: %v\n", p.Nodes))
if p.Creds != nil {
sb.WriteString(" Credentials:\n")
if p.Creds.AWS != nil {
var accessKeyId, secretAccessKey, token string
if len(p.Creds.AWS.AccessKeyId) != 0 {
accessKeyId = "***"
}
if len(p.Creds.AWS.SecretAccessKey) != 0 {
secretAccessKey = "***"
}
if len(p.Creds.AWS.Token) != 0 {
token = "***"
}
sb.WriteString(fmt.Sprintf(" AWS: AccessKeyID=%s SecretAccessKey=%s SessionToken=%s\n",
accessKeyId, secretAccessKey, token))
}
if p.Creds.OCI != nil {
sb.WriteString(" OCI:\n")
sb.WriteString(fmt.Sprintf(" UserID=%s\n", p.Creds.OCI.UserID))
sb.WriteString(fmt.Sprintf(" TenancyID=%s\n", p.Creds.OCI.TenancyID))
sb.WriteString(fmt.Sprintf(" Region=%s\n", p.Creds.OCI.Region))
}
sb.WriteString("TopologyRequest:\n")
sb.WriteString(fmt.Sprintf(" Provider: %s\n", p.Provider.Name))
sb.WriteString(" Credentials: ")
for key := range p.Provider.Creds {
sb.WriteString(fmt.Sprintf("%s=***,", key))
}
sb.WriteString("\n")
sb.WriteString(fmt.Sprintf(" Engine: %s\n", p.Engine.Name))
sb.WriteString(fmt.Sprintf(" Parameters: %v\n", p.Engine.Params))
sb.WriteString(fmt.Sprintf(" Nodes: %s\n", p.Nodes))

return sb.String()
}

func GetPayload(body []byte) (*Payload, error) {
var payload Payload
func GetTopologyRequest(body []byte) (*TopologyRequest, error) {
var payload TopologyRequest

if len(body) == 0 {
return &payload, nil
Expand All @@ -139,13 +128,5 @@ func GetPayload(body []byte) (*Payload, error) {
return nil, fmt.Errorf("failed to parse payload: %v", err)
}

if payload.Creds != nil {
if payload.Creds.AWS != nil {
if len(payload.Creds.AWS.AccessKeyId) == 0 || len(payload.Creds.AWS.SecretAccessKey) == 0 {
return nil, fmt.Errorf("invalid payload: must provide access_key_id and secret_access_key for AWS")
}
}
}

return &payload, nil
}
88 changes: 43 additions & 45 deletions pkg/common/types_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,18 @@ func TestPayload(t *testing.T) {
testCases := []struct {
name string
input string
payload *Payload
payload *TopologyRequest
print string
err string
}{
{
name: "Case 1: no input",
payload: &Payload{},
print: `Payload:
payload: &TopologyRequest{},
print: `TopologyRequest:
Provider:
Credentials:
Engine:
Parameters: map[]
Nodes: []
`,
},
Expand All @@ -43,36 +47,26 @@ func TestPayload(t *testing.T) {
"nodes": 5
}
`,
err: "failed to parse payload: json: cannot unmarshal number into Go struct field Payload.nodes of type []common.ComputeInstances",
err: "failed to parse payload: json: cannot unmarshal number into Go struct field TopologyRequest.nodes of type []common.ComputeInstances",
},
{
name: "Case 3: invalid creds",
name: "Case 3: valid input",
input: `
{
"nodes": [
{
"region": "region1",
"instances": {
"instance1": "node1",
"instance2": "node2",
"instance3": "node3"
}
}
],
"creds": {
"aws": {
"provider": {
"name": "aws",
"creds": {
"access_key_id": "id",
"token": "token"
"secret_access_key": "secret"
}
}
}
`,
err: "invalid payload: must provide access_key_id and secret_access_key for AWS",
},
{
name: "Case 4: valid input",
input: `
{
},
"engine": {
"name": "slurm",
"params": {
"plugin": "topology/block",
"block_sizes": "30,120"
}
},
"nodes": [
{
"region": "region1",
Expand All @@ -90,16 +84,24 @@ func TestPayload(t *testing.T) {
"instance6": "node6"
}
}
],
"creds": {
"aws": {
"access_key_id": "id",
"secret_access_key": "secret"
}
}
]
}
`,
payload: &Payload{
payload: &TopologyRequest{
Provider: provider{
Name: "aws",
Creds: map[string]string{
"access_key_id": "id",
"secret_access_key": "secret",
},
},
Engine: engine{
Name: "slurm",
Params: map[string]string{
"plugin": "topology/block",
"block_sizes": "30,120",
},
},
Nodes: []ComputeInstances{
{
Region: "region1",
Expand All @@ -118,24 +120,20 @@ func TestPayload(t *testing.T) {
},
},
},
Creds: &Credentials{
AWS: &AWSCredentials{
AccessKeyId: "id",
SecretAccessKey: "secret",
},
},
},
print: `Payload:
print: `TopologyRequest:
Provider: aws
Credentials: access_key_id=***,secret_access_key=***,
Engine: slurm
Parameters: map[block_sizes:30,120 plugin:topology/block]
Nodes: [{region1 map[instance1:node1 instance2:node2 instance3:node3]} {region2 map[instance4:node4 instance5:node5 instance6:node6]}]
Credentials:
AWS: AccessKeyID=*** SecretAccessKey=*** SessionToken=
`,
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
payload, err := GetPayload([]byte(tc.input))
payload, err := GetTopologyRequest([]byte(tc.input))
if len(tc.err) != 0 {
require.EqualError(t, err, tc.err)
} else {
Expand Down
3 changes: 1 addition & 2 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
"gopkg.in/yaml.v3"
"k8s.io/klog/v2"

"github.com/NVIDIA/topograph/pkg/common"
"github.com/NVIDIA/topograph/pkg/utils"
)

Expand All @@ -39,7 +38,7 @@ type Config struct {
Env map[string]string `yaml:"env"`

// derived
Credentials common.Credentials
Credentials map[string]string
}

type Endpoint struct {
Expand Down
23 changes: 21 additions & 2 deletions pkg/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,13 @@ import (
"github.com/stretchr/testify/require"
)

var configTemplate = `
const (
credentials = `
access_key_id: id
secret_access_key: key
`

configTemplate = `
http:
port: 49021
ssl: true
Expand All @@ -34,10 +40,12 @@ ssl:
cert: %s
key: %s
ca_cert: %s
credentials_path: %s
env:
SLURM_CONF: /etc/slurm/config.yaml
PATH: /a/b/c
`
)

func TestConfig(t *testing.T) {
file, err := os.CreateTemp("", "test-cfg-*.yml")
Expand All @@ -60,7 +68,16 @@ func TestConfig(t *testing.T) {
defer func() { _ = os.Remove(caCert.Name()) }()
defer func() { _ = caCert.Close() }()

_, err = file.WriteString(fmt.Sprintf(configTemplate, cert.Name(), key.Name(), caCert.Name()))
creds, err := os.CreateTemp("", "test-creds-*.yml")
require.NoError(t, err)
defer func() { _ = os.Remove(creds.Name()) }()
defer func() { _ = creds.Close() }()
credsPath := creds.Name()

_, err = creds.WriteString(credentials)
require.NoError(t, err)

_, err = file.WriteString(fmt.Sprintf(configTemplate, cert.Name(), key.Name(), caCert.Name(), creds.Name()))
require.NoError(t, err)

cfg, err := NewFromFile(file.Name())
Expand All @@ -77,6 +94,8 @@ func TestConfig(t *testing.T) {
Key: key.Name(),
CaCert: caCert.Name(),
},
CredsPath: &credsPath,
Credentials: map[string]string{"access_key_id": "id", "secret_access_key": "key"},
Env: map[string]string{
"SLURM_CONF": "/etc/slurm/config.yaml",
"PATH": "/a/b/c",
Expand Down
2 changes: 1 addition & 1 deletion pkg/factory/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func GetTestProvider() *testProvider {
return p
}

func (p *testProvider) GetCredentials(_ *common.Credentials) (interface{}, error) {
func (p *testProvider) GetCredentials(_ map[string]string) (interface{}, error) {
return nil, nil
}

Expand Down
Loading

0 comments on commit 0edc13b

Please sign in to comment.