Skip to content

Commit

Permalink
shows all tasks on cluster.
Browse files Browse the repository at this point in the history
  • Loading branch information
@fujiwara
fujiwara committed Dec 3, 2021
1 parent 19e8c70 commit 75973ff
Show file tree
Hide file tree
Showing 3 changed files with 150 additions and 67 deletions.
111 changes: 59 additions & 52 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,58 +25,65 @@ Environment variable `AWS_REGION` is required.
Run a task successfully and shutdown.

```console
$ tracer default 9f654c76cde14c7c85cf54dce087658a
2021-11-29T10:42:33.521+09:00 TASK Created
2021-11-29T10:42:37.103+09:00 TASK Connected
2021-11-29T10:42:48.298+09:00 TASK Pull started
2021-11-29T10:42:55.473+09:00 TASK Pull stopped
2021-11-29T10:42:56.374+09:00 CONTAINER:nginx /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
2021-11-29T10:42:56.374+09:00 CONTAINER:nginx /docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
2021-11-29T10:42:56.375+09:00 CONTAINER:nginx /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
2021-11-29T10:42:56.380+09:00 CONTAINER:nginx 10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf
2021-11-29T10:42:56.418+09:00 TASK Started
2021-11-29T10:42:56.446+09:00 CONTAINER:nginx 10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
2021-11-29T10:42:56.446+09:00 CONTAINER:nginx /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
2021-11-29T10:42:56.448+09:00 CONTAINER:nginx /docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
2021-11-29T10:42:56.449+09:00 CONTAINER:nginx /docker-entrypoint.sh: Configuration complete; ready for start up
2021-11-29T10:42:56.452+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: using the "epoll" event method
2021-11-29T10:42:56.452+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: nginx/1.21.4
2021-11-29T10:42:56.452+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: built by gcc 10.2.1 20210110 (Debian 10.2.1-6)
2021-11-29T10:42:56.452+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: OS: Linux 4.14.248-189.473.amzn2.aarch64
2021-11-29T10:42:56.452+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1024:4096
2021-11-29T10:42:56.452+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: start worker processes
2021-11-29T10:42:56.453+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: start worker process 31
2021-11-29T10:42:56.453+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: start worker process 32
2021-11-29T10:43:12.913+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:43:12 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-11-29T10:43:42.898+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:43:42 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-11-29T10:44:09.172+09:00 CONTAINER:nginx 10.3.1.178 - - [29/Nov/2021:01:44:09 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-11-29T10:44:12.929+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:44:12 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-11-29T10:44:39.197+09:00 CONTAINER:nginx 10.3.1.178 - - [29/Nov/2021:01:44:39 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-11-29T10:44:42.960+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:44:42 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-11-29T10:45:09.227+09:00 CONTAINER:nginx 10.3.1.178 - - [29/Nov/2021:01:45:09 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-11-29T10:45:12.966+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:45:12 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-11-29T10:45:39.237+09:00 CONTAINER:nginx 10.3.1.178 - - [29/Nov/2021:01:45:39 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-11-29T10:45:42.973+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:45:42 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-11-29T10:46:09.225+09:00 CONTAINER:nginx 10.3.1.178 - - [29/Nov/2021:01:46:09 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-11-29T10:46:13.004+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:46:13 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-11-29T10:46:30.906+09:00 TASK Stopping
2021-11-29T10:46:30.906+09:00 TASK StoppedReason:Scaling activity initiated by (deployment ecs-svc/8709920613704280865)
2021-11-29T10:46:30.906+09:00 TASK StoppedCode:ServiceSchedulerInitiated
2021-11-29T10:46:31.937+09:00 CONTAINER:nginx 2021/11/29 01:46:31 [notice] 1#1: signal 15 (SIGTERM) received, exiting
2021-11-29T10:46:31.938+09:00 CONTAINER:nginx 2021/11/29 01:46:31 [notice] 32#32: exiting
2021-11-29T10:46:31.938+09:00 CONTAINER:nginx 2021/11/29 01:46:31 [notice] 32#32: exit
2021-11-29T10:46:31.938+09:00 CONTAINER:nginx 2021/11/29 01:46:31 [notice] 31#31: exiting
2021-11-29T10:46:31.938+09:00 CONTAINER:nginx 2021/11/29 01:46:31 [notice] 31#31: exit
2021-11-29T10:46:31.988+09:00 CONTAINER:nginx 2021/11/29 01:46:31 [notice] 1#1: signal 14 (SIGALRM) received
2021-11-29T10:46:32.035+09:00 CONTAINER:nginx 2021/11/29 01:46:32 [notice] 1#1: signal 17 (SIGCHLD) received from 32
2021-11-29T10:46:32.035+09:00 CONTAINER:nginx 2021/11/29 01:46:32 [notice] 1#1: worker process 32 exited with code 0
2021-11-29T10:46:32.035+09:00 CONTAINER:nginx 2021/11/29 01:46:32 [notice] 1#1: signal 29 (SIGIO) received
2021-11-29T10:46:32.035+09:00 CONTAINER:nginx 2021/11/29 01:46:32 [notice] 1#1: signal 17 (SIGCHLD) received from 31
2021-11-29T10:46:32.035+09:00 CONTAINER:nginx 2021/11/29 01:46:32 [notice] 1#1: worker process 31 exited with code 0
2021-11-29T10:46:32.035+09:00 CONTAINER:nginx 2021/11/29 01:46:32 [notice] 1#1: exit
2021-11-29T10:46:43.234+09:00 TASK Execution stopped
2021-11-29T10:47:06.292+09:00 TASK Stopped
2021-11-29T10:47:06.292+09:00 CONTAINER:nginx STOPPED (exit code: 0)
$ tracer default 834a5628bef14f2dbb81c7bc0b272160
2021-12-03T11:06:21.633+09:00 TASK Created
2021-12-03T11:06:21.664+09:00 SERVICE (service nginx-local) has started 1 tasks: (task 834a5628bef14f2dbb81c7bc0b272160).
2021-12-03T11:06:22.342+09:00 SERVICE (service nginx-local) was unable to place a task. Reason: Capacity is unavailable at this time. Please try again later or in a different availability zone. For more information, see the Troubleshooting section of the Amazon ECS Developer Guide.
2021-12-03T11:06:24.906+09:00 TASK Connected
2021-12-03T11:06:39.602+09:00 TASK Pull started
2021-12-03T11:06:46.366+09:00 TASK Pull stopped
2021-12-03T11:06:46.746+09:00 CONTAINER:nginx /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
2021-12-03T11:06:46.746+09:00 CONTAINER:nginx /docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
2021-12-03T11:06:46.746+09:00 CONTAINER:nginx /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
2021-12-03T11:06:46.758+09:00 CONTAINER:nginx 10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf
2021-12-03T11:06:46.762+09:00 CONTAINER:nginx 10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
2021-12-03T11:06:46.762+09:00 CONTAINER:nginx /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
2021-12-03T11:06:46.768+09:00 TASK Started
2021-12-03T11:06:46.820+09:00 CONTAINER:nginx /docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
2021-12-03T11:06:46.832+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: using the "epoll" event method
2021-12-03T11:06:46.832+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: nginx/1.21.4
2021-12-03T11:06:46.832+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: built by gcc 10.2.1 20210110 (Debian 10.2.1-6)
2021-12-03T11:06:46.832+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: OS: Linux 4.14.248-189.473.amzn2.aarch64
2021-12-03T11:06:46.832+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1024:4096
2021-12-03T11:06:46.832+09:00 CONTAINER:nginx /docker-entrypoint.sh: Configuration complete; ready for start up
2021-12-03T11:06:46.832+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: start worker processes
2021-12-03T11:06:46.837+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: start worker process 37
2021-12-03T11:06:46.837+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: start worker process 38
2021-12-03T11:21:36.818+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:21:36 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-12-03T11:21:36.836+09:00 CONTAINER:nginx 10.3.3.10 - - [03/Dec/2021:02:21:36 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-12-03T11:21:46.819+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:21:46 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-12-03T11:21:46.837+09:00 CONTAINER:nginx 10.3.3.10 - - [03/Dec/2021:02:21:46 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-12-03T11:21:56.820+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:21:56 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-12-03T11:21:56.839+09:00 CONTAINER:nginx 10.3.3.10 - - [03/Dec/2021:02:21:56 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-12-03T11:22:06.821+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:22:06 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-12-03T11:22:06.840+09:00 CONTAINER:nginx 10.3.3.10 - - [03/Dec/2021:02:22:06 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-12-03T11:22:12.681+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:22:12 +0000] "GET / HTTP/1.1" 200 615 "-" "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)" "209.17.96.194"
2021-12-03T11:22:16.821+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:22:16 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-12-03T11:22:16.841+09:00 CONTAINER:nginx 10.3.3.10 - - [03/Dec/2021:02:22:16 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-12-03T11:22:19.833+09:00 SERVICE (service nginx-local) deregistered 1 targets in (target-group arn:aws:elasticloadbalancing:ap-northeast-1:314472643515:targetgroup/alpha/6a301850702273d9)
2021-12-03T11:22:19.834+09:00 SERVICE (service nginx-local) has begun draining connections on 1 tasks.
2021-12-03T11:22:26.822+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:22:26 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-12-03T11:22:26.842+09:00 CONTAINER:nginx 10.3.3.10 - - [03/Dec/2021:02:22:26 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-"
2021-12-03T11:22:28.910+09:00 TASK Stopping
2021-12-03T11:22:28.910+09:00 TASK StoppedReason:Scaling activity initiated by (deployment ecs-svc/8709920613704280865)
2021-12-03T11:22:28.910+09:00 TASK StoppedCode:ServiceSchedulerInitiated
2021-12-03T11:22:28.938+09:00 SERVICE (service nginx-local) has stopped 1 running tasks: (task 834a5628bef14f2dbb81c7bc0b272160).
2021-12-03T11:22:29.244+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: signal 15 (SIGTERM) received, exiting
2021-12-03T11:22:29.245+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 37#37: exiting
2021-12-03T11:22:29.245+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 37#37: exit
2021-12-03T11:22:29.245+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 38#38: exiting
2021-12-03T11:22:29.245+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 38#38: exit
2021-12-03T11:22:29.294+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: signal 14 (SIGALRM) received
2021-12-03T11:22:29.328+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: signal 17 (SIGCHLD) received from 37
2021-12-03T11:22:29.328+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: worker process 37 exited with code 0
2021-12-03T11:22:29.328+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: signal 29 (SIGIO) received
2021-12-03T11:22:29.329+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: signal 17 (SIGCHLD) received from 38
2021-12-03T11:22:29.329+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: worker process 38 exited with code 0
2021-12-03T11:22:29.329+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: exit
2021-12-03T11:22:38.224+09:00 SERVICE (service nginx-local) has reached a steady state.
2021-12-03T11:22:40.527+09:00 TASK Execution stopped
2021-12-03T11:23:04.873+09:00 TASK Stopped
2021-12-03T11:23:04.873+09:00 CONTAINER:nginx STOPPED (exit code: 0)
```

Failed to run task. (typo container image URL)
Expand Down
5 changes: 4 additions & 1 deletion cmd/tracer/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,12 @@ func main() {
flag.Parse()

args := flag.Args()
if len(args) < 2 {
if len(args) < 1 {
usage()
}
if len(args) == 1 {
args = append(args, "")
}
if err := t.Run(args[0], args[1]); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
Expand Down
101 changes: 87 additions & 14 deletions tracer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ type Tracer struct {
timeline *Timeline
Duration time.Duration

now time.Time
headBegin time.Time
headEnd time.Time
tailBegin time.Time
Expand Down Expand Up @@ -111,6 +112,12 @@ func newEvent(ts *time.Time, src, msg string) *TimeLineEvent {
}

func (t *Tracer) Run(cluster string, taskID string) error {
t.now = time.Now()

if taskID == "" {
return t.listAllTasks(cluster)
}

task, err := t.traceTask(cluster, taskID)
if err != nil {
return err
Expand All @@ -135,14 +142,7 @@ func (t *Tracer) traceTask(cluster string, taskID string) (*ecs.Task, error) {
}
task := res.Tasks[0]

t.headBegin = task.CreatedAt.Add(-t.Duration)
t.headEnd = task.CreatedAt.Add(t.Duration)
if task.StoppingAt != nil {
t.tailBegin = task.StoppingAt.Add(-t.Duration)
} else {
t.tailBegin = time.Now().Add(-t.Duration)
}
t.tailEnd = time.Now()
t.setBoundaries(task)

taskGroup := strings.SplitN(aws.StringValue(task.Group), ":", 2)
if len(taskGroup) == 2 && taskGroup[0] == "service" {
Expand Down Expand Up @@ -175,7 +175,7 @@ func (t *Tracer) traceTask(cluster string, taskID string) (*ecs.Task, error) {
}
var ts *time.Time
if aws.StringValue(c.LastStatus) == "RUNNING" {
ts = now()
ts = &t.now
} else {
ts = task.StoppedAt
}
Expand Down Expand Up @@ -218,11 +218,6 @@ func (t *Tracer) traceLogs(task *ecs.Task) error {
return nil
}

func now() *time.Time {
now := time.Now()
return &now
}

func taskID(task *ecs.Task) string {
an := aws.StringValue(task.TaskArn)
return an[strings.LastIndex(an, "/")+1:]
Expand Down Expand Up @@ -290,10 +285,88 @@ func (t *Tracer) fetchLogs(containerName, group, stream string, from, to *time.T
return nil
}

func (t *Tracer) listAllTasks(cluster string) error {
for _, s := range []string{"RUNNING", "PENDING", "STOPPED"} {
err := t.listTasks(cluster, s)
if err != nil {
return err
}
}
return nil
}

func (t *Tracer) listTasks(cluster, status string) error {
var nextToken *string
for {
listRes, err := t.ecs.ListTasksWithContext(t.ctx, &ecs.ListTasksInput{
Cluster: &cluster,
DesiredStatus: aws.String(status),
NextToken: nextToken,
})
if err != nil {
return errors.Wrap(err, "failed to list tasks")
}
if len(listRes.TaskArns) == 0 {
break
}
res, err := t.ecs.DescribeTasksWithContext(t.ctx, &ecs.DescribeTasksInput{
Cluster: &cluster,
Tasks: listRes.TaskArns,
})
if err != nil {
return errors.Wrap(err, "failed to describe tasks")
}
for _, task := range res.Tasks {
fmt.Println(strings.Join(taskToColumns(task), "\t"))
}
if nextToken = listRes.NextToken; nextToken == nil {
break
}
}
return nil
}

func (t *Tracer) setBoundaries(task *ecs.Task) {
t.headBegin = task.CreatedAt.Add(-t.Duration)
if task.StartedAt != nil {
t.headEnd = task.StartedAt.Add(t.Duration)
} else {
t.headEnd = task.CreatedAt.Add(t.Duration)
}

if task.StoppingAt != nil {
t.tailBegin = task.StoppingAt.Add(-t.Duration)
} else {
t.tailBegin = t.now.Add(-t.Duration)
}
if task.StoppedAt != nil {
t.tailEnd = task.StoppedAt.Add(t.Duration)
} else {
t.tailEnd = t.now
}
}

func msecToTime(i int64) time.Time {
return epochBase.Add(time.Duration(i) * time.Millisecond)
}

func timeToInt64msec(t time.Time) int64 {
return int64(t.Sub(epochBase) / time.Millisecond)
}

func arnToName(arn string) string {
return arn[strings.LastIndex(arn, "/")+1:]
}

func taskToColumns(task *ecs.Task) []string {
return []string{
arnToName(*task.TaskArn),
arnToName(*task.TaskDefinitionArn),
arnToName(aws.StringValue(task.ContainerInstanceArn)),
aws.StringValue(task.LastStatus),
aws.StringValue(task.DesiredStatus),
task.CreatedAt.In(time.Local).Format(time.RFC3339),
aws.StringValue(task.Group),
aws.StringValue(task.LaunchType),
}
}

0 comments on commit 75973ff

Please sign in to comment.