diff --git a/README.md b/README.md index f4b28c0..155ff59 100644 --- a/README.md +++ b/README.md @@ -25,58 +25,65 @@ Environment variable `AWS_REGION` is required. Run a task successfully and shutdown. ```console -$ tracer default 9f654c76cde14c7c85cf54dce087658a -2021-11-29T10:42:33.521+09:00 TASK Created -2021-11-29T10:42:37.103+09:00 TASK Connected -2021-11-29T10:42:48.298+09:00 TASK Pull started -2021-11-29T10:42:55.473+09:00 TASK Pull stopped -2021-11-29T10:42:56.374+09:00 CONTAINER:nginx /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration -2021-11-29T10:42:56.374+09:00 CONTAINER:nginx /docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/ -2021-11-29T10:42:56.375+09:00 CONTAINER:nginx /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh -2021-11-29T10:42:56.380+09:00 CONTAINER:nginx 10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf -2021-11-29T10:42:56.418+09:00 TASK Started -2021-11-29T10:42:56.446+09:00 CONTAINER:nginx 10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf -2021-11-29T10:42:56.446+09:00 CONTAINER:nginx /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh -2021-11-29T10:42:56.448+09:00 CONTAINER:nginx /docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh -2021-11-29T10:42:56.449+09:00 CONTAINER:nginx /docker-entrypoint.sh: Configuration complete; ready for start up -2021-11-29T10:42:56.452+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: using the "epoll" event method -2021-11-29T10:42:56.452+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: nginx/1.21.4 -2021-11-29T10:42:56.452+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: built by gcc 10.2.1 20210110 (Debian 10.2.1-6) -2021-11-29T10:42:56.452+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: OS: Linux 4.14.248-189.473.amzn2.aarch64 -2021-11-29T10:42:56.452+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1024:4096 -2021-11-29T10:42:56.452+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: start worker processes -2021-11-29T10:42:56.453+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: start worker process 31 -2021-11-29T10:42:56.453+09:00 CONTAINER:nginx 2021/11/29 01:42:56 [notice] 1#1: start worker process 32 -2021-11-29T10:43:12.913+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:43:12 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" -2021-11-29T10:43:42.898+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:43:42 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" -2021-11-29T10:44:09.172+09:00 CONTAINER:nginx 10.3.1.178 - - [29/Nov/2021:01:44:09 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" -2021-11-29T10:44:12.929+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:44:12 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" -2021-11-29T10:44:39.197+09:00 CONTAINER:nginx 10.3.1.178 - - [29/Nov/2021:01:44:39 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" -2021-11-29T10:44:42.960+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:44:42 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" -2021-11-29T10:45:09.227+09:00 CONTAINER:nginx 10.3.1.178 - - [29/Nov/2021:01:45:09 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" -2021-11-29T10:45:12.966+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:45:12 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" -2021-11-29T10:45:39.237+09:00 CONTAINER:nginx 10.3.1.178 - - [29/Nov/2021:01:45:39 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" -2021-11-29T10:45:42.973+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:45:42 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" -2021-11-29T10:46:09.225+09:00 CONTAINER:nginx 10.3.1.178 - - [29/Nov/2021:01:46:09 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" -2021-11-29T10:46:13.004+09:00 CONTAINER:nginx 10.3.3.10 - - [29/Nov/2021:01:46:13 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" -2021-11-29T10:46:30.906+09:00 TASK Stopping -2021-11-29T10:46:30.906+09:00 TASK StoppedReason:Scaling activity initiated by (deployment ecs-svc/8709920613704280865) -2021-11-29T10:46:30.906+09:00 TASK StoppedCode:ServiceSchedulerInitiated -2021-11-29T10:46:31.937+09:00 CONTAINER:nginx 2021/11/29 01:46:31 [notice] 1#1: signal 15 (SIGTERM) received, exiting -2021-11-29T10:46:31.938+09:00 CONTAINER:nginx 2021/11/29 01:46:31 [notice] 32#32: exiting -2021-11-29T10:46:31.938+09:00 CONTAINER:nginx 2021/11/29 01:46:31 [notice] 32#32: exit -2021-11-29T10:46:31.938+09:00 CONTAINER:nginx 2021/11/29 01:46:31 [notice] 31#31: exiting -2021-11-29T10:46:31.938+09:00 CONTAINER:nginx 2021/11/29 01:46:31 [notice] 31#31: exit -2021-11-29T10:46:31.988+09:00 CONTAINER:nginx 2021/11/29 01:46:31 [notice] 1#1: signal 14 (SIGALRM) received -2021-11-29T10:46:32.035+09:00 CONTAINER:nginx 2021/11/29 01:46:32 [notice] 1#1: signal 17 (SIGCHLD) received from 32 -2021-11-29T10:46:32.035+09:00 CONTAINER:nginx 2021/11/29 01:46:32 [notice] 1#1: worker process 32 exited with code 0 -2021-11-29T10:46:32.035+09:00 CONTAINER:nginx 2021/11/29 01:46:32 [notice] 1#1: signal 29 (SIGIO) received -2021-11-29T10:46:32.035+09:00 CONTAINER:nginx 2021/11/29 01:46:32 [notice] 1#1: signal 17 (SIGCHLD) received from 31 -2021-11-29T10:46:32.035+09:00 CONTAINER:nginx 2021/11/29 01:46:32 [notice] 1#1: worker process 31 exited with code 0 -2021-11-29T10:46:32.035+09:00 CONTAINER:nginx 2021/11/29 01:46:32 [notice] 1#1: exit -2021-11-29T10:46:43.234+09:00 TASK Execution stopped -2021-11-29T10:47:06.292+09:00 TASK Stopped -2021-11-29T10:47:06.292+09:00 CONTAINER:nginx STOPPED (exit code: 0) +$ tracer default 834a5628bef14f2dbb81c7bc0b272160 +2021-12-03T11:06:21.633+09:00 TASK Created +2021-12-03T11:06:21.664+09:00 SERVICE (service nginx-local) has started 1 tasks: (task 834a5628bef14f2dbb81c7bc0b272160). +2021-12-03T11:06:22.342+09:00 SERVICE (service nginx-local) was unable to place a task. Reason: Capacity is unavailable at this time. Please try again later or in a different availability zone. For more information, see the Troubleshooting section of the Amazon ECS Developer Guide. +2021-12-03T11:06:24.906+09:00 TASK Connected +2021-12-03T11:06:39.602+09:00 TASK Pull started +2021-12-03T11:06:46.366+09:00 TASK Pull stopped +2021-12-03T11:06:46.746+09:00 CONTAINER:nginx /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration +2021-12-03T11:06:46.746+09:00 CONTAINER:nginx /docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/ +2021-12-03T11:06:46.746+09:00 CONTAINER:nginx /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh +2021-12-03T11:06:46.758+09:00 CONTAINER:nginx 10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf +2021-12-03T11:06:46.762+09:00 CONTAINER:nginx 10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf +2021-12-03T11:06:46.762+09:00 CONTAINER:nginx /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh +2021-12-03T11:06:46.768+09:00 TASK Started +2021-12-03T11:06:46.820+09:00 CONTAINER:nginx /docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh +2021-12-03T11:06:46.832+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: using the "epoll" event method +2021-12-03T11:06:46.832+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: nginx/1.21.4 +2021-12-03T11:06:46.832+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: built by gcc 10.2.1 20210110 (Debian 10.2.1-6) +2021-12-03T11:06:46.832+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: OS: Linux 4.14.248-189.473.amzn2.aarch64 +2021-12-03T11:06:46.832+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1024:4096 +2021-12-03T11:06:46.832+09:00 CONTAINER:nginx /docker-entrypoint.sh: Configuration complete; ready for start up +2021-12-03T11:06:46.832+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: start worker processes +2021-12-03T11:06:46.837+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: start worker process 37 +2021-12-03T11:06:46.837+09:00 CONTAINER:nginx 2021/12/03 02:06:46 [notice] 1#1: start worker process 38 +2021-12-03T11:21:36.818+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:21:36 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" +2021-12-03T11:21:36.836+09:00 CONTAINER:nginx 10.3.3.10 - - [03/Dec/2021:02:21:36 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" +2021-12-03T11:21:46.819+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:21:46 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" +2021-12-03T11:21:46.837+09:00 CONTAINER:nginx 10.3.3.10 - - [03/Dec/2021:02:21:46 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" +2021-12-03T11:21:56.820+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:21:56 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" +2021-12-03T11:21:56.839+09:00 CONTAINER:nginx 10.3.3.10 - - [03/Dec/2021:02:21:56 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" +2021-12-03T11:22:06.821+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:22:06 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" +2021-12-03T11:22:06.840+09:00 CONTAINER:nginx 10.3.3.10 - - [03/Dec/2021:02:22:06 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" +2021-12-03T11:22:12.681+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:22:12 +0000] "GET / HTTP/1.1" 200 615 "-" "Mozilla/5.0 (compatible; Nimbostratus-Bot/v1.3.2; http://cloudsystemnetworks.com)" "209.17.96.194" +2021-12-03T11:22:16.821+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:22:16 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" +2021-12-03T11:22:16.841+09:00 CONTAINER:nginx 10.3.3.10 - - [03/Dec/2021:02:22:16 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" +2021-12-03T11:22:19.833+09:00 SERVICE (service nginx-local) deregistered 1 targets in (target-group arn:aws:elasticloadbalancing:ap-northeast-1:314472643515:targetgroup/alpha/6a301850702273d9) +2021-12-03T11:22:19.834+09:00 SERVICE (service nginx-local) has begun draining connections on 1 tasks. +2021-12-03T11:22:26.822+09:00 CONTAINER:nginx 10.3.1.18 - - [03/Dec/2021:02:22:26 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" +2021-12-03T11:22:26.842+09:00 CONTAINER:nginx 10.3.3.10 - - [03/Dec/2021:02:22:26 +0000] "GET / HTTP/1.1" 200 615 "-" "ELB-HealthChecker/2.0" "-" +2021-12-03T11:22:28.910+09:00 TASK Stopping +2021-12-03T11:22:28.910+09:00 TASK StoppedReason:Scaling activity initiated by (deployment ecs-svc/8709920613704280865) +2021-12-03T11:22:28.910+09:00 TASK StoppedCode:ServiceSchedulerInitiated +2021-12-03T11:22:28.938+09:00 SERVICE (service nginx-local) has stopped 1 running tasks: (task 834a5628bef14f2dbb81c7bc0b272160). +2021-12-03T11:22:29.244+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: signal 15 (SIGTERM) received, exiting +2021-12-03T11:22:29.245+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 37#37: exiting +2021-12-03T11:22:29.245+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 37#37: exit +2021-12-03T11:22:29.245+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 38#38: exiting +2021-12-03T11:22:29.245+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 38#38: exit +2021-12-03T11:22:29.294+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: signal 14 (SIGALRM) received +2021-12-03T11:22:29.328+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: signal 17 (SIGCHLD) received from 37 +2021-12-03T11:22:29.328+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: worker process 37 exited with code 0 +2021-12-03T11:22:29.328+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: signal 29 (SIGIO) received +2021-12-03T11:22:29.329+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: signal 17 (SIGCHLD) received from 38 +2021-12-03T11:22:29.329+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: worker process 38 exited with code 0 +2021-12-03T11:22:29.329+09:00 CONTAINER:nginx 2021/12/03 02:22:29 [notice] 1#1: exit +2021-12-03T11:22:38.224+09:00 SERVICE (service nginx-local) has reached a steady state. +2021-12-03T11:22:40.527+09:00 TASK Execution stopped +2021-12-03T11:23:04.873+09:00 TASK Stopped +2021-12-03T11:23:04.873+09:00 CONTAINER:nginx STOPPED (exit code: 0) ``` Failed to run task. (typo container image URL) diff --git a/cmd/tracer/main.go b/cmd/tracer/main.go index d8a61c3..76b8449 100644 --- a/cmd/tracer/main.go +++ b/cmd/tracer/main.go @@ -29,9 +29,12 @@ func main() { flag.Parse() args := flag.Args() - if len(args) < 2 { + if len(args) < 1 { usage() } + if len(args) == 1 { + args = append(args, "") + } if err := t.Run(args[0], args[1]); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) diff --git a/tracer.go b/tracer.go index be08e66..b59dfd8 100644 --- a/tracer.go +++ b/tracer.go @@ -30,6 +30,7 @@ type Tracer struct { timeline *Timeline Duration time.Duration + now time.Time headBegin time.Time headEnd time.Time tailBegin time.Time @@ -111,6 +112,12 @@ func newEvent(ts *time.Time, src, msg string) *TimeLineEvent { } func (t *Tracer) Run(cluster string, taskID string) error { + t.now = time.Now() + + if taskID == "" { + return t.listAllTasks(cluster) + } + task, err := t.traceTask(cluster, taskID) if err != nil { return err @@ -135,14 +142,7 @@ func (t *Tracer) traceTask(cluster string, taskID string) (*ecs.Task, error) { } task := res.Tasks[0] - t.headBegin = task.CreatedAt.Add(-t.Duration) - t.headEnd = task.CreatedAt.Add(t.Duration) - if task.StoppingAt != nil { - t.tailBegin = task.StoppingAt.Add(-t.Duration) - } else { - t.tailBegin = time.Now().Add(-t.Duration) - } - t.tailEnd = time.Now() + t.setBoundaries(task) taskGroup := strings.SplitN(aws.StringValue(task.Group), ":", 2) if len(taskGroup) == 2 && taskGroup[0] == "service" { @@ -175,7 +175,7 @@ func (t *Tracer) traceTask(cluster string, taskID string) (*ecs.Task, error) { } var ts *time.Time if aws.StringValue(c.LastStatus) == "RUNNING" { - ts = now() + ts = &t.now } else { ts = task.StoppedAt } @@ -218,11 +218,6 @@ func (t *Tracer) traceLogs(task *ecs.Task) error { return nil } -func now() *time.Time { - now := time.Now() - return &now -} - func taskID(task *ecs.Task) string { an := aws.StringValue(task.TaskArn) return an[strings.LastIndex(an, "/")+1:] @@ -290,6 +285,67 @@ func (t *Tracer) fetchLogs(containerName, group, stream string, from, to *time.T return nil } +func (t *Tracer) listAllTasks(cluster string) error { + for _, s := range []string{"RUNNING", "PENDING", "STOPPED"} { + err := t.listTasks(cluster, s) + if err != nil { + return err + } + } + return nil +} + +func (t *Tracer) listTasks(cluster, status string) error { + var nextToken *string + for { + listRes, err := t.ecs.ListTasksWithContext(t.ctx, &ecs.ListTasksInput{ + Cluster: &cluster, + DesiredStatus: aws.String(status), + NextToken: nextToken, + }) + if err != nil { + return errors.Wrap(err, "failed to list tasks") + } + if len(listRes.TaskArns) == 0 { + break + } + res, err := t.ecs.DescribeTasksWithContext(t.ctx, &ecs.DescribeTasksInput{ + Cluster: &cluster, + Tasks: listRes.TaskArns, + }) + if err != nil { + return errors.Wrap(err, "failed to describe tasks") + } + for _, task := range res.Tasks { + fmt.Println(strings.Join(taskToColumns(task), "\t")) + } + if nextToken = listRes.NextToken; nextToken == nil { + break + } + } + return nil +} + +func (t *Tracer) setBoundaries(task *ecs.Task) { + t.headBegin = task.CreatedAt.Add(-t.Duration) + if task.StartedAt != nil { + t.headEnd = task.StartedAt.Add(t.Duration) + } else { + t.headEnd = task.CreatedAt.Add(t.Duration) + } + + if task.StoppingAt != nil { + t.tailBegin = task.StoppingAt.Add(-t.Duration) + } else { + t.tailBegin = t.now.Add(-t.Duration) + } + if task.StoppedAt != nil { + t.tailEnd = task.StoppedAt.Add(t.Duration) + } else { + t.tailEnd = t.now + } +} + func msecToTime(i int64) time.Time { return epochBase.Add(time.Duration(i) * time.Millisecond) } @@ -297,3 +353,20 @@ func msecToTime(i int64) time.Time { func timeToInt64msec(t time.Time) int64 { return int64(t.Sub(epochBase) / time.Millisecond) } + +func arnToName(arn string) string { + return arn[strings.LastIndex(arn, "/")+1:] +} + +func taskToColumns(task *ecs.Task) []string { + return []string{ + arnToName(*task.TaskArn), + arnToName(*task.TaskDefinitionArn), + arnToName(aws.StringValue(task.ContainerInstanceArn)), + aws.StringValue(task.LastStatus), + aws.StringValue(task.DesiredStatus), + task.CreatedAt.In(time.Local).Format(time.RFC3339), + aws.StringValue(task.Group), + aws.StringValue(task.LaunchType), + } +}