forked from kevinkk525/storj-log-exporter
-
Notifications
You must be signed in to change notification settings - Fork 0
/
config.yml
130 lines (124 loc) · 13.5 KB
/
config.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
global:
config_version: 3
input:
type: file
path: /app/logs/node.log
readall: false # Read from the beginning of the file? False means we start at the end of the file and read only new lines.
imports:
- type: grok_patterns
dir: /app/logstash-patterns-core/patterns
grok_patterns:
- 'MyException (\w*)Exception'
metrics:
- type: counter
name: storj_pieces_delete_successful
help: Total number of deleted pieces
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*%{LOGLEVEL}\s*piecedeleter\s*deleted\s*\{\"Satellite ID\"\: \"%{DATA:SatelliteID}\"\, \"Piece ID\"\: \"%{DATA:PieceID}\"\}'
labels:
satellite: '{{.SatelliteID}}'
url: '{{if eq .SatelliteID "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}}asia-east-1.tardigrade.io:7777{{else if eq .SatelliteID "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}}us-central-1.tardigrade.io:7777{{else if eq .SatelliteID "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}}europe-west-1.tardigrade.io:7777{{else if eq .SatelliteID "12rfG3sh9NCWiX3ivPjq2HtdLmbqCrvHVEzJubnzFzosMuawymB"}}europe-north-1.tardigrade.io:7777{{else if eq .SatelliteID "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}}saltlake.tardigrade.io:7777{{else if eq .SatelliteID "12tRQrMTWUWwzwGh18i7Fqs67kmdhH9t6aToeiwbo5mfS2rUmo"}}us2.tardigrade.io:7777{{end}}'
- type: counter
name: storj_pieces_delete_failed
help: Total number of failed deletes
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*%{LOGLEVEL}\s*piecedeleter\s*delete failed\s*\{\"Satellite ID\"\: \"%{DATA:SatelliteID}\"\, \"Piece ID\"\: \"%{DATA:PieceID}\"\}'
labels:
satellite: '{{.SatelliteID}}'
url: '{{if eq .SatelliteID "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}}asia-east-1.tardigrade.io:7777{{else if eq .SatelliteID "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}}us-central-1.tardigrade.io:7777{{else if eq .SatelliteID "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}}europe-west-1.tardigrade.io:7777{{else if eq .SatelliteID "12rfG3sh9NCWiX3ivPjq2HtdLmbqCrvHVEzJubnzFzosMuawymB"}}europe-north-1.tardigrade.io:7777{{else if eq .SatelliteID "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}}saltlake.tardigrade.io:7777{{else if eq .SatelliteID "12tRQrMTWUWwzwGh18i7Fqs67kmdhH9t6aToeiwbo5mfS2rUmo"}}us2.tardigrade.io:7777{{end}}'
- type: counter
name: storj_pieces_upload_started
help: Total number of uploads started
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*%{LOGLEVEL}\s*piecestore\s*upload started\s*\{\"Piece ID\"\: \"%{DATA:PieceID}\"\, \"Satellite ID\"\: \"%{DATA:SatelliteID}\"\, \"Action\"\: \"%{DATA:Action}\"\, \"Available Space\"\: %{INT:AvailableSpace}\}'
labels:
satellite: '{{.SatelliteID}}'
url: '{{if eq .SatelliteID "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}}asia-east-1.tardigrade.io:7777{{else if eq .SatelliteID "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}}us-central-1.tardigrade.io:7777{{else if eq .SatelliteID "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}}europe-west-1.tardigrade.io:7777{{else if eq .SatelliteID "12rfG3sh9NCWiX3ivPjq2HtdLmbqCrvHVEzJubnzFzosMuawymB"}}europe-north-1.tardigrade.io:7777{{else if eq .SatelliteID "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}}saltlake.tardigrade.io:7777{{else if eq .SatelliteID "12tRQrMTWUWwzwGh18i7Fqs67kmdhH9t6aToeiwbo5mfS2rUmo"}}us2.tardigrade.io:7777{{end}}'
type: '{{if eq .Action "PUT"}}usage{{else if eq .Action "PUT_REPAIR"}}repair{{else}}{{.Action}}{{end}}'
- type: counter
name: storj_pieces_upload_successful
help: Total number of successful uploads
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*%{LOGLEVEL}\s*piecestore\s*uploaded\s*\{\"Piece ID\"\: \"%{DATA:PieceID}\"\, \"Satellite ID\"\: \"%{DATA:SatelliteID}\"\, \"Action\"\: \"%{DATA:Action}\"\, \"Size\"\: %{INT:Size}\}'
labels:
satellite: '{{.SatelliteID}}'
url: '{{if eq .SatelliteID "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}}asia-east-1.tardigrade.io:7777{{else if eq .SatelliteID "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}}us-central-1.tardigrade.io:7777{{else if eq .SatelliteID "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}}europe-west-1.tardigrade.io:7777{{else if eq .SatelliteID "12rfG3sh9NCWiX3ivPjq2HtdLmbqCrvHVEzJubnzFzosMuawymB"}}europe-north-1.tardigrade.io:7777{{else if eq .SatelliteID "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}}saltlake.tardigrade.io:7777{{else if eq .SatelliteID "12tRQrMTWUWwzwGh18i7Fqs67kmdhH9t6aToeiwbo5mfS2rUmo"}}us2.tardigrade.io:7777{{end}}'
type: '{{if eq .Action "PUT"}}usage{{else if eq .Action "PUT_REPAIR"}}repair{{else}}{{.Action}}{{end}}'
- type: counter
name: storj_pieces_upload_canceled
help: Total number of canceled uploads
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*%{LOGLEVEL}\s*piecestore\s*upload canceled\s*\{\"Piece ID\"\: \"%{DATA:PieceID}\"\, \"Satellite ID\"\: \"%{DATA:SatelliteID}\"\, \"Action\"\: \"%{DATA:Action}\"\, \"Size\"\: %{INT:Size}\}'
labels:
satellite: '{{.SatelliteID}}'
url: '{{if eq .SatelliteID "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}}asia-east-1.tardigrade.io:7777{{else if eq .SatelliteID "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}}us-central-1.tardigrade.io:7777{{else if eq .SatelliteID "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}}europe-west-1.tardigrade.io:7777{{else if eq .SatelliteID "12rfG3sh9NCWiX3ivPjq2HtdLmbqCrvHVEzJubnzFzosMuawymB"}}europe-north-1.tardigrade.io:7777{{else if eq .SatelliteID "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}}saltlake.tardigrade.io:7777{{else if eq .SatelliteID "12tRQrMTWUWwzwGh18i7Fqs67kmdhH9t6aToeiwbo5mfS2rUmo"}}us2.tardigrade.io:7777{{end}}'
type: '{{if eq .Action "PUT"}}usage{{else if eq .Action "PUT_REPAIR"}}repair{{else}}{{.Action}}{{end}}'
- type: counter
name: storj_pieces_upload_rejected
help: Total number of rejected uploads
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*%{LOGLEVEL}\s*piecestore\s*upload rejected\s*\{\"Piece ID\"\: \"%{DATA:PieceID}\"\, \"Satellite ID\"\: \"%{DATA:SatelliteID}\"\, \"Action\"\: \"%{DATA:Action}\"(\, \"Size\"\: %{INT:Size})?\}'
labels:
satellite: '{{.SatelliteID}}'
url: '{{if eq .SatelliteID "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}}asia-east-1.tardigrade.io:7777{{else if eq .SatelliteID "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}}us-central-1.tardigrade.io:7777{{else if eq .SatelliteID "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}}europe-west-1.tardigrade.io:7777{{else if eq .SatelliteID "12rfG3sh9NCWiX3ivPjq2HtdLmbqCrvHVEzJubnzFzosMuawymB"}}europe-north-1.tardigrade.io:7777{{else if eq .SatelliteID "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}}saltlake.tardigrade.io:7777{{else if eq .SatelliteID "12tRQrMTWUWwzwGh18i7Fqs67kmdhH9t6aToeiwbo5mfS2rUmo"}}us2.tardigrade.io:7777{{end}}'
type: '{{if eq .Action "PUT"}}usage{{else if eq .Action "PUT_REPAIR"}}repair{{else}}{{.Action}}{{end}}'
- type: counter
name: storj_pieces_upload_failed
help: Total number of failed uploads
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*%{LOGLEVEL}\s*piecestore\s*upload failed\s*\{\"Piece ID\"\: \"%{DATA:PieceID}\"\, \"Satellite ID\"\: \"%{DATA:SatelliteID}\"\, \"Action\"\: \"%{DATA:Action}\"(\,\s*"error": \"%{DATA:Error}\",\s*"errorVerbose": \"%{DATA:ErrorVerbose}\")?(\, \"Size\"\: %{INT:Size})?\}'
labels:
satellite: '{{.SatelliteID}}'
url: '{{if eq .SatelliteID "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}}asia-east-1.tardigrade.io:7777{{else if eq .SatelliteID "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}}us-central-1.tardigrade.io:7777{{else if eq .SatelliteID "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}}europe-west-1.tardigrade.io:7777{{else if eq .SatelliteID "12rfG3sh9NCWiX3ivPjq2HtdLmbqCrvHVEzJubnzFzosMuawymB"}}europe-north-1.tardigrade.io:7777{{else if eq .SatelliteID "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}}saltlake.tardigrade.io:7777{{else if eq .SatelliteID "12tRQrMTWUWwzwGh18i7Fqs67kmdhH9t6aToeiwbo5mfS2rUmo"}}us2.tardigrade.io:7777{{end}}'
type: '{{if eq .Action "PUT"}}usage{{else if eq .Action "PUT_REPAIR"}}repair{{else}}{{.Action}}{{end}}'
error: '{{.Error}}'
- type: counter
name: storj_pieces_download_started
help: Total number of downloads started
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*%{LOGLEVEL}\s*piecestore\s*download started\s*\{\"Piece ID\"\: \"%{DATA:PieceID}\"\, \"Satellite ID\"\: \"%{DATA:SatelliteID}\"\, \"Action\"\: \"%{DATA:Action}\"\}'
labels:
satellite: '{{.SatelliteID}}'
url: '{{if eq .SatelliteID "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}}asia-east-1.tardigrade.io:7777{{else if eq .SatelliteID "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}}us-central-1.tardigrade.io:7777{{else if eq .SatelliteID "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}}europe-west-1.tardigrade.io:7777{{else if eq .SatelliteID "12rfG3sh9NCWiX3ivPjq2HtdLmbqCrvHVEzJubnzFzosMuawymB"}}europe-north-1.tardigrade.io:7777{{else if eq .SatelliteID "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}}saltlake.tardigrade.io:7777{{else if eq .SatelliteID "12tRQrMTWUWwzwGh18i7Fqs67kmdhH9t6aToeiwbo5mfS2rUmo"}}us2.tardigrade.io:7777{{end}}'
type: '{{if eq .Action "GET"}}usage{{else if eq .Action "GET_REPAIR"}}repair{{else if eq .Action "GET_AUDIT"}}audit{{else}}{{.Action}}{{end}}'
- type: counter
name: storj_pieces_download_successful
help: Total number of successful downloads
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*%{LOGLEVEL}\s*piecestore\s*downloaded\s*\{\"Piece ID\"\: \"%{DATA:PieceID}\"\, \"Satellite ID\"\: \"%{DATA:SatelliteID}\"\, \"Action\"\: \"%{DATA:Action}\"\}'
labels:
satellite: '{{.SatelliteID}}'
url: '{{if eq .SatelliteID "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}}asia-east-1.tardigrade.io:7777{{else if eq .SatelliteID "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}}us-central-1.tardigrade.io:7777{{else if eq .SatelliteID "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}}europe-west-1.tardigrade.io:7777{{else if eq .SatelliteID "12rfG3sh9NCWiX3ivPjq2HtdLmbqCrvHVEzJubnzFzosMuawymB"}}europe-north-1.tardigrade.io:7777{{else if eq .SatelliteID "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}}saltlake.tardigrade.io:7777{{else if eq .SatelliteID "12tRQrMTWUWwzwGh18i7Fqs67kmdhH9t6aToeiwbo5mfS2rUmo"}}us2.tardigrade.io:7777{{end}}'
type: '{{if eq .Action "GET"}}usage{{else if eq .Action "GET_REPAIR"}}repair{{else if eq .Action "GET_AUDIT"}}audit{{else}}{{.Action}}{{end}}'
- type: counter
name: storj_pieces_download_canceled
help: Total number of canceled downloads
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*%{LOGLEVEL}\s*piecestore\s*download canceled\s*\{\"Piece ID\"\: \"%{DATA:PieceID}\"\, \"Satellite ID\"\: \"%{DATA:SatelliteID}\"\, \"Action\"\: \"%{DATA:Action}\"\}'
labels:
satellite: '{{.SatelliteID}}'
url: '{{if eq .SatelliteID "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}}asia-east-1.tardigrade.io:7777{{else if eq .SatelliteID "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}}us-central-1.tardigrade.io:7777{{else if eq .SatelliteID "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}}europe-west-1.tardigrade.io:7777{{else if eq .SatelliteID "12rfG3sh9NCWiX3ivPjq2HtdLmbqCrvHVEzJubnzFzosMuawymB"}}europe-north-1.tardigrade.io:7777{{else if eq .SatelliteID "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}}saltlake.tardigrade.io:7777{{else if eq .SatelliteID "12tRQrMTWUWwzwGh18i7Fqs67kmdhH9t6aToeiwbo5mfS2rUmo"}}us2.tardigrade.io:7777{{end}}'
type: '{{if eq .Action "GET"}}usage{{else if eq .Action "GET_REPAIR"}}repair{{else if eq .Action "GET_AUDIT"}}audit{{else}}{{.Action}}{{end}}'
- type: counter
name: storj_pieces_download_failed
help: Total number of failed downloads
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*%{LOGLEVEL:LOGLEVEL}\s*piecestore\s*download failed\s*\{\"Piece ID\"\: \"%{DATA:PieceID}\"\, \"Satellite ID\"\: \"%{DATA:SatelliteID}\"\, \"Action\"\: \"%{DATA:Action}\"(\,\s*"error": \"%{DATA:Error}\",\s*"errorVerbose": \"%{DATA:ErrorVerbose}\")?\}'
labels:
satellite: '{{.SatelliteID}}'
url: '{{if eq .SatelliteID "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6"}}asia-east-1.tardigrade.io:7777{{else if eq .SatelliteID "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S"}}us-central-1.tardigrade.io:7777{{else if eq .SatelliteID "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs"}}europe-west-1.tardigrade.io:7777{{else if eq .SatelliteID "12rfG3sh9NCWiX3ivPjq2HtdLmbqCrvHVEzJubnzFzosMuawymB"}}europe-north-1.tardigrade.io:7777{{else if eq .SatelliteID "1wFTAgs9DP5RSnCqKV1eLf6N9wtk4EAtmN5DpSxcs8EjT69tGE"}}saltlake.tardigrade.io:7777{{else if eq .SatelliteID "12tRQrMTWUWwzwGh18i7Fqs67kmdhH9t6aToeiwbo5mfS2rUmo"}}us2.tardigrade.io:7777{{end}}'
type: '{{if eq .Action "GET"}}usage{{else if eq .Action "GET_REPAIR"}}repair{{else if eq .Action "GET_AUDIT"}}audit{{else}}{{.Action}}{{end}}'
error: '{{if eq .Action "GET_AUDIT"}}.Error{{else}}use of closed network connection{{end}}' # workaround for all unique write tcp error messages..
- type: counter
name: storj_log_levels
help: Total number of loglevel messages in log
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*%{LOGLEVEL:LOGLEVEL}\s*%{DATA}\}'
labels:
loglevel: '{{.LOGLEVEL}}'
- type: counter
name: storj_log_errors
help: Total number of error messages in log and their type
match: '%{TIMESTAMP_ISO8601:UNWANTED}\s*ERROR\s*%{DATA:TYPE}\s*\{%{DATA}\}'
labels:
loglevel: 'ERROR'
type: '{{.TYPE}}'
- type: counter
name: my_exception
help: Total number of my-exception messages in log and their type
match: '%{MyException:myexception}\s*ERROR\s*%{DATA:TYPE}\s*\{%{DATA}\}'
labels:
loglevel: 'ERROR'
myexception: '{{.myexception}}'
server:
protocol: http
port: 9144
path: /