-
Notifications
You must be signed in to change notification settings - Fork 2
/
steps for docker.txt
346 lines (293 loc) · 10.9 KB
/
steps for docker.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
1) Install docker
Follow these steps https://docs.docker.com/install/linux/docker-ce/debian/
2) Install and run ElasticSearch
https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html
$ sudo sysctl -w vm.max_map_count=262144
Note: this fixes Elasticsearch error "[1]: max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]"
$ docker pull docker.elastic.co/elasticsearch/elasticsearch:6.4.3
There is no need to open the TCP ports as the other containers will use the docker bridge interface to contact Elasticsearch.
$ docker run -d --name elasticsearch -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:6.4.3
(add -d to run the image in detached mode.)
$ docker run curl localhost:9200
{
"name" : "-RAvpJ5",
"cluster_name" : "docker-cluster",
"cluster_uuid" : "0DZTV2FDSzyIUUDBbxuGCw",
"version" : {
"number" : "6.4.3",
"build_flavor" : "default",
"build_type" : "tar",
"build_hash" : "04711c2",
"build_date" : "2018-09-26T13:34:09.098244Z",
"build_snapshot" : false,
"lucene_version" : "7.4.0",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}
3) Install Kibana
https://www.elastic.co/guide/en/kibana/current/docker.html
$ sudo docker pull docker.elastic.co/kibana/kibana:6.4.3
https://docs.docker.com/samples/library/kibana/#running-in-development-mode
$ docker run -d --name kibana --link "elasticsearch:elasticsearch" -p 5601:5601 docker.elastic.co/kibana/kibana:6.4.3
$ sudo docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
776e51941216 docker.elastic.co/elasticsearch/elasticsearch:6.4.3 "/usr/local/bin/dock…" 5 seconds ago Up 4 seconds 9200/tcp, 9300/tcp elasticsearch
826e2b8b7817 docker.elastic.co/kibana/kibana:6.4.3 "/usr/local/bin/kiba…" About a minute ago Up About a minute 0.0.0.0:5601->5601/tcp kibana
$ curl localhost:5601
<script>var hashRoute = '/app/kibana';
var defaultRoute = '/app/kibana';
var hash = window.location.hash;
if (hash.length) {
window.location = hashRoute + hash;
} else {
window.location = defaultRoute;
}</script>
$ docker network inspect bridge
[
{
"Name": "bridge",
"Id": "abfadc2da70761c504878e7031c513caf71b4ce9052dc78c6a459de758e5cce8",
"Created": "2018-10-12T19:43:23.402133616Z",
"Scope": "local",
"Driver": "bridge",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "172.17.0.0/16",
"Gateway": "172.17.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {
"2fb0d212ba72c330750c234969e6b31750a4ac63bc1de8dda642b62225368b7c": {
"Name": "elasticsearch",
"EndpointID": "6dfa4bd39ba3870615f2aeb6d1ac5dc45949a4165dcdbe0df02d0e663b1b4da1",
"MacAddress": "02:42:ac:11:00:02",
"IPv4Address": "172.17.0.2/16",
"IPv6Address": ""
},
"52e6f5406159ee399ed2b59a4540e2cd68c0c1351ec89eab0ca69064c369723d": {
"Name": "kibana",
"EndpointID": "912217e5d01dfa24084df3867e244ab446f0eda205f5f12313e75b1878ba87c0",
"MacAddress": "02:42:ac:11:00:03",
"IPv4Address": "172.17.0.3/16",
"IPv6Address": ""
}
},
"Options": {
"com.docker.network.bridge.default_bridge": "true",
"com.docker.network.bridge.enable_icc": "true",
"com.docker.network.bridge.enable_ip_masquerade": "true",
"com.docker.network.bridge.host_binding_ipv4": "0.0.0.0",
"com.docker.network.bridge.name": "docker0",
"com.docker.network.driver.mtu": "1500"
},
"Labels": {}
}
]
In order to configure logstash and integrate it with Elasticsearch and Kibana, follow these steps:
- Inside data/logs/logs, download and uncompress https://github.com/elastic/elk-index-size-tests/blob/master/logs.gz
- Have this file inside /pipeline/logstash.conf:
input
{
file {
path => "/usr/share/logstash/data/logs/logs"
type => "logs"
start_position => "beginning"
}
}
filter
{
grok{
match => {
"message" => "%{COMBINEDAPACHELOG}"
}
}
mutate{
convert => { "bytes" => "integer" }
}
date {
match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
locale => en
remove_field => "timestamp"
}
geoip {
source => "clientip"
}
useragent {
source => "agent"
target => "useragent"
}
}
output
{
stdout {
codec => dots
}
elasticsearch {
hosts => ["http://elasticsearch:9200"]
}
}
- run the logstash container
$ docker run -d -p 5140:5140 --name logstash --link "elasticsearch:elasticsearch" -v ~/proyectos/elk-docker/logstash/pipeline/:/usr/share/logstash/pipeline/ -v ~/proyectos/elk-docker/logstash/data/logs/:/usr/share/logstash/data/logs/ docker.elastic.co/logstash/logstash:6.4.3
Note: use the command realpath to find out the directories' real path:
$ realpath elk-docker/logstash/pipeline/
/home/fernando/elk-docker/logstash/pipeline
Note: we're opening TCP port 5140 because logstash does not have root privileges to start ports < 1024.
syslog {
port => 5140
}
Note: review this tutorial https://www.kartar.net/2014/09/when-logstash-and-syslog-go-wrong/
***********
Steps for basic auth on Kibana through Nginx:
$ docker run -d --name kibana --link "elasticsearch:elasticsearch" docker.elastic.co/kibana/kibana:6.4.3
(Note: do not open any TCP ports on Kibana's container as all requests will be proxied by Nginx)
$ docker run -d --name nginx --link "kibana:kibana" -p 5601:5601 nginx
Enter the nginx container's interactive shell:
$ docker exec -it nginx /bin/bash
Confirm the Nginx container can access Kibana:
# apt-get update
# apt-get install curl -y
# curl -IL kibana:5601
HTTP/1.1 302 Found
location: /login?next=%2F
kbn-name: kibana
cache-control: no-cache
Date: Wed, 14 Nov 2018 18:18:10 GMT
Connection: keep-alive
HTTP/1.1 200 OK
kbn-name: kibana
cache-control: no-cache
Date: Wed, 14 Nov 2018 18:18:10 GMT
Connection: keep-alive
Get the htpasswd tool:
# apt-get install apache2-utils -y
(Note: review whether httpd-tools is a lighter alternative)
Create the user on the nginx container:
# htpasswd -c /etc/nginx/htpasswd.kibanauser kibana
New password:
Re-type new password:
Adding password for user kibana
Alternatively we may use the batch mode, for example: htpasswd -b -c htpasswd.test user userpassword.
This would allow us to create scripts to automate the deployment.
Replace the /etc/nginx/conf.d/default.conf file by:
(If you like the nano editor: apt-get install nano -y)
server {
listen *:5601;
server_name _;
location / {
proxy_pass http://kibana:5601;
auth_basic "Restricted";
auth_basic_user_file /etc/nginx/htpasswd.kibanauser;
}
}
Note: check the possibility of using this command
printf '%s\n%s\n' "to be prepended" "$(cat text.txt)" >text.txt
to automate prepending text to this file.
Reload nginx config
# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
# nginx -s reload
(Note: I'm using this command because if I run 'service nginx reload' the docker container crashes and silently exits from the bash.
If I try to restart it, I get this error:
$ docker exec -it nginx /bin/bash
Error response from daemon: Container 3dfcd5f89ff02927ba9b04945c18b7da37e0e000b696f682a01eec04ebcda450 is not running)
Confirm the restriction is working:
# curl -IL localhost:5601
HTTP/1.1 401 Unauthorized
Server: nginx/1.15.6
Date: Fri, 16 Nov 2018 16:48:20 GMT
Content-Type: text/html
Content-Length: 179
Connection: keep-alive
WWW-Authenticate: Basic realm="Restricted"
Navigate with a browser. Once you make sure the image is doing its job, commit the changes:
$ docker commit nginx nginx_basic_auth
Note: it may be a good idea to attach the logs directory to an the local filesystem, for debugging.
root@fa42b11e32c0:/# ls -al /var/log/nginx/
total 8
drwxr-xr-x 1 root root 4096 Nov 16 13:32 .
drwxr-xr-x 1 root root 4096 Nov 16 19:37 ..
lrwxrwxrwx 1 root root 11 Nov 16 13:32 access.log -> /dev/stdout
lrwxrwxrwx 1 root root 11 Nov 16 13:32 error.log -> /dev/stderr
Note: Create a Dockerfile that will build the nginx image then run all the other components as well.
$ cat Dockerfile
FROM nginx:latest
LABEL version=1
LABEL maintainer="Fernando <[email protected]>"
LABEL description="Authenticate requests to Kibana."
RUN apt-get install curl -y
RUN apt-get install apache2-utils -y
RUN htpasswd -b -c /etc/nginx/htpasswd.kibanauser kibana Kibana
RUN printf "server\n{\n listen *:5601;\n server_name _;\n location / {\n proxy_pass http://kibana:5601;\n auth_basic "Restricted";\n auth_basic_user_file /etc/nginx/htpasswd.kibanauser;\n }\n}\n" > /etc/nginx/conf.d/default.conf
RUN cat /etc/nginx/conf.d/default.conf
EXPOSE 5601/tcp
ENTRYPOINT ["nginx"]
CMD ["-g", "daemon off;"]
$ cat docker-compose.yml
version: "3"
services:
kibana:
image: docker.elastic.co/kibana/kibana:6.4.3
restart: always
container_name: kibana
nginx_auth:
depends_on:
- kibana
image: local:nginx_auth2
ports:
- "5601:5601"
restart: always
================
Follow these steps to combine nginx with php-fpm
$ cat docker-compose.yml
web:
image: nginx:latest
ports:
- "8080:80"
volumes:
- /home/fernando/elk-docker/nginx:/etc/nginx/conf.d/
- /home/fernando/elk-docker/php:/var/www/html/
links:
- php
php:
image: php:fpm-alpine
volumes:
- /home/fernando/elk-docker/php:/var/www/html/
~/elk-docker$ ls -al php
total 16
drwxr-xr-x 2 fernando fernando 4096 Nov 23 15:43 .
drwxr-xr-x 7 fernando fernando 4096 Nov 23 16:20 ..
-rw-r--r-- 1 fernando fernando 612 Nov 23 15:43 index.html
-rw-r--r-- 1 fernando fernando 27 Nov 23 15:43 index.php
$ cat nginx/default.conf
server {
index index.html;
server_name php-docker.local;
error_log /var/log/nginx/error.log;
access_log /var/log/nginx/access.log;
root /var/www/html;
location ~ \.php$ {
try_files $uri =404;
fastcgi_split_path_info ^(.+\.php)(/.+)$;
fastcgi_pass php:9000;
fastcgi_index index.php;
include fastcgi_params;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_param PATH_INFO $fastcgi_path_info;
}
}
thanks to http://geekyplatypus.com/dockerise-your-php-application-with-nginx-and-php7-fpm/ !