forked from splunk/kafka-connect-splunk
-
Notifications
You must be signed in to change notification settings - Fork 0
354 lines (331 loc) · 16.1 KB
/
ci_build_test.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
name: CI Build Test
on:
pull_request_target:
branches-ignore:
- /^release\/.*/
- master
workflow_call:
secrets:
FOSSA_API_KEY:
description: API token for FOSSA app
required: true
SEMGREP_PUBLISH_TOKEN:
description: Publish token for Semgrep
required: true
permissions:
checks: write
pull-requests: write
jobs:
workflow_approval:
name: Approve workflow
runs-on: ubuntu-20.04
environment: workflow-approval
steps:
- name: Approve workflow
run: echo For security reasons, all pull requests need to be approved first before running any automated CI.
fossa-scan:
continue-on-error: true
runs-on: ubuntu-latest
needs:
- workflow_approval
steps:
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{github.event.pull_request.head.sha}}
repository: ${{github.event.pull_request.head.repo.full_name}}
- name: run fossa anlyze and create report
run: |
curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash
fossa analyze --debug
fossa report attribution --format text > /tmp/THIRDPARTY
env:
FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }}
- name: upload THIRDPARTY file
uses: actions/upload-artifact@v2
with:
name: THIRDPARTY
path: /tmp/THIRDPARTY
- name: run fossa test
run: |
fossa test --debug
env:
FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }}
semgrep:
runs-on: ubuntu-latest
needs:
- workflow_approval
name: security-sast-semgrep
if: github.actor != 'dependabot[bot]'
steps:
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{github.event.pull_request.head.sha}}
repository: ${{github.event.pull_request.head.repo.full_name}}
- name: Semgrep
id: semgrep
uses: returntocorp/semgrep-action@v1
with:
publishToken: ${{ secrets.SEMGREP_PUBLISH_TOKEN }}
build-unit-test:
name: build and run unit test
runs-on: ubuntu-20.04
needs:
- workflow_approval
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: ${{github.event.pull_request.head.sha}}
repository: ${{github.event.pull_request.head.repo.full_name}}
- name: Get maven dependencies
run: |
mvn dependency:go-offline
- name: Run Unit tests
run: |
mvn package -Dsurefire.useSystemClassLoader=false -q
cp -R target/splunk-kafka-connect*.jar /tmp
- name: Upload artifact
uses: actions/upload-artifact@v2
with:
name: splunk-kafka-connector
path: /tmp/splunk-kafka-connect*.jar
- name: Publish Unit Test Results
uses: EnricoMi/publish-unit-test-result-action@v2
if: always()
with:
check_name: Unit Test Results
files: "target/surefire-reports/*.xml"
e2e_test:
name: e2e test - kafka version-${{ matrix.kafka_version }}
runs-on: ubuntu-20.04
needs:
- build-unit-test
strategy:
fail-fast: false
matrix:
include:
- kafka_version: "1.1.1"
kafka_package: "kafka_2.11-1.1.1.tgz"
- kafka_version: "2.0.0"
kafka_package: "kafka_2.11-2.0.0.tgz"
- kafka_version: "2.1.0"
kafka_package: "kafka_2.12-2.1.0.tgz"
- kafka_version: "2.6.0"
kafka_package: "kafka_2.13-2.6.0.tgz"
- kafka_version: "2.7.1"
kafka_package: "kafka_2.13-2.7.1.tgz"
- kafka_version: "2.8.0"
kafka_package: "kafka_2.13-2.8.0.tgz"
- kafka_version: "3.0.0"
kafka_package: "kafka_2.13-3.0.0.tgz"
- kafka_version: "3.1.0"
kafka_package: "kafka_2.13-3.1.0.tgz"
- kafka_version: "3.3.1"
kafka_package: "kafka_2.13-3.3.1.tgz"
env:
CI_SPLUNK_VERSION: "9.0.2"
CI_SPLUNK_FILENAME: splunk-9.0.2-17e00c557dc1-Linux-x86_64.tgz
CI_SPLUNK_HOST: 127.0.0.1
CI_SPLUNK_PORT: 8089
CI_SPLUNK_USERNAME: admin
CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959
CI_SPLUNK_HEC_TOKEN_ACK: a6b5e77f-d5f6-415a-bd43-930cecb12950
CI_SPLUNK_PASSWORD: helloworld
CI_INDEX_EVENTS: main
CI_KAFKA_HEADER_INDEX: kafka
CI_DATAGEN_IMAGE: rock1017/log-generator:latest
CI_OLD_CONNECTOR_VERSION: v2.0.1
SCHEMA_REGISTRY_URL: ${{ Secrets.SCHEMA_REGISTRY_URL }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: ${{github.event.pull_request.head.sha}}
repository: ${{github.event.pull_request.head.repo.full_name}}
- name: Install Splunk
run: |
cd /opt && wget -O $CI_SPLUNK_FILENAME 'https://d7wz6hmoaavd0.cloudfront.net/products/splunk/releases/'$CI_SPLUNK_VERSION'/linux/'$CI_SPLUNK_FILENAME''
sudo tar xzvf $CI_SPLUNK_FILENAME
# Set user seed
hashed_pwd=$(sudo /opt/splunk/bin/splunk hash-passwd $CI_SPLUNK_PASSWORD)
# sudo tee /opt/splunk/etc/system/local/user-seed.conf > /dev/null \<< EOF
sudo tee /opt/splunk/etc/system/local/user-seed.conf <<EOF >/dev/null
[user_info]
USERNAME = $CI_SPLUNK_USERNAME
HASHED_PASSWORD = $hashed_pwd
EOF
# Add delete capability to admin role
sudo tee /opt/splunk/etc/system/local/authorize.conf <<EOF >/dev/null
[role_admin]
delete_by_keyword = enabled
EOF
sudo /opt/splunk/bin/splunk start --accept-license --answer-yes --no-prompt
# Enable HEC services
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable
# Create new HEC token
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token&token=$CI_SPLUNK_HEC_TOKEN" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http
# Enable HEC new-token
sudo /opt/splunk/bin/splunk http-event-collector enable -name splunk_hec_token -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
# Create new HEC token with ack
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token_ack&token=$CI_SPLUNK_HEC_TOKEN_ACK&useACK=1" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http
# Enable HEC new-token
sudo /opt/splunk/bin/splunk http-event-collector enable -name splunk_hec_token_ack -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
# Setup Indexes
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=$CI_INDEX_EVENTS&datatype=event" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/-/search/data/indexes
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=$CI_KAFKA_HEADER_INDEX&datatype=event" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/-/search/data/indexes
# Restart Splunk
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/server/control/restart -X POST
- name: Install Kafka ${{ matrix.kafka_version }}
run: |
cd /tmp && wget https://archive.apache.org/dist/kafka/${{ matrix.kafka_version }}/${{ matrix.kafka_package }}
sudo tar xzf ${{ matrix.kafka_package }}
rm ${{ matrix.kafka_package }}
sudo mv kafka_* /usr/local/kafka
cd /usr/local/kafka && ls
- name: Start ZooKeeper Server
working-directory: /usr/local/kafka
run: |
echo "Start ZooKeeper"
sudo bin/zookeeper-server-start.sh config/zookeeper.properties &
- name: Start Kafka Server
working-directory: /usr/local/kafka
run: |
echo "Start kafka server"
sudo bin/kafka-server-start.sh config/server.properties &
- uses: actions/setup-python@v2
with:
python-version: 3.7
- name: Download artifact
uses: actions/download-artifact@v2
with:
name: splunk-kafka-connector
path: /tmp
- name: Test kafka connect upgrade
run: |
echo "Download kafka connect "$CI_OLD_CONNECTOR_VERSION
# Summary for the test
#1)We will deploy old kafka connector and create 2 tasks for that to check ack and without ack functionality
#2)then we will remove that old kafka connector and deploy new kafka connector with updation of two tasks
#3) Here in the updation we will check for the new functionality("splunk.hec.json.event.formatted" and "splunk.hec.raw") so that we can check if we can successfully upgrade the connector
#4)At last we will check if we have recieved 2000 events for both the tasks
sudo mkdir -p /usr/local/share/kafka/plugins/
wget https://github.com/splunk/kafka-connect-splunk/releases/download/$CI_OLD_CONNECTOR_VERSION/splunk-kafka-connect-$CI_OLD_CONNECTOR_VERSION.jar
sudo cp splunk-kafka-connect-$CI_OLD_CONNECTOR_VERSION.jar /usr/local/share/kafka/plugins/
sudo mkdir -p /usr/local/share/kafka-connector/
sudo cp /tmp/splunk-kafka-connect*.jar /usr/local/share/kafka-connector/
test /usr/local/share/kafka-connector/splunk-kafka-connect*.jar && echo /usr/local/share/kafka-connector/splunk-kafka-connect*.jar
sed -i 's/plugin\.path\=connectors\//plugin\.path\=\/usr\/local\/share\/kafka\/plugins\//' $GITHUB_WORKSPACE/config/connect-distributed-quickstart.properties
sed -i 's/key\.converter\=org\.apache\.kafka\.connect\.storage\.StringConverter/key\.converter\=org\.apache\.kafka\.connect\.json\.JsonConverter/' $GITHUB_WORKSPACE/config/connect-distributed-quickstart.properties
sed -i 's/value\.converter\=org\.apache\.kafka\.connect\.storage\.StringConverter/value\.converter\=org\.apache\.kafka\.connect\.json\.JsonConverter/' $GITHUB_WORKSPACE/config/connect-distributed-quickstart.properties
pip install --upgrade pip
pip install -r test/requirements.txt
export PYTHONWARNINGS="ignore:Unverified HTTPS request"
echo "Test kafka connect upgrade ..."
source $GITHUB_WORKSPACE/test/config.sh
test -f $connector_path/$old_connector_name && echo $connector_path /$old_connector_name
# Starting Old Connector
sudo $kafka_home/bin/connect-distributed.sh $GITHUB_WORKSPACE/config/connect-distributed-quickstart.properties > output.log 2>&1 &
sleep 20
# Creating the two tasks (with ack and without ack)
curl ${kafka_connect_url}/connectors -X POST -H "Content-Type: application/json" -d '{
"name": "kafka_connect",
"config": {
"connector.class": "com.splunk.kafka.connect.SplunkSinkConnector",
"tasks.max": "1",
"splunk.indexes": "'"$splunk_index"'",
"topics": "kafka_connect_upgrade",
"splunk.hec.ack.enabled": "false",
"splunk.hec.uri": "'"$splunk_hec_url"'",
"splunk.hec.ssl.validate.certs": "false",
"splunk.hec.token": "'"$splunk_token"'" ,
"splunk.sources": "kafka_connect",
"splunk.hec.raw": "true",
"splunk.sourcetypes":"upgraded_test"
}
}'
sleep 5
curl ${kafka_connect_url}/connectors -X POST -H "Content-Type: application/json" -d '{
"name": "kafka_connect_ack",
"config": {
"connector.class": "com.splunk.kafka.connect.SplunkSinkConnector",
"tasks.max": "1",
"splunk.indexes": "'"$splunk_index"'",
"topics": "kafka_connect_upgrade",
"splunk.hec.ack.enabled": "true",
"splunk.hec.uri": "'"$splunk_hec_url"'",
"splunk.hec.ssl.validate.certs": "false",
"splunk.hec.token": "'"$splunk_token_ack"'" ,
"splunk.sources": "kafka_connect_ack",
"splunk.hec.raw": "true",
"splunk.sourcetypes":"upgraded_test"
}
}'
sleep 5
# Generating 1000 events
python test/lib/eventproducer_connector_upgrade.py 1000 --log-level=INFO
sudo kill $(sudo lsof -t -i:8083) && sleep 2
sudo rm $connector_path/$old_connector_name && sleep 2
sudo cp $connector_build_target/splunk-kafka-connect*.jar $connector_path && sleep 2
# Starting New Connector
sudo $kafka_home/bin/connect-distributed.sh $GITHUB_WORKSPACE/config/connect-distributed-quickstart.properties > output.log 2>&1 &
# Updating the two tasks (with ack and without ack)
sleep 10
curl ${kafka_connect_url}/connectors/kafka_connect/config -X PUT -H "Content-Type: application/json" -d '{
"connector.class": "com.splunk.kafka.connect.SplunkSinkConnector",
"tasks.max": "1",
"splunk.indexes": "'"$splunk_index"'",
"topics": "kafka_connect_upgrade",
"splunk.hec.ack.enabled": "false",
"splunk.hec.uri": "'"$splunk_hec_url"'",
"splunk.hec.ssl.validate.certs": "false",
"splunk.hec.token": "'"$splunk_token"'" ,
"splunk.sources": "kafka_connect",
"splunk.hec.raw": "false",
"splunk.hec.json.event.formatted": "true",
"splunk.sourcetypes":"upgraded_test"
}'
sleep 5
curl ${kafka_connect_url}/connectors/kafka_connect_ack/config -X PUT -H "Content-Type: application/json" -d '{
"connector.class": "com.splunk.kafka.connect.SplunkSinkConnector",
"tasks.max": "1",
"splunk.indexes": "'"$splunk_index"'",
"topics": "kafka_connect_upgrade",
"splunk.hec.ack.enabled": "true",
"splunk.hec.uri": "'"$splunk_hec_url"'",
"splunk.hec.ssl.validate.certs": "false",
"splunk.hec.token": "'"$splunk_token_ack"'" ,
"splunk.sources": "kafka_connect_ack",
"splunk.hec.raw": "false",
"splunk.hec.json.event.formatted": "true",
"splunk.sourcetypes":"upgraded_test"
}'
sleep 5
# Generating 1000 events
python test/lib/eventproducer_connector_upgrade.py 2000 --log-level=INFO
# Check in splunk that we have recieved 2000 events for with ack and without ack tasks
python test/lib/connector_upgrade.py --log-level=INFO
- uses: actions/upload-artifact@v3
if: failure()
with:
name: kafka-connect-logs-${{ matrix.kafka_version }}
path: output.log
- name: Install kafka connect
run: |
sudo rm -f /usr/local/share/kafka/plugins/splunk-kafka-connect*.jar
sudo cp /tmp/splunk-kafka-connect*.jar /usr/local/share/kafka/plugins/
sudo /usr/local/kafka/bin/connect-distributed.sh $GITHUB_WORKSPACE/config/connect-distributed-quickstart.properties &
- name: Run functional tests
run: |
sleep 5
pip install --upgrade pip
pip install -r test/requirements.txt
export PYTHONWARNINGS="ignore:Unverified HTTPS request"
echo "Running functional tests....."
python -m pytest --log-level=INFO
- uses: actions/upload-artifact@v3
if: failure()
with:
name: splunk-events-${{ matrix.kafka_version }}
path: events.txt