From 5453a384c84c2d566fa9c938850c33bf557741d4 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 13 Jul 2021 07:46:46 -0400 Subject: [PATCH 01/12] feat: Support Dell iDrac --- docs/sources/Dell/index.md | 43 ++++++++++++++++ mkdocs.yml | 1 + .../syslog/app-dell_poweredge_idrac.conf | 42 ++++++++++++++++ tests/test_dell_idrac.py | 50 +++++++++++++++++++ 4 files changed, 136 insertions(+) create mode 100644 docs/sources/Dell/index.md create mode 100644 package/etc/conf.d/conflib/syslog/app-dell_poweredge_idrac.conf create mode 100644 tests/test_dell_idrac.py diff --git a/docs/sources/Dell/index.md b/docs/sources/Dell/index.md new file mode 100644 index 0000000000..c40a35d325 --- /dev/null +++ b/docs/sources/Dell/index.md @@ -0,0 +1,43 @@ +# Vendor - Dell + +## Product - EPV + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | na | +| Add-on Manual | https://www.dell.com/support/manuals/en-au/dell-opnmang-sw-v8.1/eemi_13g_v1.2-v1/introduction?guid=guid-8f22a1a9-ac01-43d1-a9d2-390ca6708d5e&lang=en-us | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| dell:poweredge:idrac:syslog | None | + +### Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| dell_poweredge_idrac | dell:poweredge:idrac:syslog | infraops | none | + +### Filter type + +MSG Parse: This filter parses message content + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_DELL_POWEREDGE_IDRAC_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers | +| SC4S_LISTEN_DELL_POWEREDGE_IDRAC_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers | + + +### Verification + +An active site will generate frequent events use the following search to check for new events + +Verify timestamp, and host values match as expected + +``` +index= (sourcetype=cef sourcetype="UDP") +``` diff --git a/mkdocs.yml b/mkdocs.yml index 92c2e6cc24..acd8d3870f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -25,6 +25,7 @@ nav: - Citrix: sources/Citrix/index.md - "Common Event Format": sources/CommonEventFormat/index.md - CyberArk: sources/CyberArk/index.md + - Dell: sources/Dell/index.md - "Dell RSA": sources/Dell_RSA/index.md - "Dell EMC": sources/Dell_EMC/index.md - F5: sources/F5/index.md diff --git a/package/etc/conf.d/conflib/syslog/app-dell_poweredge_idrac.conf b/package/etc/conf.d/conflib/syslog/app-dell_poweredge_idrac.conf new file mode 100644 index 0000000000..7c1aaf7754 --- /dev/null +++ b/package/etc/conf.d/conflib/syslog/app-dell_poweredge_idrac.conf @@ -0,0 +1,42 @@ +block parser dell_poweredge_idrac-parser() { + channel { + + parser { + kv-parser( + value-separator(":") + prefix(".idrac.") + template("${PROGRAM}: ${MESSAGE}") + ); + + }; + + filter { + "${.idrac.Severity}" ne "" + and "${.idrac.Category}" ne "" + and "${.idrac.MessageID}" ne "" + and "${.idrac.Message}" ne "" + and match('[A-Z]{1,3}\d{1,4}' , value(".idrac.MessageID")) + }; + + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('dell:poweredge:idrac:syslog') + vendor_product("dell_poweredge_idrac") + ); + }; + rewrite{ + set("${PROGRAM}: $MSG" value("MSG")); + unset(value('PROGRAM')); + }; + + + }; +}; +application dell_poweredge_idrac[sc4s-syslog] { + filter { + program('Severity'); + }; + parser { dell_poweredge_idrac-parser(); }; +}; + diff --git a/tests/test_dell_idrac.py b/tests/test_dell_idrac.py new file mode 100644 index 0000000000..78b96ae4f3 --- /dev/null +++ b/tests/test_dell_idrac.py @@ -0,0 +1,50 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * +from .timeutils import * + +import pytest +env = Environment() + + +#<134>Feb 18 09:37:41 xxxxxx swlogd: bcmd esm info(5) phy_nlp_enable_set: u=0 p=1 enable:1 phyPresent:YES +testdata = [ + "{{ mark }}{{ bsd }} {{ host }} Severity: Informational, Category: Audit, MessageID: LOG007, Message: The previous log entry was repeated 0 times.", + "{{ mark }}{{ bsd }} {{ host }} Severity: Informational, Category: Audit, MessageID: LOG006, Message: Test event generated for message ID LOG007.", + "{{ mark }}{{ bsd }} {{ host }} Severity: Informational, Category: Audit, MessageID: USR0032, Message: The session for root from 10.110.161.37 using GUI is logged off.", + "{{ mark }}{{ bsd }} {{ host }} Severity: Informational, Category: Audit, MessageID: USR0030, Message: Successfully logged in using root, from 10.110.161.37 and GUI.", +] + +@pytest.mark.parametrize("event", testdata) +def test_dell_idrac(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event): + host = get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string(event + "\n") + message = mt.render(mark="<166>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + "search index=infraops _time={{ epoch }} sourcetype=\"dell:poweredge:idrac:syslog\" (host=\"{{ host }}\" OR \"{{ host }}\")") + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 From 84414a45b72418fc6f525473054a99f03d050337 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 13 Jul 2021 07:51:32 -0400 Subject: [PATCH 02/12] doc: Update app reference for ite work fixes #1188 --- docs/gettingstarted/index.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/gettingstarted/index.md b/docs/gettingstarted/index.md index fd939bff28..7de95e60b0 100644 --- a/docs/gettingstarted/index.md +++ b/docs/gettingstarted/index.md @@ -59,9 +59,7 @@ using the SC4S defaults. SC4S can be easily customized to use different indexes Install the following: -* [Splunk App for Infrastructure](https://splunkbase.splunk.com/app/3975/) -* [Splunk Add-on for Infrastructure](https://splunkbase.splunk.com/app/4217/) -* [Splunk Metrics Workspace](https://splunkbase.splunk.com/app/4192/) *NOTE Included in Splunk 7.3.0 and above* +* [IT Essentials Work](https://splunkbase.splunk.com/app/5403/) #### Configure the Splunk HTTP Event Collector From 999e1644ba7463064aad748dacb5ee800c9da7da Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 13 Jul 2021 08:29:12 -0400 Subject: [PATCH 03/12] feat(source): Support custom topic sets for extreme snowflakes fixes #1189 --- .../etc/conf.d/sc4slib/source_syslog/plugin.jinja | 14 ++++++++------ package/etc/conf.d/sc4slib/source_syslog/plugin.py | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/package/etc/conf.d/sc4slib/source_syslog/plugin.jinja b/package/etc/conf.d/sc4slib/source_syslog/plugin.jinja index f04975fc0c..96383cf5b5 100644 --- a/package/etc/conf.d/sc4slib/source_syslog/plugin.jinja +++ b/package/etc/conf.d/sc4slib/source_syslog/plugin.jinja @@ -84,9 +84,10 @@ source s_{{ port_id }} { rewrite(r_set_splunk_default); + if { parser { - app-parser(topic(sc4s-raw-syslog)); + app-parser(topic({{ topic }}-raw-syslog)); }; } elif { filter{ @@ -117,9 +118,10 @@ source s_{{ port_id }} { if { parser { - app-parser(topic(sc4s-syslog)); + app-parser(topic({{ topic }}-syslog)); }; - }; + }; + if { parser(p_add_context_host); @@ -137,7 +139,7 @@ source s_{{ port_id }} { parser(vendor_product_by_source); if { parser { - app-parser(topic(sc4s-network-source)); + app-parser(topic({{ topic }}-network-source)); }; }; @@ -257,7 +259,7 @@ source s_{{ port_id }} { }; if { parser { - app-parser(topic(sc4s-syslog)); + app-parser(topic({{ topic }}-syslog)); }; }; @@ -275,7 +277,7 @@ source s_{{ port_id }} { parser(vendor_product_by_source); if { parser { - app-parser(topic(sc4s-network-source)); + app-parser(topic({{ topic }}-network-source)); }; }; diff --git a/package/etc/conf.d/sc4slib/source_syslog/plugin.py b/package/etc/conf.d/sc4slib/source_syslog/plugin.py index 110634bf66..d804caf9cb 100755 --- a/package/etc/conf.d/sc4slib/source_syslog/plugin.py +++ b/package/etc/conf.d/sc4slib/source_syslog/plugin.py @@ -63,6 +63,7 @@ use_reverse_dns=use_reverse_dns, use_tls=use_tls, tls_dir=os.getenv(f"SC4S_TLS", 17039360), + topic=os.getenv(f"SC4S_LISTEN_{ port_id }_TOPIC", "sc4s"), port_udp=os.getenv(f"SC4S_LISTEN_{ port_id }_UDP_PORT", "disabled").split(","), port_udp_sockets=int(os.getenv(f"SC4S_SOURCE_LISTEN_UDP_SOCKETS", 4)), port_udp_sorecvbuff=os.getenv(f"SC4S_SOURCE_UDP_SO_RCVBUFF", 17039360), From f3faba5868c294edfc6da861a678961842517504 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 13 Jul 2021 08:42:11 -0400 Subject: [PATCH 04/12] feat(thycotic): support secret server fixes #1183 --- docs/sources/Thycotic/index.md | 39 +++++++++++++++++++ mkdocs.yml | 1 + .../cef/app-cef-thycotic_secret_server.conf | 18 +++++++++ 3 files changed, 58 insertions(+) create mode 100644 docs/sources/Thycotic/index.md create mode 100644 package/etc/conf.d/conflib/cef/app-cef-thycotic_secret_server.conf diff --git a/docs/sources/Thycotic/index.md b/docs/sources/Thycotic/index.md new file mode 100644 index 0000000000..5687ee65e7 --- /dev/null +++ b/docs/sources/Thycotic/index.md @@ -0,0 +1,39 @@ +# Vendor - Tenable + + +## Product - Tenable.nnm + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/4060/ | +| Product Manual | https://docs.tenable.com/integrations/Splunk/Content/Splunk2/ProcessWorkflow.htm | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| thycotic:syslog | None | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| Thycotic Software_Secret Server | thycotic:syslog | netauth | none | + +### Filter type + +CEF + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| + +### Verification + +An active device will generate frequent events. Use the following search to validate events are present per source device + +``` +index= sourcetype=thycotic:syslog | stats count by host +``` \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 92c2e6cc24..464633e86e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -58,6 +58,7 @@ nav: - Broadcom: sources/Broadcom/index.md - Tanium: sources/Tanium/index.md - Tenable: sources/Tenable/index.md + - Thycotic: sources/Thycotic/index.md - Tintri: sources/Tintri/index.md - Trend: sources/Trend/index.md - Ubiquiti: sources/Ubiquiti/index.md diff --git a/package/etc/conf.d/conflib/cef/app-cef-thycotic_secret_server.conf b/package/etc/conf.d/conflib/cef/app-cef-thycotic_secret_server.conf new file mode 100644 index 0000000000..152d878f3c --- /dev/null +++ b/package/etc/conf.d/conflib/cef/app-cef-thycotic_secret_server.conf @@ -0,0 +1,18 @@ +block parser cef-thycotic_secret_server-parser() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netauth'), + sourcetype('thycotic:syslog') + ); + }; + + }; +}; +application cef-thycotic_secret_server[cef] { + filter{ + match("Thycotic Software" value("fields.cef_device_vendor")) + and match("Secret Server" value("fields.cef_device_product")); + }; + parser { cef-thycotic_secret_server-parser(); }; +}; From 1b6fef017961c6fcbdbc10ece75e36221db7891e Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 13 Jul 2021 08:44:13 -0400 Subject: [PATCH 05/12] fix(netscaler): Correct appfw sourcetype Fixes #1190 --- package/etc/conf.d/conflib/raw/app-citrix_netscaler.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/etc/conf.d/conflib/raw/app-citrix_netscaler.conf b/package/etc/conf.d/conflib/raw/app-citrix_netscaler.conf index 61f81c0524..e067436478 100644 --- a/package/etc/conf.d/conflib/raw/app-citrix_netscaler.conf +++ b/package/etc/conf.d/conflib/raw/app-citrix_netscaler.conf @@ -33,7 +33,7 @@ block parser citrix_netscaler-parser() { ); set("citrix_netscaler", value("fields.sc4s_syslog_format")); r_set_splunk_dest_update( - sourcetype('sourcetype::citrix:netscaler:appfw') condition(message('[^|]APPFW[^|]')) + sourcetype('citrix:netscaler:appfw') condition(message('[^|]APPFW[^|]')) ); }; From a24eaf7f5ac601a012d9fdd85bc05cb948bcc6f7 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 13 Jul 2021 08:48:09 -0400 Subject: [PATCH 06/12] doc: Add proxy support to doc fixes #1187 --- docs/configuration.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 73386a9bf7..5a7a67bb60 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -20,6 +20,16 @@ syslog. (typically a caching nameserver) is not performant. If you notice events being indexed far later than their actual timestamp in the event (latency between `_indextime` and `_time`), this is the first place to check. +## Configure use of external http proxy + +Warning: Many http proxies are not provisioned with application traffic in mind. Ensure adequate capacity is available to avoid data +loss and or proxy outages. Note: the follow vairables are lower case + + +| Variable | Values | Description | +|----------|---------------|-------------| +| http_proxy | undefined | Use libcurl format proxy string "http://username:password@proxy.server:port" | +| https_proxy | undefined | Use libcurl format proxy string "http://username:password@proxy.server:port" | ## Splunk HEC Destination Configuration From 6c9d46f4455e7159cc3c3ad8bd818f595698b758 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 13 Jul 2021 08:50:24 -0400 Subject: [PATCH 07/12] doc: fix typo in buffer size fixes #1155 --- docs/troubleshooting/troubleshoot_SC4S_server.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/troubleshooting/troubleshoot_SC4S_server.md b/docs/troubleshooting/troubleshoot_SC4S_server.md index 8a76b4d605..2c5e2f025f 100644 --- a/docs/troubleshooting/troubleshoot_SC4S_server.md +++ b/docs/troubleshooting/troubleshoot_SC4S_server.md @@ -156,8 +156,8 @@ The kernel refused to set the receive buffer (SO_RCVBUF) to the requested size, Make changes to /etc/sysctl.conf. Changing receive buffer values here to 16 MB: ``` -net.core.rmem_default = 1703936 -net.core.rmem_max = 1703936. +net.core.rmem_default = 17039360 +net.core.rmem_max = 17039360 ``` Run following commands for changes to be affected. ``` From 66f8b8049db2adc76c71ffb5ba0124d8f69d38e7 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 13 Jul 2021 09:25:15 -0400 Subject: [PATCH 08/12] feat(polycom): Support RPRM fixes #1186 --- docs/sources/Polycom/index.md | 44 ++++++++++++++++++ mkdocs.yml | 1 + .../conflib/syslog/app-polycom_rprm.conf | 21 +++++++++ tests/test_polycom.py | 45 +++++++++++++++++++ 4 files changed, 111 insertions(+) create mode 100644 docs/sources/Polycom/index.md create mode 100644 package/etc/conf.d/conflib/syslog/app-polycom_rprm.conf create mode 100644 tests/test_polycom.py diff --git a/docs/sources/Polycom/index.md b/docs/sources/Polycom/index.md new file mode 100644 index 0000000000..6624f68742 --- /dev/null +++ b/docs/sources/Polycom/index.md @@ -0,0 +1,44 @@ +# Vendor - Polycom + +## Product - RPRM + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | none | +| Product Manual | unknown | + + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| polycom:rprm:syslog | | + +### Sourcetype and Index Configuration + +| key | sourcetype | index | notes | +|----------------|----------------|----------------|----------------| +| polycom_rprm | polycom:rprm:syslog | netops | none | + + +### Filter type + +MSG Parse: This filter parses message content + + +### Options + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_POLYCOM_RPRM_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers. | +| SC4S_POLYCOM_RPRM_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers. | +| SC4S_ARCHIVE_POLYCOM_RPRM | no | Enable archive to disk for this specific source | +| SC4S_DEST_POLYCOM_RPRM_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + +### Verification + +One or two sourcetypes are included in Proofpoint PPS logs. The search below will surface both of them: + +``` +index= sourcetype=polycom:rprm:syslog| stats count by host +``` diff --git a/mkdocs.yml b/mkdocs.yml index acd8d3870f..aeed729693 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -47,6 +47,7 @@ nav: - Nix: sources/nix/index.md - "Palo Alto Networks": sources/PaloaltoNetworks/index.md - "pfSense": sources/Pfsense/index.md + - Polycom: sources/Polycom/index.md - Pulse: sources/Pulse/index.md - Proofpoint: sources/Proofpoint/index.md - Radware: sources/Radware/index.md diff --git a/package/etc/conf.d/conflib/syslog/app-polycom_rprm.conf b/package/etc/conf.d/conflib/syslog/app-polycom_rprm.conf new file mode 100644 index 0000000000..6c315394a2 --- /dev/null +++ b/package/etc/conf.d/conflib/syslog/app-polycom_rprm.conf @@ -0,0 +1,21 @@ +block parser polycom_rprm-parser() { + channel { + rewrite { + r_set_splunk_dest_default( + index('netops') + sourcetype('polycom:rprm:syslog') + vendor_product("polycom_rprm") + template('t_5424_hdr_sdata_msg') + ); + }; + + + }; +}; +application polycom_rprm[sc4s-syslog] { + filter { + program('RPRM'); + }; + parser { polycom_rprm-parser(); }; +}; + diff --git a/tests/test_polycom.py b/tests/test_polycom.py new file mode 100644 index 0000000000..9bbb20d6c0 --- /dev/null +++ b/tests/test_polycom.py @@ -0,0 +1,45 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * +from .timeutils import * +import pytest + +env = Environment() + +polycom_data = [ + r'{{ mark }} {{ iso }}Z {{ host }} RPRM 107463 Jserver - DEBUG|||http-nio-5443-exec-22|com.polycom.rpum.epm.engine.ruleengine.ProfileFillingAction| ...df8-46f4-8ed1-2acc1bd62f97, profileUuid=47e0d340-5c83-4f6b-9692-9507cb6b3e83, tagName=call.autoOffHook.3.enabled, tagValue=1, required=true, canModify=true], ProfileTag [tagId=3e2fb279-c386-410b-866e-b427aaea80c4, profileUuid=47e0d340-5c83-4f6b-9692-9507cb6b3e83, tagName=call.teluri.showPrompt, tagValue=0, required=true, canModify=true], ProfileTag [tagId=6168b060-fe0e-414d-a25a-acbe629f963c, profileUuid=47e0d340-5c83-4f6b-9692-9507cb6b3e83, tagName=dialplan.3.applyToDirectoryDial, tagValue=1, required=true, canModify=true], ProfileTag [tagId=a835bbaf-1202-415a-8933-360a54acced1, profileUuid=47e0d340-5c83-4f6b-9692-9507cb6b3e83, tagName=dialplan.3.digitmap, tagValue=sip\:x.\.x.\@zoomcrc\.com|sip\:x.\@zoomcrc\.com|x.\.x.\@zoomcrc\.com|x.\@zoomcrc\.com|xxxxxxxxx.T|xxxxxxxxxx| , required=true, canModify=true], ProfileTag [tagId=67e41d5e-1112-4e36-8f78-e682ed61b4cc, profileUuid=47e0d340-5c83-4f6b-9692-9507cb6b3e83, tagName=dialplan.3.digitmap.timeOut, tagValue=4, required=true, canModify=true], ProfileTag [tagId=577dd248-7fdd-4730-aa90-ef7f1aa2f19b, profileUuid=47e0d340-5c83-4f6b-9692-9507cb6b3e83, tagName=dialplan.applyToDirectoryDial, tagValue=1, required=true, canModify=true], ProfileTag [tagId=f44bd920-fa45-4d11-90ff-2e294a45d1e1, profileUuid=47e0d340-5c83-4f6b-9692-9507cb6b3e83, tagName=dialplan.digitmap.lineSwitching.enable, tagValue=1, required=true, canModify=true], ProfileTag [tagId=5d1f9d8f-6583-4f5d-83c3-76194c299971, profileUuid=47e0d340-5c83-4f6b-9692-9507cb6b3e83, tagName=exchange.meeting.parseAllowedSipUriDomains, tagValue=zoomcrc.com,zoom.us,vip2.zoomus.com,bjn.vc,polycom.com, required=true, canModify=true], ProfileTag [tagId=b8a2dd79-7b8f-48be-b452-e529e2071003, profileUuid=47e0d340-5c83-4f6b-9692-9507cb6b3e83, tagName=exchange.meeting.parseEmailsAsSipUris, tagValue=1, required=true, canModify=true], ProfileTag [tagId=bfe8cd05...2048', +] + +@pytest.mark.parametrize("event", polycom_data) +def test_polycom(record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s,event): + host = get_host_key + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + iso = dt.isoformat()[0:23] + epoch = epoch[:-3] + + mt = env.from_string(event + "\n") + message = mt.render(mark="<29>1", iso=iso, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string('search _time={{ epoch }} index=netops host="{{ host }}" sourcetype="polycom:rprm:syslog"') + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 From 7bfa52252fb1bbb16a1803d85d68aa97406a3755 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 13 Jul 2021 11:59:43 -0400 Subject: [PATCH 09/12] feat(nix): Improve support for re-sourcetyping nix --- .../conflib/fallback/app-nix_syslog.conf | 2 +- .../conflib/net_source/app-nix_syslog.conf | 9 +----- .../conf.d/sc4slib/source_syslog/plugin.jinja | 3 +- .../splunk_metadata.csv.example | 1 + .../vendor_product_by_source.conf.example | 4 +++ .../vendor_product_by_source.csv.example | 3 +- tests/test_vmware.py | 29 +++++++++++++++++++ 7 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 package/etc/context_templates/splunk_metadata.csv.example diff --git a/package/etc/conf.d/conflib/fallback/app-nix_syslog.conf b/package/etc/conf.d/conflib/fallback/app-nix_syslog.conf index a4660e96bc..1d1cdc61a9 100644 --- a/package/etc/conf.d/conflib/fallback/app-nix_syslog.conf +++ b/package/etc/conf.d/conflib/fallback/app-nix_syslog.conf @@ -13,9 +13,9 @@ block parser nix_syslog_fallback-parser() { subst('^\/(?:[^\/]+\/)+', "" , value(".PROGRAM")); r_set_splunk_dest_update( source('program:${.PROGRAM}') + meta_key('${.netsource.sc4s_vendor_product}_nix_syslog') ); }; - }; }; application nix_syslog_fallback[fallback] { diff --git a/package/etc/conf.d/conflib/net_source/app-nix_syslog.conf b/package/etc/conf.d/conflib/net_source/app-nix_syslog.conf index 076395400b..d76b827924 100644 --- a/package/etc/conf.d/conflib/net_source/app-nix_syslog.conf +++ b/package/etc/conf.d/conflib/net_source/app-nix_syslog.conf @@ -11,14 +11,7 @@ block parser nix_syslog-parser() { ); }; - - - rewrite { - r_set_splunk_dest_update( - meta_key('${.netsource.sc4s_vendor_product}_nix_syslog') - - ); - }; + }; }; application nix_syslog[sc4s-network-source] { diff --git a/package/etc/conf.d/sc4slib/source_syslog/plugin.jinja b/package/etc/conf.d/sc4slib/source_syslog/plugin.jinja index 96383cf5b5..51544b8765 100644 --- a/package/etc/conf.d/sc4slib/source_syslog/plugin.jinja +++ b/package/etc/conf.d/sc4slib/source_syslog/plugin.jinja @@ -141,8 +141,7 @@ source s_{{ port_id }} { parser { app-parser(topic({{ topic }}-network-source)); }; - }; - + }; if { filter { "${fields.sc4s_vendor_product}" eq "" diff --git a/package/etc/context_templates/splunk_metadata.csv.example b/package/etc/context_templates/splunk_metadata.csv.example new file mode 100644 index 0000000000..104ba9f7bb --- /dev/null +++ b/package/etc/context_templates/splunk_metadata.csv.example @@ -0,0 +1 @@ +vmware_esx_nix_syslog,index,infraops \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.conf.example b/package/etc/context_templates/vendor_product_by_source.conf.example index 0efdf3165e..3d639fc9cc 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf.example +++ b/package/etc/context_templates/vendor_product_by_source.conf.example @@ -83,4 +83,8 @@ filter f_spectracom_ntp { }; filter f_sophos_webappliance { host("test-sophos-webapp-" type(string) flags(prefix)) +}; + +filter f_vmware_esx { + host("testvmwg-" type(string) flags(prefix)) }; \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.csv.example b/package/etc/context_templates/vendor_product_by_source.csv.example index a9908b4192..9ee0102756 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv.example +++ b/package/etc/context_templates/vendor_product_by_source.csv.example @@ -18,4 +18,5 @@ f_spectracom_ntp,sc4s_vendor_productm",spectracom_ntp" f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" f_tzfixny,sc4s_time_zone,"America/New_York" f_cisco_esa,sc4s_vendor_product,"cisco_esa" -f_sophos_webappliance,sc4s_vendor_product,"sophos_webappliance" \ No newline at end of file +f_sophos_webappliance,sc4s_vendor_product,"sophos_webappliance" +f_vmware_esx,sc4s_vendor_product,"vmware_esx" \ No newline at end of file diff --git a/tests/test_vmware.py b/tests/test_vmware.py index 124f851c56..dbd2b9a045 100644 --- a/tests/test_vmware.py +++ b/tests/test_vmware.py @@ -45,6 +45,35 @@ def test_linux_vmware(record_property, setup_wordlist, setup_splunk, setup_sc4s) assert resultCount == 1 +def test_linux_vmware_nix(record_property, setup_wordlist, setup_splunk, setup_sc4s): + host = "testvmwg-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) + pid = random.randint(1000, 32000) + + dt = datetime.datetime.now(datetime.timezone.utc) + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + # iso from included timeutils is from local timezone; need to keep iso as UTC + iso = dt.isoformat()[0:26] + iso_header = dt.isoformat()[0:23] + epoch = epoch[:-3] + + mt = env.from_string("{{ mark }}1 {{ iso_header }}Z {{ host }} sshd {{ pid }} - - - Generic event\n") + message = mt.render(mark="<144>", iso_header=iso_header, iso=iso, host=host, pid=pid) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string("search _time={{ epoch }} index=infraops host={{ host }} {{ pid }} sourcetype=\"nix:syslog\"") + search = st.render(epoch=epoch, host=host, pid=pid) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1 + #<46>1 2019-10-24T21:00:02.403Z {{ host }} NSXV 5996 - [nsxv@6876 comp="nsx-manager" subcomp="manager"] Invoking EventHistoryCollector.readNext on session[52db61bf-9c30-1e1f-5a26-8cd7e6f9f552]52032c51-240a-7c30-cd84-4b4246508dbe, operationID=opId-688ef-9725704 def test_linux_vmware_nsx_ietf(record_property, setup_wordlist, setup_splunk, setup_sc4s): host = "testvmw-{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist)) From a9bb77c6317dc1b1ab55bbf827712008f24bcbd3 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 13 Jul 2021 13:22:56 -0400 Subject: [PATCH 10/12] doc: Correct generated HEC destination in docs --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5a7a67bb60..1b724ec723 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -100,7 +100,7 @@ for a given data source. ## Creation of Additional Splunk HEC Destinations Additional Splunk HEC destinations can be dynamically created through environment variables. When set, the destinations will be -created with the `DESTID` appended, for example: `d_hec_FOO`. These destinations can then be specified for use (along with any other +created with the `DESTID` appended, for example: `d_hec_fmt_FOO`. These destinations can then be specified for use (along with any other destinations created locally) either globally or per source. See the "Alternate Destination Use" in the next section for details. | Variable | Values | Description | From d7ff96ee752b83afad16b505602ffc30d8115b2d Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 13 Jul 2021 13:23:05 -0400 Subject: [PATCH 11/12] chore: cleanup template --- .../conf.d/sc4slib/lp_dest_filtered_alts_select/plugin.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/etc/conf.d/sc4slib/lp_dest_filtered_alts_select/plugin.jinja b/package/etc/conf.d/sc4slib/lp_dest_filtered_alts_select/plugin.jinja index 2548acd17d..9ff65c01af 100644 --- a/package/etc/conf.d/sc4slib/lp_dest_filtered_alts_select/plugin.jinja +++ b/package/etc/conf.d/sc4slib/lp_dest_filtered_alts_select/plugin.jinja @@ -11,13 +11,13 @@ log{ "${.dest_key}" eq "{{ dest_key }}" }; - {%- for f in filters %} + {% for f in filters %} log { filter({{ f }}); {%- for destination in filters[f] %} destination({{ destination }}); {%- endfor %} }; - {%- endfor %} + {% endfor %} flags(catchall,flow-control, final); }; From 8a19f4915dad9b069d96f9876142cc978a686a4c Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Tue, 13 Jul 2021 14:27:13 -0400 Subject: [PATCH 12/12] fix(cisco_ios): Correct failure to open port for CISCO_IOS key --- package/sbin/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 88a8f9c7e1..a844492b85 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -49,9 +49,9 @@ if [ -n "${SC4S_ARCHIVE_CISCO_ASA_LEGACY}" ]; then export SC4S_ARCHIVE_CISCO_ASA if [ -n "${SC4S_DEST_CISCO_ASA_LEGACY_HEC}" ]; then export SC4S_DEST_CISCO_ASA_HEC=$SC4S_DEST_CISCO_ASA_LEGACY_HEC; fi export SC4S_LISTEN_CISCO_IOS_TCP_PORT=$(join_by , $SC4S_LISTEN_CISCO_APIC_TCP_PORT $SC4S_LISTEN_CISCO_NX_OS_TCP_PORT $SC4S_LISTEN_CISCO_IOS_TCP_PORT) -[ ! "$SC4S_LISTEN_CISCO_IOS_TCP_PORT" ] || unset SC4S_LISTEN_CISCO_IOS_TCP_PORT +[ -z "$SC4S_LISTEN_CISCO_IOS_TCP_PORT" ] && unset SC4S_LISTEN_CISCO_IOS_TCP_PORT export SC4S_LISTEN_CISCO_IOS_UDP_PORT=$(join_by , $SC4S_LISTEN_CISCO_APIC_UDP_PORT $SC4S_LISTEN_CISCO_NX_OS_UDP_PORT $SC4S_LISTEN_CISCO_IOS_UDP_PORT) -[ ! "$SC4S_LISTEN_CISCO_IOS_UDP_PORT" ] || unset SC4S_LISTEN_CISCO_IOS_UDP_PORT +[ -z "$SC4S_LISTEN_CISCO_IOS_UDP_PORT" ] && unset SC4S_LISTEN_CISCO_IOS_UDP_PORT # The unique port environment variables associated with SC4S_LISTEN__6587_PORT will be renamed to # SC4S_LISTEN__RFC6587_PORT to indicate compliance with the RFC.