Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes: #1298 - Add SnipeIT source #1299

Merged
merged 30 commits into from
Aug 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
a95c8f5
feat: adding snipeit models
chandanchowdhury Apr 26, 2024
c4f9a95
feat: adding snipeit intel modules
chandanchowdhury Apr 26, 2024
3a877ad
feat: adding snipeit parameters and sync
chandanchowdhury Apr 26, 2024
b6ab585
test: adding snipeit test data
chandanchowdhury Apr 26, 2024
31ca134
test: adding snipeit tests
chandanchowdhury Apr 26, 2024
85850b1
docs: adding snipeit docs
chandanchowdhury Apr 26, 2024
deb1cc1
refactor: lint
chandanchowdhury Apr 26, 2024
e724915
refactor: mo lint
chandanchowdhury Apr 26, 2024
69fb1dc
Merge branch 'master' into snipeit
chandanchowdhury May 13, 2024
f584572
Merge branch 'master' into snipeit
chandanchowdhury May 15, 2024
bac45e4
Merge branch 'master' into snipeit
chandanchowdhury Jun 4, 2024
75b7f88
Merge branch 'master' into snipeit
chandanchowdhury Jun 17, 2024
fd9e02a
Merge branch 'master' into snipeit
chandanchowdhury Jun 18, 2024
a2be131
Merge branch 'master' into snipeit
chandanchowdhury Jun 18, 2024
2612ba2
docs: updating platform and company
chandanchowdhury Jun 21, 2024
90653bb
Merge branch 'master' into snipeit
chandanchowdhury Jul 3, 2024
96dcd81
Merge branch 'master' into snipeit
chandanchowdhury Jul 8, 2024
3e3eedb
Merge branch 'master' into snipeit
chandanchowdhury Jul 8, 2024
60849c5
Merge branch 'master' into snipeit
chandanchowdhury Jul 18, 2024
815dc7f
Merge branch 'master' into snipeit
achantavy Jul 20, 2024
48f4a86
Merge branch 'master' into snipeit
chandanchowdhury Jul 22, 2024
9a89efe
Merge branch 'master' into snipeit
chandanchowdhury Jul 23, 2024
b14576e
fix: add missing at sign
chandanchowdhury Aug 11, 2024
56833d5
fix: relationship direction
chandanchowdhury Aug 11, 2024
34ae2c6
fix: bug - snipeit returns max 500 items
chandanchowdhury Aug 11, 2024
d565c86
Merge branch 'master' into snipeit
chandanchowdhury Aug 11, 2024
9ae2c82
docs: relationship direction
chandanchowdhury Aug 11, 2024
4c55613
Merge branch 'snipeit' of github.com:chandanchowdhury/cartography int…
chandanchowdhury Aug 11, 2024
5f962b9
test: fix relationship names
chandanchowdhury Aug 12, 2024
f1353a8
docs: update relationship direction
chandanchowdhury Aug 21, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ Start [here](https://lyft.github.io/cartography/install.html).
- [Lastpass](https://lyft.github.io/cartography/modules/lastpass/index.html) - users
- [BigFix](https://lyft.github.io/cartography/modules/bigfix/index.html) - Computers
- [Duo](https://lyft.github.io/cartography/modules/duo/index.html) - Users, Groups, Endpoints

- [Kandji](https://lyft.github.io/cartography/modules/kandji/index.html) - Devices
- [SnipeIT](https://lyft.github.io/cartography/modules/snipeit/index.html) - Users, Assets

## Usage
Start with our [tutorial](https://lyft.github.io/cartography/usage/tutorial.html). Our [data schema](https://lyft.github.io/cartography/usage/schema.html) is a helpful reference when you get stuck.
Expand Down Expand Up @@ -74,6 +75,7 @@ and follow the instructions to sign the CLA.
1. [MessageBird](https://messagebird.com)
1. [Cloudanix](https://www.cloudanix.com/)
1. [ZeusCloud](https://www.zeuscloud.io/)
1. [Corelight](https://www.corelight.com/)
1. {Your company here} :-)

If your organization uses Cartography, please file a PR and update this list. Say hi on Slack too!
42 changes: 42 additions & 0 deletions cartography/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,28 @@ def _build_parser(self):
'Required if you are using the Semgrep intel module. Ignored otherwise.'
),
)
parser.add_argument(
'--snipeit-base-uri',
type=str,
default=None,
help=(
'Your SnipeIT base URI'
'Required if you are using the SnipeIT intel module. Ignored otherwise.'
),
)
parser.add_argument(
'--snipeit-token-env-var',
type=str,
default=None,
help='The name of an environment variable containing token with which to authenticate to SnipeIT.',
)
parser.add_argument(
'--snipeit-tenant-id',
type=str,
default=None,
help='An ID for the SnipeIT tenant.',
)

return parser

def main(self, argv: str) -> int:
Expand Down Expand Up @@ -744,6 +766,26 @@ def main(self, argv: str) -> int:
else:
config.cve_api_key = None

# SnipeIT config
if config.snipeit_base_uri:
if config.snipeit_token_env_var:
logger.debug(
"Reading SnipeIT API token from environment variable '%s'.",
config.snipeit_token_env_var,
)
config.snipeit_token = os.environ.get(config.snipeit_token_env_var)
elif os.environ.get('SNIPEIT_TOKEN'):
logger.debug(
"Reading SnipeIT API token from environment variable 'SNIPEIT_TOKEN'.",
)
config.snipeit_token = os.environ.get('SNIPEIT_TOKEN')
else:
logger.warning("A SnipeIT base URI was provided but a token was not.")
config.kandji_token = None
else:
logger.warning("A SnipeIT base URI was not provided.")
config.snipeit_base_uri = None

# Run cartography
try:
return cartography.sync.run_with_config(self.sync, config)
Expand Down
12 changes: 12 additions & 0 deletions cartography/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,12 @@ class Config:
:param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional.
:param semgrep_app_token: The Semgrep api token. Optional.
:type semgrep_app_token: str
:type snipeit_base_uri: string
:param snipeit_base_uri: SnipeIT data provider base URI. Optional.
:type snipeit_token: string
:param snipeit_token: Token used to authenticate to the SnipeIT data provider. Optional.
:type snipeit_tenant_id: string
:param snipeit_tenant_id: Token used to authenticate to the SnipeIT data provider. Optional.
"""

def __init__(
Expand Down Expand Up @@ -170,6 +176,9 @@ def __init__(
duo_api_secret=None,
duo_api_hostname=None,
semgrep_app_token=None,
snipeit_base_uri=None,
snipeit_token=None,
snipeit_tenant_id=None,
):
self.neo4j_uri = neo4j_uri
self.neo4j_user = neo4j_user
Expand Down Expand Up @@ -226,3 +235,6 @@ def __init__(
self.duo_api_secret = duo_api_secret
self.duo_api_hostname = duo_api_hostname
self.semgrep_app_token = semgrep_app_token
self.snipeit_base_uri = snipeit_base_uri
self.snipeit_token = snipeit_token
self.snipeit_tenant_id = snipeit_tenant_id
30 changes: 30 additions & 0 deletions cartography/intel/snipeit/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import logging

import neo4j

from cartography.config import Config
from cartography.intel.snipeit import asset
from cartography.intel.snipeit import user
from cartography.stats import get_stats_client
from cartography.util import timeit

logger = logging.getLogger(__name__)
stat_handler = get_stats_client(__name__)


@timeit
def start_snipeit_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
if config.snipeit_base_uri is None or config.snipeit_token is None or config.snipeit_tenant_id is None:
logger.warning(
"Required parameter(s) missing. Skipping sync.",
)
return

common_job_parameters = {
"UPDATE_TAG": config.update_tag,
"TENANT_ID": config.snipeit_tenant_id,
}

# Ingest SnipeIT users and assets
user.sync(neo4j_session, common_job_parameters, config.snipeit_base_uri, config.snipeit_token)
asset.sync(neo4j_session, common_job_parameters, config.snipeit_base_uri, config.snipeit_token)
74 changes: 74 additions & 0 deletions cartography/intel/snipeit/asset.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import logging
from typing import Any
from typing import Dict
from typing import List

import neo4j

from .util import call_snipeit_api
from cartography.client.core.tx import load
from cartography.graph.job import GraphJob
from cartography.models.snipeit.asset import SnipeitAssetSchema
from cartography.models.snipeit.tenant import SnipeitTenantSchema
from cartography.util import timeit


logger = logging.getLogger(__name__)


@timeit
def get(base_uri: str, token: str) -> List[Dict]:
api_endpoint = "/api/v1/hardware"
results: List[Dict[str, Any]] = []
while True:
offset = len(results)
api_endpoint = f"{api_endpoint}?order='asc'&offset={offset}"
response = call_snipeit_api(api_endpoint, base_uri, token)
results.extend(response['rows'])

total = response['total']
results_count = len(results)
if results_count >= total:
break

return results


@timeit
def load_assets(
neo4j_session: neo4j.Session,
common_job_parameters: Dict,
data: List[Dict[str, Any]],
) -> None:
# Create the SnipeIT Tenant
load(
neo4j_session,
SnipeitTenantSchema(),
[{'id': common_job_parameters["TENANT_ID"]}],
lastupdated=common_job_parameters["UPDATE_TAG"],
)

load(
neo4j_session,
SnipeitAssetSchema(),
data,
lastupdated=common_job_parameters["UPDATE_TAG"],
TENANT_ID=common_job_parameters["TENANT_ID"],
)


@timeit
def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
GraphJob.from_node_schema(SnipeitAssetSchema(), common_job_parameters).run(neo4j_session)


@timeit
def sync(
neo4j_session: neo4j.Session,
common_job_parameters: Dict,
base_uri: str,
token: str,
) -> None:
assets = get(base_uri=base_uri, token=token)
load_assets(neo4j_session=neo4j_session, common_job_parameters=common_job_parameters, data=assets)
cleanup(neo4j_session, common_job_parameters)
75 changes: 75 additions & 0 deletions cartography/intel/snipeit/user.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import logging
from typing import Any
from typing import Dict
from typing import List

import neo4j

from .util import call_snipeit_api
from cartography.client.core.tx import load
from cartography.graph.job import GraphJob
from cartography.models.snipeit.tenant import SnipeitTenantSchema
from cartography.models.snipeit.user import SnipeitUserSchema
from cartography.util import timeit

logger = logging.getLogger(__name__)


@timeit
def get(base_uri: str, token: str) -> List[Dict]:
api_endpoint = "/api/v1/users"
results: List[Dict[str, Any]] = []
while True:
offset = len(results)
api_endpoint = f"{api_endpoint}?order='asc'&offset={offset}"
response = call_snipeit_api(api_endpoint, base_uri, token)
results.extend(response['rows'])

total = response['total']
results_count = len(results)
if results_count >= total:
break

return results


@timeit
def load_users(
neo4j_session: neo4j.Session,
common_job_parameters: Dict,
data: List[Dict[str, Any]],
) -> None:
logger.debug(data[0])

# Create the SnipeIT Tenant
load(
neo4j_session,
SnipeitTenantSchema(),
[{'id': common_job_parameters["TENANT_ID"]}],
lastupdated=common_job_parameters["UPDATE_TAG"],
)

load(
neo4j_session,
SnipeitUserSchema(),
data,
lastupdated=common_job_parameters["UPDATE_TAG"],
TENANT_ID=common_job_parameters["TENANT_ID"],
)


@timeit
def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
GraphJob.from_node_schema(SnipeitUserSchema(), common_job_parameters).run(neo4j_session)


@timeit
def sync(
neo4j_session: neo4j.Session,
common_job_parameters: Dict,
base_uri: str,
token: str,
) -> None:
users = get(base_uri=base_uri, token=token)
load_users(neo4j_session, common_job_parameters, users)
cleanup(neo4j_session, common_job_parameters)
35 changes: 35 additions & 0 deletions cartography/intel/snipeit/util.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import logging
from typing import Any
from typing import Dict

import requests

from cartography.util import timeit

logger = logging.getLogger(__name__)
# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
_TIMEOUT = (60, 60)


@timeit
def call_snipeit_api(api_and_parameters: str, base_uri: str, token: str) -> Dict[str, Any]:
chandanchowdhury marked this conversation as resolved.
Show resolved Hide resolved
uri = base_uri + api_and_parameters
try:
logger.debug(
"SnipeIT: Get %s", uri,
chandanchowdhury marked this conversation as resolved.
Show resolved Hide resolved
)
response = requests.get(
uri,
headers={
'Accept': 'application/json',
'Authorization': f'Bearer {token}',
},
timeout=_TIMEOUT,
)
except requests.exceptions.Timeout:
# Add context and re-raise for callers to handle
logger.warning(f"SnipeIT: requests.get('{uri}') timed out.")
raise
# if call failed, use requests library to raise an exception
response.raise_for_status()
return response.json()
Empty file.
Loading
Loading