Fixes: #1298 - Add SnipeIT source (#1299)

### Summary

Adding new intel module for SnipeIT source.

### Related issues
#1298
___
Read through our [developer
docs](https://lyft.github.io/cartography/dev/developer-guide.html)

- [x] PR Title starts with "Fixes: [issue number]"

If you are modifying or implementing a new intel module
- [x] Update the
[schema](https://github.com/lyft/cartography/tree/master/docs/root/modules)
and
[readme](https://github.com/lyft/cartography/blob/master/docs/schema/README.md)
- [x] Use our NodeSchema [data
model](https://lyft.github.io/cartography/dev/writing-intel-modules.html#defining-a-node)
- [x] Use specialized functions `get_`, `transform_`, `load_`, and
`cleanup_` functions
- [x] Add
[tests](https://lyft.github.io/cartography/dev/writing-intel-modules.html#making-tests)
- [x] Unit tests: Test your `transform_` function with sample data (Not
applicable)
  - Integration tests
- [x] Use our test [helper
functions](https://github.com/lyft/cartography/blob/master/tests/integration/util.py)
    - [x] Test a cleanup job

README.md

@@ -34,7 +34,8 @@ Start [here](https://lyft.github.io/cartography/install.html).
 - [Lastpass](https://lyft.github.io/cartography/modules/lastpass/index.html) - users
 - [BigFix](https://lyft.github.io/cartography/modules/bigfix/index.html) - Computers
 - [Duo](https://lyft.github.io/cartography/modules/duo/index.html) - Users, Groups, Endpoints
-
+- [Kandji](https://lyft.github.io/cartography/modules/kandji/index.html) - Devices
+- [SnipeIT](https://lyft.github.io/cartography/modules/snipeit/index.html) - Users, Assets
 
 ## Usage
 Start with our [tutorial](https://lyft.github.io/cartography/usage/tutorial.html). Our [data schema](https://lyft.github.io/cartography/usage/schema.html) is a helpful reference when you get stuck.
@@ -74,6 +75,7 @@ and follow the instructions to sign the CLA.
 1. [MessageBird](https://messagebird.com)
 1. [Cloudanix](https://www.cloudanix.com/)
 1. [ZeusCloud](https://www.zeuscloud.io/)
+1. [Corelight](https://www.corelight.com/)
 1. {Your company here} :-)
 
 If your organization uses Cartography, please file a PR and update this list. Say hi on Slack too!

cartography/cli.py

@@ -541,6 +541,28 @@ def _build_parser(self):
                 'Required if you are using the Semgrep intel module. Ignored otherwise.'
             ),
         )
+        parser.add_argument(
+            '--snipeit-base-uri',
+            type=str,
+            default=None,
+            help=(
+                'Your SnipeIT base URI'
+                'Required if you are using the SnipeIT intel module. Ignored otherwise.'
+            ),
+        )
+        parser.add_argument(
+            '--snipeit-token-env-var',
+            type=str,
+            default=None,
+            help='The name of an environment variable containing token with which to authenticate to SnipeIT.',
+        )
+        parser.add_argument(
+            '--snipeit-tenant-id',
+            type=str,
+            default=None,
+            help='An ID for the SnipeIT tenant.',
+        )
+
         return parser
 
     def main(self, argv: str) -> int:
@@ -744,6 +766,26 @@ def main(self, argv: str) -> int:
         else:
             config.cve_api_key = None
 
+        # SnipeIT config
+        if config.snipeit_base_uri:
+            if config.snipeit_token_env_var:
+                logger.debug(
+                    "Reading SnipeIT API token from environment variable '%s'.",
+                    config.snipeit_token_env_var,
+                )
+                config.snipeit_token = os.environ.get(config.snipeit_token_env_var)
+            elif os.environ.get('SNIPEIT_TOKEN'):
+                logger.debug(
+                    "Reading SnipeIT API token from environment variable 'SNIPEIT_TOKEN'.",
+                )
+                config.snipeit_token = os.environ.get('SNIPEIT_TOKEN')
+            else:
+                logger.warning("A SnipeIT base URI was provided but a token was not.")
+                config.kandji_token = None
+        else:
+            logger.warning("A SnipeIT base URI was not provided.")
+            config.snipeit_base_uri = None
+
         # Run cartography
         try:
             return cartography.sync.run_with_config(self.sync, config)

cartography/config.py

@@ -111,6 +111,12 @@ class Config:
     :param duo_api_hostname: The Duo api hostname, e.g. "api-abc123.duosecurity.com". Optional.
     :param semgrep_app_token: The Semgrep api token. Optional.
     :type semgrep_app_token: str
+    :type snipeit_base_uri: string
+    :param snipeit_base_uri: SnipeIT data provider base URI. Optional.
+    :type snipeit_token: string
+    :param snipeit_token: Token used to authenticate to the SnipeIT data provider. Optional.
+    :type snipeit_tenant_id: string
+    :param snipeit_tenant_id: Token used to authenticate to the SnipeIT data provider. Optional.
     """
 
     def __init__(
@@ -170,6 +176,9 @@ def __init__(
         duo_api_secret=None,
         duo_api_hostname=None,
         semgrep_app_token=None,
+        snipeit_base_uri=None,
+        snipeit_token=None,
+        snipeit_tenant_id=None,
     ):
         self.neo4j_uri = neo4j_uri
         self.neo4j_user = neo4j_user
@@ -226,3 +235,6 @@ def __init__(
         self.duo_api_secret = duo_api_secret
         self.duo_api_hostname = duo_api_hostname
         self.semgrep_app_token = semgrep_app_token
+        self.snipeit_base_uri = snipeit_base_uri
+        self.snipeit_token = snipeit_token
+        self.snipeit_tenant_id = snipeit_tenant_id

cartography/intel/snipeit/__init__.py

@@ -0,0 +1,30 @@
+import logging
+
+import neo4j
+
+from cartography.config import Config
+from cartography.intel.snipeit import asset
+from cartography.intel.snipeit import user
+from cartography.stats import get_stats_client
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+
+
+@timeit
+def start_snipeit_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
+    if config.snipeit_base_uri is None or config.snipeit_token is None or config.snipeit_tenant_id is None:
+        logger.warning(
+            "Required parameter(s) missing. Skipping sync.",
+        )
+        return
+
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+        "TENANT_ID": config.snipeit_tenant_id,
+    }
+
+    # Ingest SnipeIT users and assets
+    user.sync(neo4j_session, common_job_parameters, config.snipeit_base_uri, config.snipeit_token)
+    asset.sync(neo4j_session, common_job_parameters, config.snipeit_base_uri, config.snipeit_token)

cartography/intel/snipeit/asset.py

@@ -0,0 +1,74 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+
+from .util import call_snipeit_api
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.snipeit.asset import SnipeitAssetSchema
+from cartography.models.snipeit.tenant import SnipeitTenantSchema
+from cartography.util import timeit
+
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get(base_uri: str, token: str) -> List[Dict]:
+    api_endpoint = "/api/v1/hardware"
+    results: List[Dict[str, Any]] = []
+    while True:
+        offset = len(results)
+        api_endpoint = f"{api_endpoint}?order='asc'&offset={offset}"
+        response = call_snipeit_api(api_endpoint, base_uri, token)
+        results.extend(response['rows'])
+
+        total = response['total']
+        results_count = len(results)
+        if results_count >= total:
+            break
+
+    return results
+
+
+@timeit
+def load_assets(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+    data: List[Dict[str, Any]],
+) -> None:
+    # Create the SnipeIT Tenant
+    load(
+        neo4j_session,
+        SnipeitTenantSchema(),
+        [{'id': common_job_parameters["TENANT_ID"]}],
+        lastupdated=common_job_parameters["UPDATE_TAG"],
+    )
+
+    load(
+        neo4j_session,
+        SnipeitAssetSchema(),
+        data,
+        lastupdated=common_job_parameters["UPDATE_TAG"],
+        TENANT_ID=common_job_parameters["TENANT_ID"],
+    )
+
+
+@timeit
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
+    GraphJob.from_node_schema(SnipeitAssetSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+    base_uri: str,
+    token: str,
+) -> None:
+    assets = get(base_uri=base_uri, token=token)
+    load_assets(neo4j_session=neo4j_session, common_job_parameters=common_job_parameters, data=assets)
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/snipeit/user.py

@@ -0,0 +1,75 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+
+import neo4j
+
+from .util import call_snipeit_api
+from cartography.client.core.tx import load
+from cartography.graph.job import GraphJob
+from cartography.models.snipeit.tenant import SnipeitTenantSchema
+from cartography.models.snipeit.user import SnipeitUserSchema
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def get(base_uri: str, token: str) -> List[Dict]:
+    api_endpoint = "/api/v1/users"
+    results: List[Dict[str, Any]] = []
+    while True:
+        offset = len(results)
+        api_endpoint = f"{api_endpoint}?order='asc'&offset={offset}"
+        response = call_snipeit_api(api_endpoint, base_uri, token)
+        results.extend(response['rows'])
+
+        total = response['total']
+        results_count = len(results)
+        if results_count >= total:
+            break
+
+    return results
+
+
+@timeit
+def load_users(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+    data: List[Dict[str, Any]],
+) -> None:
+    logger.debug(data[0])
+
+    # Create the SnipeIT Tenant
+    load(
+        neo4j_session,
+        SnipeitTenantSchema(),
+        [{'id': common_job_parameters["TENANT_ID"]}],
+        lastupdated=common_job_parameters["UPDATE_TAG"],
+    )
+
+    load(
+        neo4j_session,
+        SnipeitUserSchema(),
+        data,
+        lastupdated=common_job_parameters["UPDATE_TAG"],
+        TENANT_ID=common_job_parameters["TENANT_ID"],
+    )
+
+
+@timeit
+def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
+    GraphJob.from_node_schema(SnipeitUserSchema(), common_job_parameters).run(neo4j_session)
+
+
+@timeit
+def sync(
+    neo4j_session: neo4j.Session,
+    common_job_parameters: Dict,
+    base_uri: str,
+    token: str,
+) -> None:
+    users = get(base_uri=base_uri, token=token)
+    load_users(neo4j_session, common_job_parameters, users)
+    cleanup(neo4j_session, common_job_parameters)

cartography/intel/snipeit/util.py

@@ -0,0 +1,35 @@
+import logging
+from typing import Any
+from typing import Dict
+
+import requests
+
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+# Connect and read timeouts of 60 seconds each; see https://requests.readthedocs.io/en/master/user/advanced/#timeouts
+_TIMEOUT = (60, 60)
+
+
+@timeit
+def call_snipeit_api(api_and_parameters: str, base_uri: str, token: str) -> Dict[str, Any]:
+    uri = base_uri + api_and_parameters
+    try:
+        logger.debug(
+            "SnipeIT: Get %s", uri,
+        )
+        response = requests.get(
+            uri,
+            headers={
+                'Accept': 'application/json',
+                'Authorization': f'Bearer {token}',
+            },
+            timeout=_TIMEOUT,
+        )
+    except requests.exceptions.Timeout:
+        # Add context and re-raise for callers to handle
+        logger.warning(f"SnipeIT: requests.get('{uri}') timed out.")
+        raise
+    # if call failed, use requests library to raise an exception
+    response.raise_for_status()
+    return response.json()