From ee140e79c3541e004d6b84a81b958201820f29a1 Mon Sep 17 00:00:00 2001 From: Thomas Chopitea Date: Wed, 25 Oct 2023 10:03:23 +0200 Subject: [PATCH] More options for Timesketch auth (user/pass) (#794) * docker release * Some changes * release docker container * Add userpass login to the TS module / recipes * fix recipe params * slower but more stable * image name * oops * change image name * dockerfile on branch * Update all recipes * Fix tests * silence linter * Fix type * remove docker files from PR * Appease pylint * Add docstrings * Add default dev endpoint for timesketch * fix test * Fix pylint --- data/recipes/aws_logging_ts.json | 6 +++ data/recipes/aws_turbinia_ts.json | 6 +++ data/recipes/azure_logging_ts.json | 6 +++ data/recipes/bigquery_ts.json | 6 +++ data/recipes/gcp_logging_cloudaudit_ts.json | 6 +++ data/recipes/gcp_logging_cloudsql_ts.json | 6 +++ data/recipes/gcp_logging_gce_instance_ts.json | 6 +++ data/recipes/gcp_logging_gce_ts.json | 6 +++ data/recipes/gcp_logging_ts.json | 6 +++ data/recipes/gcp_turbinia_disk_copy_ts.json | 6 +++ data/recipes/gcp_turbinia_ts.json | 6 +++ data/recipes/grr_artifact_ts.json | 11 ++++- data/recipes/grr_huntresults_ts.json | 6 +++ data/recipes/grr_timeline_ts.json | 18 +++++-- data/recipes/gsheets_ts.json | 6 +++ data/recipes/plaso_ts.json | 12 +++-- data/recipes/upload_ts.json | 6 +++ data/recipes/upload_web_ts.json | 11 ++++- data/recipes/vt_evtx_ts.json | 6 +++ data/recipes/workspace_meet_ts.json | 6 +++ data/recipes/workspace_user_activity_ts.json | 6 +++ data/recipes/workspace_user_drive_ts.json | 6 +++ data/recipes/workspace_user_login_ts.json | 6 +++ dftimewolf/lib/enhancers/timesketch.py | 47 ++++++++++++------- dftimewolf/lib/exporters/timesketch.py | 25 +++++++--- tests/lib/exporters/timesketch.py | 34 ++++++++++---- 26 files changed, 230 insertions(+), 42 deletions(-) diff --git a/data/recipes/aws_logging_ts.json b/data/recipes/aws_logging_ts.json index 31aab01e6..69e274e48 100644 --- a/data/recipes/aws_logging_ts.json +++ b/data/recipes/aws_logging_ts.json @@ -32,6 +32,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -45,6 +48,9 @@ ["--end_time", "End time for the query.", null, {"format": "datetime", "format_string": "%Y-%m-%d %H:%M:%S", "after": "@start_time"}], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true] ] diff --git a/data/recipes/aws_turbinia_ts.json b/data/recipes/aws_turbinia_ts.json index aed258f10..4399a0d8e 100644 --- a/data/recipes/aws_turbinia_ts.json +++ b/data/recipes/aws_turbinia_ts.json @@ -79,6 +79,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -96,6 +99,9 @@ ["--aws_profile", "Source AWS profile.", null], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--request_ids", "Comma separated Turbinia request identifiers to process.", null, {"format": "regex", "comma_separated": true, "regex": "^[a-f0-9]{32}$"}], ["--turbinia_recipe", "The Turbinia recipe name to use for evidence processing.", null], diff --git a/data/recipes/azure_logging_ts.json b/data/recipes/azure_logging_ts.json index d1b14a5e2..cf193a70e 100644 --- a/data/recipes/azure_logging_ts.json +++ b/data/recipes/azure_logging_ts.json @@ -24,6 +24,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -35,6 +38,9 @@ ["--profile_name", "A profile name to use when looking for Azure credentials.", null], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true] ] diff --git a/data/recipes/bigquery_ts.json b/data/recipes/bigquery_ts.json index d88470bf5..ea401e56b 100644 --- a/data/recipes/bigquery_ts.json +++ b/data/recipes/bigquery_ts.json @@ -25,6 +25,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -36,6 +39,9 @@ ["description", "Human-readable description of the query.", null], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true] ] diff --git a/data/recipes/gcp_logging_cloudaudit_ts.json b/data/recipes/gcp_logging_cloudaudit_ts.json index 00cc954cf..9e564ac14 100644 --- a/data/recipes/gcp_logging_cloudaudit_ts.json +++ b/data/recipes/gcp_logging_cloudaudit_ts.json @@ -31,6 +31,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -42,6 +45,9 @@ ["end_date", "End date (yyyy-mm-ddTHH:MM:SSZ).", null, {"format": "datetime", "format_string": "%Y-%m-%dT%H:%M:%SZ", "after": "@start_date"}], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true], ["--backoff", "If GCP Cloud Logging API query limits are exceeded, retry with an increased delay between each query to try complete the query at a slower rate.", false], diff --git a/data/recipes/gcp_logging_cloudsql_ts.json b/data/recipes/gcp_logging_cloudsql_ts.json index 6beccb29f..6b8efe8ad 100644 --- a/data/recipes/gcp_logging_cloudsql_ts.json +++ b/data/recipes/gcp_logging_cloudsql_ts.json @@ -34,6 +34,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -46,6 +49,9 @@ ["end_date", "End date (yyyy-mm-ddTHH:MM:SSZ).", null, {"format": "datetime", "format_string": "%Y-%m-%dT%H:%M:%SZ", "after": "@start_date"}], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true], ["--backoff", "If GCP Cloud Logging API query limits are exceeded, retry with an increased delay between each query to try complete the query at a slower rate.", false], diff --git a/data/recipes/gcp_logging_gce_instance_ts.json b/data/recipes/gcp_logging_gce_instance_ts.json index ee6a6b324..55d79ed54 100644 --- a/data/recipes/gcp_logging_gce_instance_ts.json +++ b/data/recipes/gcp_logging_gce_instance_ts.json @@ -31,6 +31,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -41,6 +44,9 @@ ["instance_id", "Identifier for GCE instance (Instance ID).", null, {"format": "regex", "regex": "^[a-z][-a-z0-9]{0,61}[a-z0-9]?$"}], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true], ["--backoff", "If GCP Cloud Logging API query limits are exceeded, retry with an increased delay between each query to try complete the query at a slower rate.", false], diff --git a/data/recipes/gcp_logging_gce_ts.json b/data/recipes/gcp_logging_gce_ts.json index 7b8cb4adb..eadb9caee 100644 --- a/data/recipes/gcp_logging_gce_ts.json +++ b/data/recipes/gcp_logging_gce_ts.json @@ -32,6 +32,9 @@ "incident_id": "@incident_id", "analyzers": null, "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "wait_for_timelines": "@wait_for_timelines" } @@ -42,6 +45,9 @@ ["end_date", "End date (yyyy-mm-ddTHH:MM:SSZ).", null, {"format": "datetime", "format_string": "%Y-%m-%dT%H:%M:%SZ", "after": "@start_date"}], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true], ["--backoff", "If GCP Cloud Logging API query limits are exceeded, retry with an increased delay between each query to try complete the query at a slower rate.", false], diff --git a/data/recipes/gcp_logging_ts.json b/data/recipes/gcp_logging_ts.json index 137d1408a..a7f3e3c73 100644 --- a/data/recipes/gcp_logging_ts.json +++ b/data/recipes/gcp_logging_ts.json @@ -30,6 +30,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": "@analyzers", "wait_for_timelines": "@wait_for_timelines" @@ -43,6 +46,9 @@ ["--delay", "Number of seconds to wait between each GCP Cloud Logging query to avoid hitting API query limits", 0, {"format": "regex", "regex": "^\\d+$"}], ["--analyzers", "Timesketch analyzers to run.", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true] diff --git a/data/recipes/gcp_turbinia_disk_copy_ts.json b/data/recipes/gcp_turbinia_disk_copy_ts.json index 6b5333fc7..57c9bc27f 100644 --- a/data/recipes/gcp_turbinia_disk_copy_ts.json +++ b/data/recipes/gcp_turbinia_disk_copy_ts.json @@ -70,6 +70,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -86,6 +89,9 @@ ["--turbinia_api", "Turbinia API server endpoint.", "http://127.0.0.1:8000"], ["--incident_id", "Incident ID (used for Timesketch description and to label the VM with).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--create_analysis_vm", "Create an analysis VM in the destination project.", true], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true], diff --git a/data/recipes/gcp_turbinia_ts.json b/data/recipes/gcp_turbinia_ts.json index b8ebd081f..3551ad657 100644 --- a/data/recipes/gcp_turbinia_ts.json +++ b/data/recipes/gcp_turbinia_ts.json @@ -29,6 +29,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -41,6 +44,9 @@ ["--request_ids", "Comma separated Turbinia request identifiers to process. This parameter can only be used if --disk_names is not provided.", null, {"format": "regex", "comma_separated": true, "regex": "^[a-f0-9]{32}$"}], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--turbinia_recipe", "The Turbinia recipe name to use for evidence processing.", null], ["--turbinia_auth", "Flag to indicate whether Turbinia API server requires authentication.", false], diff --git a/data/recipes/grr_artifact_ts.json b/data/recipes/grr_artifact_ts.json index 6d04875b7..03a4b705b 100644 --- a/data/recipes/grr_artifact_ts.json +++ b/data/recipes/grr_artifact_ts.json @@ -24,14 +24,17 @@ "name": "LocalPlasoProcessor", "args": { "timezone": null, - "use_docker": true + "use_docker": "@user_docker" } }, { "wants": ["LocalPlasoProcessor"], "name": "TimesketchExporter", "args": { "incident_id": "@reason", - "token_password": "@token_password", + "token_password": null, + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": "@analyzers", "wait_for_timelines": "@wait_for_timelines" @@ -48,12 +51,16 @@ ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true], ["--analyzers", "Timesketch analyzers to run", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--grr_server_url", "GRR endpoint.", "http://localhost:8000", {"format": "url"}], ["--verify", "Whether to verify the GRR TLS certificate.", true], ["--skip_offline_clients", "Whether to skip clients that are offline.", false], ["--grr_username", "GRR username", "admin"], ["--grr_password", "GRR password", "admin"], + ["--user_docker", "Whether the LocalPlasoProcessor should use Docker or not.", true], ["--max_file_size", "Maximum size of files to collect (in bytes).", 5368709120, {"format": "regex", "regex": "^\\d+$"}] ] } diff --git a/data/recipes/grr_huntresults_ts.json b/data/recipes/grr_huntresults_ts.json index eae4754e3..6af28519f 100644 --- a/data/recipes/grr_huntresults_ts.json +++ b/data/recipes/grr_huntresults_ts.json @@ -27,6 +27,9 @@ "args": { "incident_id": "@reason", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -36,6 +39,9 @@ ["hunt_id", "ID of GRR Hunt results to fetch.", null, {"format": "regex", "comma_separated": true, "regex": "^[0-9A-F]{16}$"}], ["reason", "Reason for exporting hunt (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true], ["--approvers", "Emails for GRR approval request.", null], diff --git a/data/recipes/grr_timeline_ts.json b/data/recipes/grr_timeline_ts.json index 8346cce02..176f4efd3 100644 --- a/data/recipes/grr_timeline_ts.json +++ b/data/recipes/grr_timeline_ts.json @@ -23,7 +23,7 @@ "name": "LocalPlasoProcessor", "args": { "timezone": null, - "use_docker": true + "use_docker": "@use_docker" } }, { "wants": ["LocalPlasoProcessor"], @@ -33,7 +33,10 @@ "token_password": "@token_password", "sketch_id": "@sketch_id", "analyzers": null, - "wait_for_timelines": "@wait_for_timelines" + "wait_for_timelines": "@wait_for_timelines", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password" } }, { "wants": ["TimesketchExporter"], @@ -45,7 +48,10 @@ "include_stories": false, "token_password": "@token_password", "max_checks": 0, - "formatter": "html" + "formatter": "html", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password" } }], "args": [ @@ -56,10 +62,14 @@ ["--approvers", "Comma-separated list of usernames to ask for approval.", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], ["--grr_server_url", "GRR endpoint.", "http://localhost:8000", {"format": "url"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--timesketch_quick", "Skip waiting for analyzers to complete their run.", false], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true], ["--grr_username", "GRR username.", "admin"], - ["--grr_password", "GRR password.", "admin"] + ["--grr_password", "GRR password.", "admin"], + ["--user_docker", "Whether the LocalPlasoProcessor should use Docker or not.", true] ] } diff --git a/data/recipes/gsheets_ts.json b/data/recipes/gsheets_ts.json index 2ba95dc89..c584daed6 100644 --- a/data/recipes/gsheets_ts.json +++ b/data/recipes/gsheets_ts.json @@ -17,6 +17,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -27,6 +30,9 @@ ["--sheet_names", "Comma-separated list sheet names to collect date from. If not set all sheets in the spreadsheet will be parsed.", []], ["--validate_columns", "Set to True to check for mandatory columns required by Timesketch while extracting data. Set to False to ignore validation. Default is True.", true], ["--sketch_id", "Sketch to which the timeline should be added", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with", ""], ["--incident_id", "Incident ID (used for Timesketch description)", null], ["--wait_for_timelines", "Whether to wait for timelines to finish processing.", true] diff --git a/data/recipes/plaso_ts.json b/data/recipes/plaso_ts.json index 2aac7fabe..52aa0c039 100644 --- a/data/recipes/plaso_ts.json +++ b/data/recipes/plaso_ts.json @@ -19,10 +19,13 @@ "wants": ["LocalPlasoProcessor"], "name": "TimesketchExporter", "args": { - "incident_id": "@incident_id", - "token_password": "@token_password", + "incident_id": "@reason", + "token_password": null, + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", - "analyzers": null, + "analyzers": "@analyzers", "wait_for_timelines": "@wait_for_timelines" } }], @@ -30,6 +33,9 @@ ["paths", "Comma-separated list of paths to process.", null], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true] ] diff --git a/data/recipes/upload_ts.json b/data/recipes/upload_ts.json index 7cd380347..0f44b5810 100644 --- a/data/recipes/upload_ts.json +++ b/data/recipes/upload_ts.json @@ -14,6 +14,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": "@analyzers", "wait_for_timelines": "@wait_for_timelines" @@ -23,6 +26,9 @@ ["files", "Comma-separated list of paths to CSV files or Plaso storage files.", null], ["--analyzers", "Timesketch analyzers to run.", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true] diff --git a/data/recipes/upload_web_ts.json b/data/recipes/upload_web_ts.json index f3a81fbd4..f513c6c1e 100644 --- a/data/recipes/upload_web_ts.json +++ b/data/recipes/upload_web_ts.json @@ -14,6 +14,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "wait_for_timelines": true, "analyzers": "browser_search,browser_timeframe,account_finder,phishy_domains,evtx_gap,login,win_crash,safebrowsing,chain," @@ -28,12 +31,18 @@ "include_stories": "@timesketch_include_stories", "token_password": "@token_password", "max_checks": "@analyzer_max_checks", - "formatter": "html" + "formatter": "html", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password" } }], "args": [ ["files", "Comma-separated list of paths to CSV files or Plaso storage files.", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--wait_for_analyzers", "Wait for analyzers until they complete their run, if set to False the TS enhancer will be skipped.", true], diff --git a/data/recipes/vt_evtx_ts.json b/data/recipes/vt_evtx_ts.json index 96e917fed..414dce876 100644 --- a/data/recipes/vt_evtx_ts.json +++ b/data/recipes/vt_evtx_ts.json @@ -27,6 +27,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -39,6 +42,9 @@ ["--vt_api_key", "Virustotal API key", "admin"], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true] ] diff --git a/data/recipes/workspace_meet_ts.json b/data/recipes/workspace_meet_ts.json index 850a1bb8f..bd318456d 100644 --- a/data/recipes/workspace_meet_ts.json +++ b/data/recipes/workspace_meet_ts.json @@ -26,6 +26,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -38,6 +41,9 @@ ["--end_time", "End time (yyyy-mm-ddTHH:MM:SSZ).", null, {"format": "datetime", "format_string": "%Y-%m-%dT%H:%M:%SZ", "after": "@start_time"}], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true] ] diff --git a/data/recipes/workspace_user_activity_ts.json b/data/recipes/workspace_user_activity_ts.json index 1422bbe13..d087e4bde 100644 --- a/data/recipes/workspace_user_activity_ts.json +++ b/data/recipes/workspace_user_activity_ts.json @@ -171,6 +171,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -184,6 +187,9 @@ ["--filter_expression", "Filter expression to use to query Workspace logs. See https://developers.google.com/admin-sdk/reports/reference/rest/v1/activities/list",""], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true] ] diff --git a/data/recipes/workspace_user_drive_ts.json b/data/recipes/workspace_user_drive_ts.json index 6b3a4ff06..05c7e8a4c 100644 --- a/data/recipes/workspace_user_drive_ts.json +++ b/data/recipes/workspace_user_drive_ts.json @@ -26,6 +26,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -39,6 +42,9 @@ ["--filter_expression", "Filter expression to use to query Workspace logs. See https://developers.google.com/admin-sdk/reports/reference/rest/v1/activities/list",""], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true] ] diff --git a/data/recipes/workspace_user_login_ts.json b/data/recipes/workspace_user_login_ts.json index 9054196bc..fd15e9392 100644 --- a/data/recipes/workspace_user_login_ts.json +++ b/data/recipes/workspace_user_login_ts.json @@ -26,6 +26,9 @@ "args": { "incident_id": "@incident_id", "token_password": "@token_password", + "endpoint": "@timesketch_endpoint", + "username": "@timesketch_username", + "password": "@timesketch_password", "sketch_id": "@sketch_id", "analyzers": null, "wait_for_timelines": "@wait_for_timelines" @@ -39,6 +42,9 @@ ["--filter_expression", "Filter expression to use to query Workspace logs. See https://developers.google.com/admin-sdk/reports/reference/rest/v1/activities/list",""], ["--incident_id", "Incident ID (used for Timesketch description).", null], ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}], + ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"], + ["--timesketch_username", "Username for Timesketch server.", null], + ["--timesketch_password", "Password for Timesketch server.", null], ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""], ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true] ] diff --git a/dftimewolf/lib/enhancers/timesketch.py b/dftimewolf/lib/enhancers/timesketch.py index f98b039a5..d58e71eb8 100644 --- a/dftimewolf/lib/enhancers/timesketch.py +++ b/dftimewolf/lib/enhancers/timesketch.py @@ -2,8 +2,11 @@ """Timesketch enhancer that exports Timesketch results.""" import time +from typing import TYPE_CHECKING +from typing import List +from typing import Optional -from typing import List, Optional, TYPE_CHECKING +from timesketch_api_client import client as ts_client from dftimewolf.lib import module from dftimewolf.lib import timesketch_utils @@ -11,14 +14,13 @@ from dftimewolf.lib.containers import containers from dftimewolf.lib.modules import manager as modules_manager - if TYPE_CHECKING: - from dftimewolf.lib import state as dftw_state - from timesketch_api_client import client + from timesketch_api_client import aggregation as ts_aggregation + from timesketch_api_client import search as ts_search from timesketch_api_client import sketch as ts_sketch from timesketch_api_client import story as ts_story - from timesketch_api_client import search as ts_search - from timesketch_api_client import aggregation as ts_aggregation + + from dftimewolf.lib import state as dftw_state class TimesketchEnhancer(module.BaseModule): @@ -43,7 +45,7 @@ class TimesketchEnhancer(module.BaseModule): # For pytype _formatter: utils.FormatterInterface - timesketch_api: "client.TimesketchApi" + timesketch_api: "ts_client.TimesketchApi" def __init__(self, state: "dftw_state.DFTimewolfState", @@ -57,14 +59,17 @@ def __init__(self, self._wait_for_analyzers = True # type: bool self._searches_to_skip = [] # type: List[str] - def SetUp(self, # pylint: disable=arguments-differ - wait_for_analyzers: bool=True, - searches_to_skip: str='', - aggregations_to_skip: str='', - include_stories: bool=False, - token_password: str='', - max_checks: int=0, - formatter: str='html') -> None: + def SetUp(self, # pylint: disable=arguments-differ,too-many-arguments + wait_for_analyzers: bool, + searches_to_skip: Optional[str], + aggregations_to_skip: Optional[str], + include_stories: bool, + token_password: Optional[str], + endpoint: Optional[str], + username: Optional[str], + password: Optional[str], + max_checks: int, + formatter: str) -> None: """Sets up a Timesketch Enhancer module. Args: @@ -84,6 +89,10 @@ def SetUp(self, # pylint: disable=arguments-differ Timesketch credential storage. Defaults to an empty string since the upstream library expects a string value. An empty string means a password will be generated by the upstream library. + endpoint: Timesketch server URL (e.g. http://localhost:5000/). + Optional when token_password is provided. + username: Timesketch username. Optional when token_password is provided. + password: Timesketch password. Optional when token_password is provided. max_checks (int): The enhancer will wait for analyzers to complete before attempting to collect data from Timesketch. The tool waits 3 seconds before each check, and by default the number of checks is 60, meaning @@ -94,8 +103,12 @@ def SetUp(self, # pylint: disable=arguments-differ be used for text formatting in reports. Valid options are: "html" or "markdown", defaults to "html". """ - self.timesketch_api = timesketch_utils.GetApiClient( - self.state, token_password=token_password) + if token_password: + self.timesketch_api = timesketch_utils.GetApiClient( + self.state, token_password=token_password) + elif endpoint and username and password: + self.timesketch_api = ts_client.TimesketchApi( + endpoint, username, password) if not (self.timesketch_api or self.timesketch_api.session): self.ModuleError( diff --git a/dftimewolf/lib/exporters/timesketch.py b/dftimewolf/lib/exporters/timesketch.py index 200d39a54..d997c6258 100644 --- a/dftimewolf/lib/exporters/timesketch.py +++ b/dftimewolf/lib/exporters/timesketch.py @@ -53,10 +53,13 @@ def __init__( # pylint: disable=arguments-differ def SetUp( self, - incident_id: None = None, - sketch_id: Optional[int] = 0, - analyzers: None = None, - token_password: str = '', + incident_id: str, + sketch_id: Optional[int], + analyzers: Optional[str], + token_password: Optional[str], + endpoint: Optional[str], + username: Optional[str], + password: Optional[str], wait_for_timelines: bool = False) -> None: """Setup a connection to a Timesketch server and create a sketch if needed. @@ -71,13 +74,23 @@ def SetUp( Timesketch credential storage. Defaults to an empty string since the upstream library expects a string value. An empty string means a password will be generated by the upstream library. + endpoint: Timesketch server URL (e.g. http://localhost:5000/). + Optional when token_password is provided. + username: Timesketch username. Optional when token_password is provided. + password: Timesketch password. Optional when token_password is provided. wait_for_timelines (bool): Whether to wait until timelines are processed in the Timesketch server or not. """ self.wait_for_timelines = wait_for_timelines - self.timesketch_api = timesketch_utils.GetApiClient( - self.state, token_password=token_password) + if token_password: + self.logger.info('Using token password from recipe config.') + self.timesketch_api = timesketch_utils.GetApiClient( + self.state, token_password=token_password) + elif endpoint and username and password: + self.timesketch_api = ts_client.TimesketchApi( + endpoint, username, password) + if not self.timesketch_api: self.ModuleError( 'Unable to get a Timesketch API client, try deleting the files ' diff --git a/tests/lib/exporters/timesketch.py b/tests/lib/exporters/timesketch.py index a13a9e25d..5ecf1c003 100644 --- a/tests/lib/exporters/timesketch.py +++ b/tests/lib/exporters/timesketch.py @@ -33,9 +33,13 @@ def testSetupWithSketchId(self, mock_GetApiClient): test_state = state.DFTimewolfState(config.Config) timesketch_exporter = timesketch.TimesketchExporter(test_state) timesketch_exporter.SetUp( - incident_id=None, + incident_id='ticketId', sketch_id=1234, - analyzers=None + analyzers=None, + token_password='blah', + endpoint=None, + username=None, + password=None, ) self.assertEqual(timesketch_exporter.sketch_id, 1234) mock_api_client.get_sketch.assert_called_with(1234) @@ -52,9 +56,13 @@ def testSetupWithReadonlySketchId(self, mock_GetApiClient): timesketch_exporter = timesketch.TimesketchExporter(test_state) with self.assertRaises(errors.DFTimewolfError) as error: timesketch_exporter.SetUp( - incident_id=None, + incident_id='ticketId', sketch_id=1234, - analyzers=None + analyzers=None, + token_password='blah', + endpoint=None, + username=None, + password=None, ) self.assertEqual( @@ -75,9 +83,13 @@ def testNewSketchCreation(self, mock_GetApiClient, _): test_state.recipe = {'name': 'test_recipe'} timesketch_exporter = timesketch.TimesketchExporter(test_state) timesketch_exporter.SetUp( - incident_id=None, + incident_id='ticketId', sketch_id=None, - analyzers=None + analyzers=None, + token_password='blah', + endpoint=None, + username=None, + password=None, ) test_container = containers.File('file.ext', '/tmp/file.ext') timesketch_exporter.PreProcess() @@ -85,7 +97,7 @@ def testNewSketchCreation(self, mock_GetApiClient, _): timesketch_exporter.PostProcess() self.assertEqual(timesketch_exporter.sketch_id, 1234) mock_api_client.create_sketch.assert_called_with( - 'Untitled sketch', 'Sketch generated by dfTimewolf') + 'Sketch for incident ID: ticketId', 'Sketch generated by dfTimewolf') report = timesketch_exporter.GetContainers(containers.Report)[0] self.assertEqual(report.module_name, 'TimesketchExporter') self.assertEqual( @@ -124,10 +136,14 @@ def testWaitForTimeline(self, test_state.recipe = {'name': 'test_recipe'} timesketch_exporter = timesketch.TimesketchExporter(test_state) timesketch_exporter.SetUp( - incident_id=None, + incident_id='ticketId', sketch_id=None, analyzers=None, - wait_for_timelines=True + wait_for_timelines=True, + token_password='blah', + endpoint=None, + username=None, + password=None, ) test_container = containers.File('file.ext', '/tmp/file.ext') timesketch_exporter.PreProcess()