More options for Timesketch auth (user/pass) (#794)

* docker release

* Some changes

* release docker container

* Add userpass login to the TS module / recipes

* fix recipe params

* slower but more stable

* image name

* oops

* change image name

* dockerfile on branch

* Update all recipes

* Fix tests

* silence linter

* Fix type

* remove docker files from PR

* Appease pylint

* Add docstrings

* Add default dev endpoint for timesketch

* fix test

* Fix pylint

data/recipes/aws_logging_ts.json

@@ -32,6 +32,9 @@
         "args": {
             "incident_id": "@incident_id",
             "token_password": "@token_password",
+            "endpoint": "@timesketch_endpoint",
+            "username": "@timesketch_username",
+            "password": "@timesketch_password",
             "sketch_id": "@sketch_id",
             "analyzers": null,
             "wait_for_timelines": "@wait_for_timelines"
@@ -45,6 +48,9 @@
         ["--end_time", "End time for the query.", null, {"format": "datetime", "format_string": "%Y-%m-%d %H:%M:%S", "after": "@start_time"}],
         ["--incident_id", "Incident ID (used for Timesketch description).", null],
         ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
+        ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+        ["--timesketch_username", "Username for Timesketch server.", null],
+        ["--timesketch_password", "Password for Timesketch server.", null],
         ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
         ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true]
     ]

data/recipes/aws_turbinia_ts.json

@@ -79,6 +79,9 @@
       "args": {
         "incident_id": "@incident_id",
         "token_password": "@token_password",
+        "endpoint": "@timesketch_endpoint",
+        "username": "@timesketch_username",
+        "password": "@timesketch_password",
         "sketch_id": "@sketch_id",
         "analyzers": null,
         "wait_for_timelines": "@wait_for_timelines"
@@ -96,6 +99,9 @@
     ["--aws_profile", "Source AWS profile.", null],
     ["--incident_id", "Incident ID (used for Timesketch description).", null],
     ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
+    ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+    ["--timesketch_username", "Username for Timesketch server.", null],
+    ["--timesketch_password", "Password for Timesketch server.", null],
     ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
     ["--request_ids", "Comma separated Turbinia request identifiers to process.", null, {"format": "regex", "comma_separated": true, "regex": "^[a-f0-9]{32}$"}],
     ["--turbinia_recipe", "The Turbinia recipe name to use for evidence processing.", null],

data/recipes/azure_logging_ts.json

@@ -24,6 +24,9 @@
         "args": {
             "incident_id": "@incident_id",
             "token_password": "@token_password",
+            "endpoint": "@timesketch_endpoint",
+            "username": "@timesketch_username",
+            "password": "@timesketch_password",
             "sketch_id": "@sketch_id",
             "analyzers": null,
             "wait_for_timelines": "@wait_for_timelines"
@@ -35,6 +38,9 @@
         ["--profile_name", "A profile name to use when looking for Azure credentials.", null],
         ["--incident_id", "Incident ID (used for Timesketch description).", null],
         ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
+        ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+        ["--timesketch_username", "Username for Timesketch server.", null],
+        ["--timesketch_password", "Password for Timesketch server.", null],
         ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
         ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true]
     ]

data/recipes/bigquery_ts.json

@@ -25,6 +25,9 @@
             "args": {
                 "incident_id": "@incident_id",
                 "token_password": "@token_password",
+                "endpoint": "@timesketch_endpoint",
+                "username": "@timesketch_username",
+                "password": "@timesketch_password",
                 "sketch_id": "@sketch_id",
                 "analyzers": null,
                 "wait_for_timelines": "@wait_for_timelines"
@@ -36,6 +39,9 @@
             ["description", "Human-readable description of the query.", null],
             ["--incident_id", "Incident ID (used for Timesketch description).", null],
             ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
+            ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+            ["--timesketch_username", "Username for Timesketch server.", null],
+            ["--timesketch_password", "Password for Timesketch server.", null],
             ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
             ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true]
         ]

data/recipes/gcp_logging_cloudaudit_ts.json

@@ -31,6 +31,9 @@
         "args": {
             "incident_id": "@incident_id",
             "token_password": "@token_password",
+            "endpoint": "@timesketch_endpoint",
+            "username": "@timesketch_username",
+            "password": "@timesketch_password",
             "sketch_id": "@sketch_id",
             "analyzers": null,
             "wait_for_timelines": "@wait_for_timelines"
@@ -42,6 +45,9 @@
         ["end_date", "End date (yyyy-mm-ddTHH:MM:SSZ).", null, {"format": "datetime", "format_string": "%Y-%m-%dT%H:%M:%SZ", "after": "@start_date"}],
         ["--incident_id", "Incident ID (used for Timesketch description).", null],
         ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
+        ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+        ["--timesketch_username", "Username for Timesketch server.", null],
+        ["--timesketch_password", "Password for Timesketch server.", null],
         ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
         ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true],
         ["--backoff", "If GCP Cloud Logging API query limits are exceeded, retry with an increased delay between each query to try complete the query at a slower rate.", false],

data/recipes/gcp_logging_cloudsql_ts.json

@@ -34,6 +34,9 @@
       "args": {
         "incident_id": "@incident_id",
         "token_password": "@token_password",
+        "endpoint": "@timesketch_endpoint",
+        "username": "@timesketch_username",
+        "password": "@timesketch_password",
         "sketch_id": "@sketch_id",
         "analyzers": null,
         "wait_for_timelines": "@wait_for_timelines"
@@ -46,6 +49,9 @@
     ["end_date", "End date (yyyy-mm-ddTHH:MM:SSZ).", null, {"format": "datetime", "format_string": "%Y-%m-%dT%H:%M:%SZ", "after": "@start_date"}],
     ["--incident_id", "Incident ID (used for Timesketch description).", null],
     ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
+    ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+    ["--timesketch_username", "Username for Timesketch server.", null],
+    ["--timesketch_password", "Password for Timesketch server.", null],
     ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
     ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true],
     ["--backoff", "If GCP Cloud Logging API query limits are exceeded, retry with an increased delay between each query to try complete the query at a slower rate.", false],

data/recipes/gcp_logging_gce_instance_ts.json

@@ -31,6 +31,9 @@
         "args": {
             "incident_id": "@incident_id",
             "token_password": "@token_password",
+            "endpoint": "@timesketch_endpoint",
+            "username": "@timesketch_username",
+            "password": "@timesketch_password",
             "sketch_id": "@sketch_id",
             "analyzers": null,
             "wait_for_timelines": "@wait_for_timelines"
@@ -41,6 +44,9 @@
         ["instance_id", "Identifier for GCE instance (Instance ID).", null, {"format": "regex", "regex": "^[a-z][-a-z0-9]{0,61}[a-z0-9]?$"}],
         ["--incident_id", "Incident ID (used for Timesketch description).", null],
         ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
+        ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+        ["--timesketch_username", "Username for Timesketch server.", null],
+        ["--timesketch_password", "Password for Timesketch server.", null],
         ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
         ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true],
         ["--backoff", "If GCP Cloud Logging API query limits are exceeded, retry with an increased delay between each query to try complete the query at a slower rate.", false],

data/recipes/gcp_logging_gce_ts.json

@@ -32,6 +32,9 @@
             "incident_id": "@incident_id",
             "analyzers": null,
             "token_password": "@token_password",
+            "endpoint": "@timesketch_endpoint",
+            "username": "@timesketch_username",
+            "password": "@timesketch_password",
             "sketch_id": "@sketch_id",
             "wait_for_timelines": "@wait_for_timelines"
         }
@@ -42,6 +45,9 @@
         ["end_date", "End date (yyyy-mm-ddTHH:MM:SSZ).", null, {"format": "datetime", "format_string": "%Y-%m-%dT%H:%M:%SZ", "after": "@start_date"}],
         ["--incident_id", "Incident ID (used for Timesketch description).", null],
         ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
+        ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+        ["--timesketch_username", "Username for Timesketch server.", null],
+        ["--timesketch_password", "Password for Timesketch server.", null],
         ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
         ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true],
         ["--backoff", "If GCP Cloud Logging API query limits are exceeded, retry with an increased delay between each query to try complete the query at a slower rate.", false],

data/recipes/gcp_logging_ts.json

@@ -30,6 +30,9 @@
       "args": {
         "incident_id": "@incident_id",
         "token_password": "@token_password",
+        "endpoint": "@timesketch_endpoint",
+        "username": "@timesketch_username",
+        "password": "@timesketch_password",
         "sketch_id": "@sketch_id",
         "analyzers": "@analyzers",
         "wait_for_timelines": "@wait_for_timelines"
@@ -43,6 +46,9 @@
     ["--delay", "Number of seconds to wait between each GCP Cloud Logging query to avoid hitting API query limits", 0, {"format": "regex", "regex": "^\\d+$"}],
     ["--analyzers", "Timesketch analyzers to run.", null],
     ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
+    ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+    ["--timesketch_username", "Username for Timesketch server.", null],
+    ["--timesketch_password", "Password for Timesketch server.", null],
     ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
     ["--incident_id", "Incident ID (used for Timesketch description).", null],
     ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true]

data/recipes/gcp_turbinia_disk_copy_ts.json

@@ -70,6 +70,9 @@
         "args": {
             "incident_id": "@incident_id",
             "token_password": "@token_password",
+            "endpoint": "@timesketch_endpoint",
+            "username": "@timesketch_username",
+            "password": "@timesketch_password",
             "sketch_id": "@sketch_id",
             "analyzers": null,
             "wait_for_timelines": "@wait_for_timelines"
@@ -86,6 +89,9 @@
         ["--turbinia_api", "Turbinia API server endpoint.", "http://127.0.0.1:8000"],
         ["--incident_id", "Incident ID (used for Timesketch description and to label the VM with).", null],
         ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
+        ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+        ["--timesketch_username", "Username for Timesketch server.", null],
+        ["--timesketch_password", "Password for Timesketch server.", null],
         ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
         ["--create_analysis_vm", "Create an analysis VM in the destination project.", true],
         ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true],

data/recipes/gcp_turbinia_ts.json

@@ -29,6 +29,9 @@
         "args": {
             "incident_id": "@incident_id",
             "token_password": "@token_password",
+            "endpoint": "@timesketch_endpoint",
+            "username": "@timesketch_username",
+            "password": "@timesketch_password",
             "sketch_id": "@sketch_id",
             "analyzers": null,
             "wait_for_timelines": "@wait_for_timelines"
@@ -41,6 +44,9 @@
         ["--request_ids", "Comma separated Turbinia request identifiers to process. This parameter can only be used if --disk_names is not provided.", null, {"format": "regex", "comma_separated": true, "regex": "^[a-f0-9]{32}$"}],
         ["--incident_id", "Incident ID (used for Timesketch description).", null],
         ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
+        ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+        ["--timesketch_username", "Username for Timesketch server.", null],
+        ["--timesketch_password", "Password for Timesketch server.", null],
         ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
         ["--turbinia_recipe", "The Turbinia recipe name to use for evidence processing.", null],
         ["--turbinia_auth", "Flag to indicate whether Turbinia API server requires authentication.", false],

data/recipes/grr_artifact_ts.json

@@ -24,14 +24,17 @@
         "name": "LocalPlasoProcessor",
         "args": {
             "timezone": null,
-            "use_docker": true
+            "use_docker": "@user_docker"
         }
     }, {
         "wants": ["LocalPlasoProcessor"],
         "name": "TimesketchExporter",
         "args": {
             "incident_id": "@reason",
-            "token_password": "@token_password",
+            "token_password": null,
+            "endpoint": "@timesketch_endpoint",
+            "username": "@timesketch_username",
+            "password": "@timesketch_password",
             "sketch_id": "@sketch_id",
             "analyzers": "@analyzers",
             "wait_for_timelines": "@wait_for_timelines"
@@ -48,12 +51,16 @@
         ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true],
         ["--analyzers", "Timesketch analyzers to run", null],
         ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
+        ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+        ["--timesketch_username", "Username for Timesketch server.", null],
+        ["--timesketch_password", "Password for Timesketch server.", null],
         ["--incident_id", "Incident ID (used for Timesketch description).", null],
         ["--grr_server_url", "GRR endpoint.", "http://localhost:8000", {"format": "url"}],
         ["--verify", "Whether to verify the GRR TLS certificate.", true],
         ["--skip_offline_clients", "Whether to skip clients that are offline.", false],
         ["--grr_username", "GRR username", "admin"],
         ["--grr_password", "GRR password", "admin"],
+        ["--user_docker", "Whether the LocalPlasoProcessor should use Docker or not.", true],
         ["--max_file_size", "Maximum size of files to collect (in bytes).", 5368709120, {"format": "regex", "regex": "^\\d+$"}]
     ]
 }

data/recipes/grr_huntresults_ts.json

@@ -27,6 +27,9 @@
         "args": {
             "incident_id": "@reason",
             "token_password": "@token_password",
+            "endpoint": "@timesketch_endpoint",
+            "username": "@timesketch_username",
+            "password": "@timesketch_password",
             "sketch_id": "@sketch_id",
             "analyzers": null,
             "wait_for_timelines": "@wait_for_timelines"
@@ -36,6 +39,9 @@
         ["hunt_id", "ID of GRR Hunt results to fetch.", null, {"format": "regex", "comma_separated": true, "regex": "^[0-9A-F]{16}$"}],
         ["reason", "Reason for exporting hunt (used for Timesketch description).", null],
         ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
+        ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+        ["--timesketch_username", "Username for Timesketch server.", null],
+        ["--timesketch_password", "Password for Timesketch server.", null],
         ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
         ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true],
         ["--approvers", "Emails for GRR approval request.", null],

data/recipes/grr_timeline_ts.json

@@ -23,7 +23,7 @@
     "name": "LocalPlasoProcessor",
     "args": {
       "timezone": null,
-      "use_docker": true
+      "use_docker": "@use_docker"
     }
   }, {
     "wants": ["LocalPlasoProcessor"],
@@ -33,7 +33,10 @@
       "token_password": "@token_password",
       "sketch_id": "@sketch_id",
       "analyzers": null,
-      "wait_for_timelines": "@wait_for_timelines"
+      "wait_for_timelines": "@wait_for_timelines",
+      "endpoint": "@timesketch_endpoint",
+      "username": "@timesketch_username",
+      "password": "@timesketch_password"
     }
   }, {
     "wants": ["TimesketchExporter"],
@@ -45,7 +48,10 @@
       "include_stories": false,
       "token_password": "@token_password",
       "max_checks": 0,
-      "formatter": "html"
+      "formatter": "html",
+      "endpoint": "@timesketch_endpoint",
+      "username": "@timesketch_username",
+      "password": "@timesketch_password"
     }
   }],
   "args": [
@@ -56,10 +62,14 @@
     ["--approvers", "Comma-separated list of usernames to ask for approval.", null],
     ["--sketch_id", "Timesketch sketch to which the timeline should be added.", null, {"format": "regex", "regex": "^\\d+$"}],
     ["--grr_server_url", "GRR endpoint.", "http://localhost:8000", {"format": "url"}],
+    ["--timesketch_endpoint", "Timesketch endpoint", "http://localhost:5000/"],
+    ["--timesketch_username", "Username for Timesketch server.", null],
+    ["--timesketch_password", "Password for Timesketch server.", null],
     ["--token_password", "Optional custom password to decrypt Timesketch credential file with.", ""],
     ["--timesketch_quick", "Skip waiting for analyzers to complete their run.", false],
     ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true],
     ["--grr_username", "GRR username.", "admin"],
-    ["--grr_password", "GRR password.", "admin"]
+    ["--grr_password", "GRR password.", "admin"],
+    ["--user_docker", "Whether the LocalPlasoProcessor should use Docker or not.", true]
   ]
 }