diff --git a/docs/content/api/modules.json.gz b/docs/content/api/modules.json.gz index 3930da2c16840..b3304c506e641 100644 Binary files a/docs/content/api/modules.json.gz and b/docs/content/api/modules.json.gz differ diff --git a/docs/content/api/searchindex.json.gz b/docs/content/api/searchindex.json.gz index 1faf44f410954..9410b61bf844b 100644 Binary files a/docs/content/api/searchindex.json.gz and b/docs/content/api/searchindex.json.gz differ diff --git a/docs/content/api/sections.json.gz b/docs/content/api/sections.json.gz index aa1367b7d9f80..faa9cc8d64d64 100644 Binary files a/docs/content/api/sections.json.gz and b/docs/content/api/sections.json.gz differ diff --git a/docs/content/concepts/dagster-pipes/aws-glue.mdx b/docs/content/concepts/dagster-pipes/aws-glue.mdx index 6c4c584597ff0..8ecdc3b9e1eb4 100644 --- a/docs/content/concepts/dagster-pipes/aws-glue.mdx +++ b/docs/content/concepts/dagster-pipes/aws-glue.mdx @@ -46,24 +46,26 @@ Call `open_dagster_pipes` in the Glue job script to create a context that can be import boto3 from dagster_pipes import ( PipesCliArgsParamsLoader, + PipesDefaultMessageWriter, PipesS3ContextLoader, - PipesS3MessageWriter, open_dagster_pipes, ) client = boto3.client("s3") context_loader = PipesS3ContextLoader(client) -message_writer = PipesS3MessageWriter(client) params_loader = PipesCliArgsParamsLoader() def main(): with open_dagster_pipes( context_loader=context_loader, - message_writer=message_writer, params_loader=params_loader, ) as pipes: pipes.log.info("Hello from AWS Glue job!") + pipes.report_asset_materialization( + metadata={"some_metric": {"raw_value": 0, "type": "int"}}, + data_version="alpha", + ) if __name__ == "__main__": @@ -107,7 +109,7 @@ Next, add the `PipesGlueClient` resource to your project's None: + """Reads logs from AWS CloudWatch and forwards them to Dagster for events extraction and logging. + + Args: + log_group (str): CloudWatch log group name + log_stream (str): CLoudWatch log stream name + start_time (Optional[int]): The start of the time range, expressed as the number of + milliseconds after Jan 1, 1970 00:00:00 UTC. Events with a timestamp equal to this + time or later than this time are included. + Events with a timestamp earlier than this time are not included. + end_time (Optional[int]): The end of the time range, expressed as the number of + milliseconds after Jan 1, 1970 00:00:00 UTC. Events with a timestamp equal to or + later than this time are not included. + """ handler = check.not_none( self._handler, "Can only consume logs within context manager scope." ) @@ -216,7 +229,7 @@ def consume_cloudwatch_logs( extract_message_or_forward_to_stdout(handler, log_line) def no_messages_debug_text(self) -> str: - return "Attempted to read messages by extracting them from the tail of CloudWatch logs directly." + return "Attempted to read messages by extracting them from CloudWatch logs directly." def _get_all_cloudwatch_events( self,