From f5a6bfebfe7f05fadda3cdeb348d4608df127dd7 Mon Sep 17 00:00:00 2001 From: Saksham Gupta Date: Fri, 29 Jan 2021 20:17:56 +0530 Subject: [PATCH 1/5] Add table_name config - Useful when using single partitoned table - Doesn't create a new table & appends data to the given table --- lib/logstash/outputs/google_bigquery.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/logstash/outputs/google_bigquery.rb b/lib/logstash/outputs/google_bigquery.rb index 1adc2c4..dbdfe35 100644 --- a/lib/logstash/outputs/google_bigquery.rb +++ b/lib/logstash/outputs/google_bigquery.rb @@ -84,6 +84,10 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base # date suffix. config :table_separator, validate: :string, default: '_' + # BigQuery table name to be used when inserting data into an existing table + # (Useful when using partitioned table) + config :table_name, validate: :string, default: nil + # Schema for log data. It must follow the format `name1:type1(,name2:type2)*`. # For example, `path:STRING,status:INTEGER,score:FLOAT`. config :csv_schema, validate: :string, required: false, default: nil @@ -179,7 +183,7 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base def register @logger.debug('Registering plugin') - @schema = LogStash::Outputs::BigQuery::Schema.parse_csv_or_json @csv_schema, @json_schema + @schema = LogStash::Outputs::BigQuery::Schema.parse_csv_or_json @csv_schema, @json_schema if @table_name.nil? || @table_name.empty? @bq_client = LogStash::Outputs::BigQuery::StreamingClient.new @json_key_file, @project_id, @logger @batcher = LogStash::Outputs::BigQuery::Batcher.new @batch_size, @batch_size_bytes @stopping = Concurrent::AtomicBoolean.new(false) @@ -202,6 +206,7 @@ def receive(event) end def get_table_name(time=nil) + return table_name unless table_name.nil? || table_name.empty? time ||= Time.now str_time = time.strftime(@date_pattern) @@ -236,7 +241,7 @@ def publish(messages) table = get_table_name @logger.info("Publishing #{messages.length} messages to #{table}") - create_table_if_not_exists table + create_table_if_not_exists table if @table_name.nil? || @table_name.empty? failed_rows = @bq_client.append(@dataset, table, messages, @ignore_unknown_values, @skip_invalid_rows) write_to_errors_file(failed_rows, table) unless failed_rows.empty? From 3505b80b829af4465f655e5b106b35d70376459e Mon Sep 17 00:00:00 2001 From: Saksham Gupta Date: Mon, 1 Feb 2021 18:42:55 +0530 Subject: [PATCH 2/5] Add table_name in index.asciidoc --- docs/index.asciidoc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index b23030b..40008f6 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -340,6 +340,15 @@ BigQuery table separator to be added between the table_prefix and the date suffix. [id="plugins-{type}s-{plugin}-temp_directory"] +===== `table_name` + + * Value type is <> + * Default value is `nil` + +BigQuery table name to be used when inserting data into an existing table. +(Useful when using partitioned table) + +[id="plugins-{type}s-{plugin}-table_name"] ===== `temp_directory` deprecated[4.0.0, Events are uploaded in real-time without being stored to disk.] From 4d97e1a17a9529f80777ed201d591ccac365591c Mon Sep 17 00:00:00 2001 From: Saksham Gupta Date: Mon, 1 Feb 2021 18:54:35 +0530 Subject: [PATCH 3/5] Minor change --- lib/logstash/outputs/google_bigquery.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/outputs/google_bigquery.rb b/lib/logstash/outputs/google_bigquery.rb index dbdfe35..a063c58 100644 --- a/lib/logstash/outputs/google_bigquery.rb +++ b/lib/logstash/outputs/google_bigquery.rb @@ -86,7 +86,7 @@ class LogStash::Outputs::GoogleBigQuery < LogStash::Outputs::Base # BigQuery table name to be used when inserting data into an existing table # (Useful when using partitioned table) - config :table_name, validate: :string, default: nil + config :table_name, validate: :string, required: false, default: nil # Schema for log data. It must follow the format `name1:type1(,name2:type2)*`. # For example, `path:STRING,status:INTEGER,score:FLOAT`. From 03544c9893e201732cace4379e28b17b23b6ab8a Mon Sep 17 00:00:00 2001 From: Saksham Gupta Date: Mon, 1 Feb 2021 19:31:47 +0530 Subject: [PATCH 4/5] Minor change --- lib/logstash/outputs/google_bigquery.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/outputs/google_bigquery.rb b/lib/logstash/outputs/google_bigquery.rb index a063c58..17b7e3f 100644 --- a/lib/logstash/outputs/google_bigquery.rb +++ b/lib/logstash/outputs/google_bigquery.rb @@ -206,7 +206,7 @@ def receive(event) end def get_table_name(time=nil) - return table_name unless table_name.nil? || table_name.empty? + return @table_name unless @table_name.nil? || @table_name.empty? time ||= Time.now str_time = time.strftime(@date_pattern) From e231f92ff179484c51cda7efae41237e483acdb4 Mon Sep 17 00:00:00 2001 From: Saksham Gupta Date: Tue, 27 Jul 2021 15:17:11 +0530 Subject: [PATCH 5/5] Fix table_name doc --- docs/index.asciidoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 40008f6..ceed08b 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -103,6 +103,7 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No +| <> |<>|No | <> |<>|__Deprecated__ | <> |<>|__Deprecated__ | <> |<>|__Deprecated__ @@ -339,7 +340,7 @@ Table name will be `` BigQuery table separator to be added between the table_prefix and the date suffix. -[id="plugins-{type}s-{plugin}-temp_directory"] +[id="plugins-{type}s-{plugin}-table_name"] ===== `table_name` * Value type is <> @@ -348,7 +349,7 @@ date suffix. BigQuery table name to be used when inserting data into an existing table. (Useful when using partitioned table) -[id="plugins-{type}s-{plugin}-table_name"] +[id="plugins-{type}s-{plugin}-temp_directory"] ===== `temp_directory` deprecated[4.0.0, Events are uploaded in real-time without being stored to disk.]