Add example errors connector and move simple example connector to a different namespace

Gemfile

@@ -59,5 +59,7 @@ gem 'elasticsearch', '~> 8.5.0'
 # Dependencies for oauth
 gem 'signet', '~> 0.16.0'
 
+# Dependency for example connector
+gem 'faker', '~> 2.22.0'
 # Dependency for mongo connector
 gem 'mongo', '~> 2.18'

Gemfile.lock

@@ -68,6 +68,8 @@ GEM
       multi_json
     et-orbi (1.2.7)
       tzinfo
+    faker (2.22.0)
+      i18n (>= 1.8.11, < 2)
     faraday (1.10.0)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
@@ -220,6 +222,7 @@ DEPENDENCIES
   dry-validation (= 1.7.0)
   ecs-logging (~> 1.0.0)
   elasticsearch (~> 8.5.0)
+  faker (~> 2.22.0)
   faraday (~> 1.10.0)
   faraday_middleware (= 1.0.0)
   forwardable (~> 1.3.2)

VERSION

@@ -1 +1 @@
-8.7.0.0
+8.6.0.0

lib/connectors/example/simple/connector.rb

@@ -0,0 +1,98 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/base/connector'
+require 'core/filtering/validation_status'
+require 'utility'
+
+module Connectors
+  module Example
+    module Simple
+      class Connector < Connectors::Base::Connector
+        def self.service_type
+          'example'
+        end
+
+        def self.display_name
+          'Example Connector'
+        end
+
+        # Field 'Foo' won't have a default value. Field 'Bar' will have the default value 'Value'.
+        def self.configurable_fields
+          {
+            'foo' => {
+              'label' => 'Foo',
+              'value' => nil
+            },
+            :bar => {
+              :label => 'Bar',
+              :value => 'Value'
+            }
+          }
+        end
+
+        def initialize(configuration: {}, job_description: {})
+          super
+        end
+
+        def do_health_check
+          # Do the health check by trying to access 3rd-party system just to verify that everything is set up properly.
+          #
+          # To emulate unhealthy 3rd-party system situation, uncomment the following line:
+          # raise 'something went wrong'
+        end
+
+        def self.validate_filtering(filtering = {})
+          # TODO: real filtering validation will follow later
+          errors = [
+            {
+              :ids => ['missing-implementation'],
+              :messages => ['Filtering is not implemented yet for the example connector']
+            }
+          ]
+
+          return { :state => Core::Filtering::ValidationStatus::INVALID, :errors => errors } if filtering.present?
+
+          { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+        end
+
+        def yield_documents
+          attachments = [
+            load_attachment('first_attachment.txt'),
+            load_attachment('second_attachment.txt'),
+            load_attachment('third_attachment.txt'),
+          ]
+
+          attachments.each_with_index do |att, index|
+            data = { id: (index + 1).to_s, name: "example document #{index + 1}", _attachment: File.read(att) }
+
+            # Uncomment one of these two lines to simulate longer running sync jobs
+            #
+            # sleep(rand(10..60).seconds)
+            # sleep(rand(1..10).minutes)
+
+            yield data
+          end
+        end
+
+        private
+
+        def load_attachment(path)
+          attachment_dir = "#{File.dirname(__FILE__)}/attachments"
+          attachment_path = "#{attachment_dir}/#{path}"
+
+          unless File.exist?(attachment_path)
+            raise "Attachment at location '#{attachment_path}' doesn't exist. Attachments should be located under #{attachment_dir}"
+          end
+
+          File.open(attachment_path)
+        end
+      end
+    end
+  end
+end

lib/connectors/example/with_errors/connector.rb

@@ -0,0 +1,71 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+# frozen_string_literal: true
+
+require 'connectors/base/connector'
+require 'core/filtering/validation_status'
+require 'utility'
+require 'faker'
+
+module Connectors
+  module Example
+    module WithErrors
+      class Connector < Connectors::Base::Connector
+        def self.service_type
+          'example-with-errors'
+        end
+
+        def self.display_name
+          'Example Connector that produces transient errors'
+        end
+
+        # Field 'Foo' won't have a default value. Field 'Bar' will have the default value 'Value'.
+        def self.configurable_fields
+          {
+            'chance_to_raise' => {
+              'label' => 'Chance to raise an error when extracting a document (0..100)',
+              'value' => 1
+            },
+            'generated_document_count' => {
+              'label' => 'Number of documents to generate',
+              'value' => 10_000
+            }
+          }
+        end
+
+        def initialize(configuration: {}, job_description: {})
+          super
+
+          @chance_to_raise = configuration.dig('chance_to_raise', 'value').to_i
+          @generated_document_count = configuration.dig('generated_document_count', 'value').to_i
+
+          raise 'Invalid chance to raise: should be between 0 and 100' if @chance_to_raise < 0 || @chance_to_raise > 100
+
+          Faker::Config.random = Random.new(1337) # we want to have a seed to consistently generate same text over and over
+        end
+
+        def do_health_check; end
+
+        def self.validate_filtering(_filtering = {})
+          { :state => Core::Filtering::ValidationStatus::VALID, :errors => [] }
+        end
+
+        def yield_documents
+          @generated_document_count.times.map do |i|
+            yield_with_handling_tolerable_errors do
+              raise 'that is a random exception' if rand(1..100) > (100 - @chance_to_raise)
+
+              document = { 'id' => i, 'name' => Faker::Name.name, 'text' => Faker::Lorem.sentence }
+
+              yield document
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/connectors/registry.rb

@@ -39,8 +39,11 @@ def registered_connectors
 
   REGISTRY = Factory.new
 
-  require_relative './example/connector'
-  REGISTRY.register(Connectors::Example::Connector.service_type, Connectors::Example::Connector)
+  require_relative './example/simple/connector'
+  REGISTRY.register(Connectors::Example::Simple::Connector.service_type, Connectors::Example::Simple::Connector)
+
+  require_relative './example/with_errors/connector'
+  REGISTRY.register(Connectors::Example::WithErrors::Connector.service_type, Connectors::Example::WithErrors::Connector)
 
   # loading plugins (might replace this with a directory scan and conventions on names)
   require_relative './gitlab/connector'

spec/connectors/example/connector_spec.rb → spec/connectors/example/simple/connector_spec.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require 'connectors/example/connector'
+require 'connectors/example/simple/connector'
 require 'spec_helper'
 
-describe Connectors::Example::Connector do
+describe Connectors::Example::Simple::Connector do
   subject { described_class.new(configuration: configuration) }
   let(:configuration) do
     {